diff --git a/system/kernel/acpi.zig b/system/kernel/acpi.zig index b2b7656..09534e5 100644 --- a/system/kernel/acpi.zig +++ b/system/kernel/acpi.zig @@ -99,9 +99,12 @@ pub const PlatformInformation = struct { /// and the kernel says so at every boot (the fail-open platform log line). When /// true, the IOMMU core builds per-device translation domains from this record. iommu_present: bool = false, - /// MMIO base of the selected DMA-remapping hardware unit (the DRHD with - /// INCLUDE_PCI_ALL — the catch-all unit covering every device not scoped to a - /// more specific one; falls back to the first unit when none carries the flag). + /// true when the present unit is AMD-Vi (from IVRS) rather than Intel VT-d (DMAR). + /// The two are mutually exclusive on real hardware; the IOMMU core picks the backend. + iommu_is_amd: bool = false, + /// MMIO base of the selected DMA-remapping hardware unit — the VT-d DRHD with + /// INCLUDE_PCI_ALL (the catch-all unit; falls back to the first), or the AMD-Vi + /// IOMMU's control-register base from the first IVHD. iommu_base: u64 = 0, /// The unit's Version register (offset 0x00) — its low byte is major.minor; /// reading it back nonzero confirms a real, mappable VT-d unit. @@ -279,6 +282,8 @@ const SLIT: [4]u8 = "SLIT".*; const SRAT: [4]u8 = "SRAT".*; /// Secondary System Description Table (SSDT) const DMAR: [4]u8 = "DMAR".*; +/// I/O Virtualization Reporting Structure (IVRS) — the AMD-Vi analogue of DMAR. +const IVRS: [4]u8 = "IVRS".*; const SSDT: [4]u8 = "SSDT".*; /// Serial Port Console Redirection table (SPCR) — the firmware's console UART. const SPCR: [4]u8 = "SPCR".*; @@ -498,6 +503,8 @@ fn handleTable(device_tree: *DeviceTree, hal: Hal, sdt_physical: u64) !void { parseSpcr(header); } else if (std.mem.eql(u8, &sig, &DMAR)) { parseDmar(hal, header); + } else if (std.mem.eql(u8, &sig, &IVRS)) { + parseIvrs(header); } else if (std.mem.eql(u8, &sig, &SSDT)) { // Secondary namespace bytecode — collect it to publish for the ring-3 parse. addAmlBlock(sdt_physical); @@ -885,6 +892,40 @@ fn parseRmrr(base: [*]align(1) const u8, total: usize, off: usize, length: usize } } +// IVRS layout (AMD I/O Virtualization spec): 36-byte ACPI header, IVinfo u32 @36, +// 8 reserved @40, then IVHD/IVMD blocks from @48. An IVHD common header is type u8 @0, +// flags u8 @1, length u16 @2, device id u16 @4, capability offset u16 @6, IOMMU base +// address u64 @8, PCI segment u16 @16, IOMMU info u16 @18. +const ivrs_blocks_offset = 48; +const ivhd_type_10: u8 = 0x10; +const ivhd_type_11: u8 = 0x11; +const ivhd_base_offset = 8; + +/// IVRS -> detect an AMD-Vi IOMMU. Record the control-register base from the first IVHD +/// of type 0x10/0x11. Per-device entries and IVMD (the AMD analogue of RMRR) are ignored +/// in v1 — the default-deny device table is what we build anyway, and QEMU emits no IVMD; +/// a real machine that needs them is flagged untested on AMD regardless. +fn parseIvrs(header: *const SystemDescriptorTableHeader) void { + const base: [*]align(1) const u8 = @ptrCast(header); + const total: usize = header.length; + var off: usize = ivrs_blocks_offset; + while (off + 4 <= total) { + const kind = fadt(u8, base, total, off) orelse break; + const length = fadt(u16, base, total, off + 2) orelse break; + if (length < 4 or off + length > total) break; + if (kind == ivhd_type_10 or kind == ivhd_type_11) { + const iommu_base = fadt(u64, base, total, off + ivhd_base_offset) orelse 0; + if (iommu_base != 0) { + platform_information.iommu_present = true; + platform_information.iommu_is_amd = true; + platform_information.iommu_base = iommu_base; + return; // first IVHD is enough; multi-unit is future work + } + } + off += length; + } +} + // --- helpers ---------------------------------------------------------------- /// Sum `len` bytes; an ACPI table/pointer is valid when the low 8 bits are zero. diff --git a/system/kernel/iommu-amd.zig b/system/kernel/iommu-amd.zig new file mode 100644 index 0000000..1a8b6a7 --- /dev/null +++ b/system/kernel/iommu-amd.zig @@ -0,0 +1,259 @@ +//! system/kernel/iommu-amd.zig — AMD-Vi (AMD I/O Virtualization) backend for the IOMMU +//! core. The AMD analogue of iommu-intel.zig: it supplies the core's `Backend` vtable +//! with AMD-Vi's page-table entry bits and drives the device table, command buffer, and +//! event log. +//! +//! **UNTESTED on real AMD hardware.** danos is developed on Intel; this backend is +//! validated only against QEMU's `-device amd-iommu,dma-remap=on`, whose AMD-Vi +//! emulation is far less exercised than its Intel one. Every code path here should be +//! read as "QEMU-verified, real-AMD-unverified" until an AMD machine confirms it. +//! +//! Interrupt remapping is left off (the DTE forwards interrupts unmapped), so MSI writes +//! to the 0xFEE00000 range reach the APIC untranslated, exactly as on the Intel path. + +const std = @import("std"); +const abi = @import("abi"); +const boot_handoff = @import("boot-handoff"); +const pmm = @import("pmm.zig"); +const platform = @import("platform"); +const architecture = @import("architecture"); +const log = @import("log.zig"); +const iommu = @import("iommu.zig"); + +const page_size = abi.page_size; + +// MMIO register offsets from the IOMMU control-register base. +const reg_device_table_base = 0x00; // base | (pages-1) in bits 8:0 +const reg_command_buffer_base = 0x08; // base | (ComLen << 56) +const reg_event_log_base = 0x10; // base | (EventLen << 56) +const reg_control = 0x18; +const reg_command_head = 0x2000; +const reg_command_tail = 0x2008; +const reg_event_head = 0x2010; +const reg_event_tail = 0x2018; +const reg_status = 0x2020; + +const control_iommu_enable: u64 = 1 << 0; +const control_event_log_enable: u64 = 1 << 2; +const control_command_buffer_enable: u64 = 1 << 12; + +// Device table: one 32-byte DTE (4 qwords) per requester id, indexed by bdf. +const device_table_pages = 512; // 2 MiB = 65536 entries (a full 256-bus segment) +const dte_qwords = 4; +const dte_valid: u64 = 1 << 0; // V +const dte_translation_valid: u64 = 1 << 1; // TV +const dte_mode_shift = 9; // bits 11:9 — page-table levels +const dte_read: u64 = 1 << 61; // IR +const dte_write: u64 = 1 << 62; // IW +const dte_intctl_forward: u64 = @as(u64, 1) << 60; // qword2 bits 61:60 = 01b: forward interrupts unmapped + +// Page-table entry bits (AMD native format). +const pte_present: u64 = 1 << 0; // PR +const pte_next_level_shift = 9; // bits 11:9: 0 = leaf, N = pointer to a level-N table +const pte_read: u64 = 1 << 61; // IR +const pte_write: u64 = 1 << 62; // IW +const address_mask: u64 = 0x000F_FFFF_FFFF_F000; + +// Command buffer / event log: one 4 KiB frame each = 256 entries. +const ring_entries = 256; +const ring_length_code: u64 = 8; // log2(256), the ComLen/EventLen field value +const command_opcode_shift = 60; // opcode in bits 63:60 of qword 0 +const command_completion_wait: u64 = 0x01; +const command_invalidate_devtab: u64 = 0x02; +const command_invalidate_pages: u64 = 0x03; +const completion_wait_store: u64 = 1 << 1; // S: store `data` to the supplied address +const invalidate_pages_all: u64 = 0x000F_FFFF_FFFF_F000 | 1; // address bits 51:12 all-ones + S + +const levels: u8 = 4; // 48-bit IOVA, matching the Intel 4-level path + +var register_base: usize = 0; +var device_table: u64 = 0; // physical base of the device table +var command_buffer: u64 = 0; +var event_log: u64 = 0; +var completion_frame: u64 = 0; // COMPLETION_WAIT store target +var command_tail: u32 = 0; // our software copy of the command tail (bytes) + +var fault_log_budget: u32 = 32; +var completion_warned = false; + +fn read64(offset: usize) u64 { + return @as(*const volatile u64, @ptrFromInt(register_base + offset)).*; +} +fn write64(offset: usize, value: u64) void { + @as(*volatile u64, @ptrFromInt(register_base + offset)).* = value; +} +fn ram(physical: u64) [*]volatile u64 { + return @ptrFromInt(boot_handoff.physicalToVirtual(physical)); +} + +/// Map the register window, allocate the device table / command buffer / event log. +/// Returns the vtable, or null if the boot-time allocations fail. +pub fn detect(info: platform.PlatformInformation) ?iommu.Backend { + register_base = architecture.mapMmio(info.iommu_base, 16 * 1024, true); + + device_table = pmm.allocContiguous(device_table_pages, ~@as(u64, 0)) orelse return null; + zero(device_table, device_table_pages); // all-zero DTE = V=0 = deny every device + command_buffer = allocZeroedFrame() orelse return null; + event_log = allocZeroedFrame() orelse return null; + completion_frame = allocZeroedFrame() orelse return null; + + return iommu.Backend{ + .levels = levels, + .supports_huge_pages = false, // 4 KiB leaves only (AMD superpage encoding deferred) + .makeLeaf = makeLeaf, + .makeTable = makeTable, + .isPresent = isPresent, + .flushStructure = flushStructure, + .attach = attach, + .detach = detach, + .invalidateDomain = invalidateDomain, + .faultDrain = faultDrain, + }; +} + +/// Program the base registers and enable translation. The device table is already +/// zeroed (every device denied) except any entries `attach` wrote for RMRR/claimed +/// devices, so turning translation on blocks all other DMA and logs it. +pub fn enable() void { + write64(reg_device_table_base, (device_table & address_mask) | (device_table_pages - 1)); + write64(reg_command_buffer_base, (command_buffer & address_mask) | (ring_length_code << 56)); + write64(reg_event_log_base, (event_log & address_mask) | (ring_length_code << 56)); + write64(reg_command_head, 0); + write64(reg_command_tail, 0); + write64(reg_event_head, 0); + write64(reg_event_tail, 0); + command_tail = 0; + // Buffers first, then the master enable. + write64(reg_control, control_command_buffer_enable | control_event_log_enable); + write64(reg_control, control_command_buffer_enable | control_event_log_enable | control_iommu_enable); +} + +// --- Backend vtable ------------------------------------------------------------------ + +fn makeLeaf(physical: u64, huge: bool) u64 { + _ = huge; // 4 KiB only + return (physical & address_mask) | pte_present | pte_read | pte_write; // Next Level 0 = leaf +} +fn makeTable(table_physical: u64, level: u8) u64 { + // This entry (at `level`) points to a table one level down; AMD's Next Level names + // the pointed-to table's level. + const next_level: u64 = @as(u64, level) - 1; + return (table_physical & address_mask) | pte_present | pte_read | pte_write | (next_level << pte_next_level_shift); +} +fn isPresent(entry: u64) bool { + return (entry & pte_present) != 0; +} +fn flushStructure(address: usize) void { + _ = address; // AMD-Vi reads its structures coherently +} + +fn attach(bdf: u16, domain: u16, page_table_root: u64) void { + const dte = ram(device_table) + @as(usize, bdf) * dte_qwords; + dte[0] = (page_table_root & address_mask) | dte_valid | dte_translation_valid | + (@as(u64, levels) << dte_mode_shift) | dte_read | dte_write; + dte[1] = @as(u64, domain); // DomainID in bits 15:0 + dte[2] = dte_intctl_forward; // forward interrupts unmapped (no remapping) + dte[3] = 0; + invalidateDevice(bdf); + invalidateDomain(domain); +} + +fn detach(bdf: u16) void { + const dte = ram(device_table) + @as(usize, bdf) * dte_qwords; + dte[0] = 0; // V=0: deny + dte[1] = 0; + dte[2] = 0; + dte[3] = 0; + invalidateDevice(bdf); +} + +fn invalidateDomain(domain: u16) void { + submitCommand((command_invalidate_pages << command_opcode_shift) | (@as(u64, domain) << 32), invalidate_pages_all); + completeAndWait(); +} + +fn faultDrain() usize { + const tail = read64(reg_event_tail) & 0xFFFF_FFF0; + var head = read64(reg_event_head) & 0xFFFF_FFF0; + if (head == tail) return 0; + var seen: usize = 0; + while (head != tail) { + const entry = ram(event_log) + (head / 8); + const code: u4 = @truncate(entry[0] >> command_opcode_shift); + if (code == 0x2) { // IO_PAGE_FAULT + const source: u16 = @truncate(entry[0]); + logFault(source, entry[1]); + } + seen += 1; + head += 16; + if (head >= ring_entries * 16) head = 0; + } + write64(reg_event_head, head); + return seen; +} + +fn logFault(source: u16, address: u64) void { + if (fault_log_budget == 0) return; + fault_log_budget -= 1; + log.print("DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=amd-io-page-fault\n", .{ + source >> 8, + (source >> 3) & 0x1F, + source & 0x7, + address, + }); + if (fault_log_budget == 0) log.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n"); +} + +// --- command ring -------------------------------------------------------------------- + +fn invalidateDevice(bdf: u16) void { + submitCommand((command_invalidate_devtab << command_opcode_shift) | @as(u64, bdf), 0); + completeAndWait(); +} + +/// Append a 128-bit command (two qwords) to the ring and advance the tail. +fn submitCommand(qword0: u64, qword1: u64) void { + const slot = ram(command_buffer) + (command_tail / 8); + slot[0] = qword0; + slot[1] = qword1; + command_tail += 16; + if (command_tail >= ring_entries * 16) command_tail = 0; + write64(reg_command_tail, command_tail); +} + +/// Append a COMPLETION_WAIT (store form) and spin until the IOMMU writes our sentinel to +/// the completion frame. QEMU consumes the command buffer synchronously on the tail- +/// register write, so by the time we poll the prior invalidation is already applied; the +/// store confirmation is belt-and-suspenders for real hardware. If it never lands +/// (QEMU's amd-iommu does not implement the store form), warn ONCE and proceed — the +/// invalidation itself has happened. +fn completeAndWait() void { + const sentinel: u64 = 0xC0FFEE; + ram(completion_frame)[0] = 0; + submitCommand( + (command_completion_wait << command_opcode_shift) | (completion_frame & 0x000F_FFFF_FFFF_FFF8) | completion_wait_store, + sentinel, + ); + var spins: u64 = 0; + while (@as(*const volatile u64, @ptrFromInt(boot_handoff.physicalToVirtual(completion_frame))).* != sentinel) { + spins += 1; + if (spins > 100_000) { + if (!completion_warned) { + completion_warned = true; + log.write("/system/kernel: AMD-Vi COMPLETION_WAIT store not observed — proceeding (QEMU processes commands synchronously)\n"); + } + return; + } + } +} + +fn allocZeroedFrame() ?u64 { + const frame = pmm.alloc() orelse return null; + zero(frame, 1); + return frame; +} +fn zero(physical: u64, pages: usize) void { + const words = ram(physical); + var i: usize = 0; + while (i < pages * page_size / 8) : (i += 1) words[i] = 0; +} diff --git a/system/kernel/iommu.zig b/system/kernel/iommu.zig index 00523ab..5854b5a 100644 --- a/system/kernel/iommu.zig +++ b/system/kernel/iommu.zig @@ -32,6 +32,7 @@ const platform = @import("platform"); const devices_broker = @import("devices-broker.zig"); const log = @import("log.zig"); const intel = @import("iommu-intel.zig"); +const amd = @import("iommu-amd.zig"); const page_size: u64 = abi.page_size; const page_mask: u64 = page_size - 1; @@ -104,22 +105,32 @@ pub fn init() void { kind = .none; return; } - // Only Intel VT-d for now; AMD-Vi (iommu-amd.zig) selects here when its detection - // (IVRS) lands. A present-but-unsupported unit stays fail-open with a logged reason. - if (intel.detect(info)) |be| { - backend = be; - kind = .intel_vtd; + // Pick the backend by vendor. A present-but-unusable unit stays fail-open with a + // logged reason rather than half-enabling. + if (info.iommu_is_amd) { + if (amd.detect(info)) |be| { + backend = be; + kind = .amd_vi; + } else { + kind = .none; + log.write("/system/kernel: WARNING AMD-Vi present but unusable — staying fail-open\n"); + return; + } } else { - kind = .none; - log.write("/system/kernel: WARNING IOMMU present but unusable — staying fail-open\n"); - return; + if (intel.detect(info)) |be| { + backend = be; + kind = .intel_vtd; + } else { + kind = .none; + log.write("/system/kernel: WARNING IOMMU present but unusable — staying fail-open\n"); + return; + } } - // The root table starts empty: every device's context entry is not-present, so any - // DMA faults until the device's driver claims it (confineDevice gives it a private - // domain). PCI functions are enumerated post-boot by the ring-3 pci-bus driver, so - // there is nothing to attach at init anyway. - intel.enable(); + // The translation structures start empty: every device is denied until its driver + // claims it (confineDevice gives it a private domain). PCI functions are enumerated + // post-boot by the ring-3 pci-bus driver, so there is nothing to attach at init. + if (kind == .amd_vi) amd.enable() else intel.enable(); logEnabled(info); } @@ -421,6 +432,11 @@ fn hardwareId(domain: u16) u16 { } fn logEnabled(info: platform.PlatformInformation) void { + if (kind == .amd_vi) { + log.write("/system/kernel: iommu online (AMD-Vi) — UNTESTED on real AMD hardware (QEMU-verified only)\n"); + log.print(" levels : {d} (48-bit)\n", .{backend.levels}); + return; + } log.write("/system/kernel: iommu online (Intel VT-d)\n"); log.print(" version : 0x{x}\n", .{info.iommu_version}); log.print(" agaw : {d} levels\n", .{backend.levels}); diff --git a/system/kernel/tests.zig b/system/kernel/tests.zig index eeea730..5d7bb27 100644 --- a/system/kernel/tests.zig +++ b/system/kernel/tests.zig @@ -1251,9 +1251,12 @@ fn msiTest() void { fn iommuTest() void { log("DANOS-TEST-BEGIN: iommu\n", .{}); const pinfo = platform.platformInformation(); - check("IOMMU found in the DMAR table", pinfo.iommu_present); - check("VT-d unit has a register base", pinfo.iommu_base != 0); - check("VT-d version register reads back nonzero (real, mappable unit)", pinfo.iommu_version != 0); + check("IOMMU found in the firmware tables", pinfo.iommu_present); + check("IOMMU unit has a register base", pinfo.iommu_base != 0); + // The VT-d version register is a live-unit sanity check; AMD-Vi (from IVRS) records + // no version, so gate it on the vendor. + if (!pinfo.iommu_is_amd) + check("VT-d version register reads back nonzero (real, mappable unit)", pinfo.iommu_version != 0); log("DANOS-IOMMU: base=0x{x} version=0x{x} capabilities=0x{x}\n", .{ pinfo.iommu_base, pinfo.iommu_version, pinfo.iommu_capabilities }); // Translation was enabled at boot (kernel.zig: iommu.init before any driver claims diff --git a/test/qemu_test.py b/test/qemu_test.py index d73d151..2efcccb 100644 --- a/test/qemu_test.py +++ b/test/qemu_test.py @@ -193,6 +193,23 @@ CASES = [ "qemu_extra": ["-device", "intel-iommu,intremap=off", "-device", "e1000e"], "expect": r"(?s)(?=.*DANOS-IOMMU-FAULT: bdf=)(?=.*iommu-fault-test: system alive)(?=.*DANOS-TEST-RESULT: PASS)", "fail": r"iommu-fault-test: FAIL|DANOS-TEST-RESULT: FAIL|CPU EXCEPTION|KERNEL PANIC"}, + # AMD-Vi: the same detection + scratch-domain walker proof as the `iommu` case, but on + # the AMD backend (IVRS parse, device table, command buffer). QEMU's amd-iommu needs + # dma-remap=on (default off = translation silently ignored). UNTESTED on real AMD. + {"name": "amd-iommu", + "build_case": "iommu", + "qemu_extra": ["-device", "amd-iommu,dma-remap=on,intremap=off"], + "expect": r"(?s)(?=.*iommu online \(AMD-Vi\))(?=.*DANOS-IOMMU: enabled base=0x[0-9a-f]+ domains active)(?=.*DANOS-TEST-RESULT: PASS)", + "fail": r"DANOS-TEST-RESULT: FAIL|DANOS-IOMMU-FAULT"}, + # DMA under AMD-Vi translation: the full USB storage stack through AMD device-table + # translation. Tentative — land depends on QEMU amd-iommu behaving. UNTESTED on real AMD. + {"name": "amd-iommu-usb-storage", + "build_case": "fat-mount", + "smp": 4, + "timeout": 150, + "qemu_extra": ["-device", "amd-iommu,dma-remap=on,intremap=off"], + "expect": r"(?s)(?=.*iommu online \(AMD-Vi\))(?=.*fat: mounted /mnt/usb)(?=.*fat-test: ok)", + "fail": r"DANOS-TEST-RESULT: FAIL|DANOS-IOMMU-FAULT"}, # Port I/O grants: a claimed device's io_port resource lets a driver read/write its # ports (PS/2 status 0x64), gated by the claim; out-of-range/unclaimed is refused. {"name": "ioport",