//! /lib/device/mmio — typed volatile MMIO register access, plus the memory-ordering //! barriers a device driver needs. Used by drivers on top of an `mmio_map` grant. //! //! **`volatile` is not a barrier.** In Zig it means only: don't elide this access, and //! don't reorder it against *other volatile* accesses. It says nothing about ordinary //! stores — the DMA descriptor you just filled in write-back RAM — which the compiler //! (and, on weakly-ordered hardware, the CPU) may freely move past a volatile MMIO //! write. The canonical bug: //! //! ring[i] = descriptor; // ordinary store to WB RAM //! doorbell.* = i; // volatile store to UC MMIO //! // nothing orders these; the device can read a stale descriptor //! //! Put a `writeMemoryBarrier()` between them. The barriers lower per-architecture — which //! is the whole reason they are a named primitive and not scattered `asm volatile`: //! //! x86_64 aarch64 //! memoryBarrier() mfence dsb sy //! readMemoryBarrier() lfence dsb ld //! writeMemoryBarrier() sfence dsb st //! //! x86 is forgiving (TSO + strong-uncacheable MMIO), so a compiler barrier usually //! suffices; ARM is not, and ARM is the win condition (docs/vision.md) — so the //! abstraction exists now, while there is one caller (hpet) to get right. See //! docs/driver-model.md (M14) for the full ordering contract. const builtin = @import("builtin"); /// Read a register of type `T` at absolute virtual address `addr` — a location inside /// a device's `mmio_map` grant. `volatile`: never elided, never reordered against /// another volatile access. pub inline fn readRegister(comptime T: type, addr: usize) T { return @as(*const volatile T, @ptrFromInt(addr)).*; } /// Write `value` of type `T` to the register at absolute virtual address `addr`. pub inline fn writeRegister(comptime T: type, addr: usize, value: T) void { @as(*volatile T, @ptrFromInt(addr)).* = value; } /// Full memory barrier: all loads and stores before it are globally visible before any /// after it. Use when an MMIO write must complete before a following read. pub inline fn memoryBarrier() void { switch (builtin.target.cpu.arch) { .x86_64 => asm volatile ("mfence" ::: .{ .memory = true }), .aarch64 => asm volatile ("dsb sy" ::: .{ .memory = true }), else => @compileError("mmio.memoryBarrier: unsupported architecture"), } } /// Read memory barrier: loads before it complete before loads after it. Use after an IRQ /// wake, before reading what the device wrote to shared memory. pub inline fn readMemoryBarrier() void { switch (builtin.target.cpu.arch) { .x86_64 => asm volatile ("lfence" ::: .{ .memory = true }), .aarch64 => asm volatile ("dsb ld" ::: .{ .memory = true }), else => @compileError("mmio.readMemoryBarrier: unsupported architecture"), } } /// Write memory barrier: stores before it become visible before stores after it. Use /// between filling a DMA descriptor in RAM and ringing the device's doorbell. pub inline fn writeMemoryBarrier() void { switch (builtin.target.cpu.arch) { .x86_64 => asm volatile ("sfence" ::: .{ .memory = true }), .aarch64 => asm volatile ("dsb st" ::: .{ .memory = true }), else => @compileError("mmio.writeMemoryBarrier: unsupported architecture"), } } test "barriers emit and registers round-trip through a RAM cell" { // The barriers must at least assemble for the host arch; ordering can't be unit // tested, but a missing/mistyped mnemonic is caught here. writeMemoryBarrier(); readMemoryBarrier(); memoryBarrier(); var cell: u64 = 0; writeRegister(u64, @intFromPtr(&cell), 0xDEAD_BEEF); try @import("std").testing.expectEqual(@as(u64, 0xDEAD_BEEF), readRegister(u64, @intFromPtr(&cell))); }