//! /lib/mmio — typed volatile MMIO register access, plus the memory-ordering //! barriers a device driver needs. Used by drivers on top of an `mmio_map` grant. //! //! **`volatile` is not a barrier.** In Zig it means only: don't elide this access, and //! don't reorder it against *other volatile* accesses. It says nothing about ordinary //! stores — the DMA descriptor you just filled in write-back RAM — which the compiler //! (and, on weakly-ordered hardware, the CPU) may freely move past a volatile MMIO //! write. The canonical bug: //! //! ring[i] = descriptor; // ordinary store to WB RAM //! doorbell.* = i; // volatile store to UC MMIO //! // nothing orders these; the device can read a stale descriptor //! //! Put a `wmb()` between them. The barriers lower per-architecture — which is the whole //! reason they are a named primitive and not scattered `asm volatile`: //! //! x86_64 aarch64 //! mb() mfence dsb sy //! rmb() lfence dsb ld //! wmb() sfence dsb st //! //! x86 is forgiving (TSO + strong-uncacheable MMIO), so a compiler barrier usually //! suffices; ARM is not, and ARM is the win condition (docs/vision.md) — so the //! abstraction exists now, while there is one caller (hpet) to get right. See //! docs/driver-model.md (M14) for the full ordering contract. const builtin = @import("builtin"); /// Read a register of type `T` at absolute virtual address `addr` — a location inside /// a device's `mmio_map` grant. `volatile`: never elided, never reordered against /// another volatile access. pub inline fn read(comptime T: type, addr: usize) T { return @as(*const volatile T, @ptrFromInt(addr)).*; } /// Write `value` of type `T` to the register at absolute virtual address `addr`. pub inline fn write(comptime T: type, addr: usize, value: T) void { @as(*volatile T, @ptrFromInt(addr)).* = value; } /// Full barrier: all loads and stores before it are globally visible before any after /// it. Use when an MMIO write must complete before a following read. pub inline fn mb() void { switch (builtin.target.cpu.arch) { .x86_64 => asm volatile ("mfence" ::: .{ .memory = true }), .aarch64 => asm volatile ("dsb sy" ::: .{ .memory = true }), else => @compileError("mmio.mb: unsupported architecture"), } } /// Read barrier: loads before it complete before loads after it. Use after an IRQ /// wake, before reading what the device wrote to shared memory. pub inline fn rmb() void { switch (builtin.target.cpu.arch) { .x86_64 => asm volatile ("lfence" ::: .{ .memory = true }), .aarch64 => asm volatile ("dsb ld" ::: .{ .memory = true }), else => @compileError("mmio.rmb: unsupported architecture"), } } /// Write barrier: stores before it become visible before stores after it. Use between /// filling a DMA descriptor in RAM and ringing the device's doorbell. pub inline fn wmb() void { switch (builtin.target.cpu.arch) { .x86_64 => asm volatile ("sfence" ::: .{ .memory = true }), .aarch64 => asm volatile ("dsb st" ::: .{ .memory = true }), else => @compileError("mmio.wmb: unsupported architecture"), } } test "barriers emit and registers round-trip through a RAM cell" { // The barriers must at least assemble for the host arch; ordering can't be unit // tested, but a missing/mistyped mnemonic is caught here. wmb(); rmb(); mb(); var cell: u64 = 0; write(u64, @intFromPtr(&cell), 0xDEAD_BEEF); try @import("std").testing.expectEqual(@as(u64, 0xDEAD_BEEF), read(u64, @intFromPtr(&cell))); }