81 lines
3.8 KiB
Zig
81 lines
3.8 KiB
Zig
//! /lib/device/mmio — typed volatile MMIO register access, plus the memory-ordering
|
|
//! barriers a device driver needs. Used by drivers on top of an `mmio_map` grant.
|
|
//!
|
|
//! **`volatile` is not a barrier.** In Zig it means only: don't elide this access, and
|
|
//! don't reorder it against *other volatile* accesses. It says nothing about ordinary
|
|
//! stores — the DMA descriptor you just filled in write-back RAM — which the compiler
|
|
//! (and, on weakly-ordered hardware, the CPU) may freely move past a volatile MMIO
|
|
//! write. The canonical bug:
|
|
//!
|
|
//! ring[i] = descriptor; // ordinary store to WB RAM
|
|
//! doorbell.* = i; // volatile store to UC MMIO
|
|
//! // nothing orders these; the device can read a stale descriptor
|
|
//!
|
|
//! Put a `writeMemoryBarrier()` between them. The barriers lower per-architecture — which
|
|
//! is the whole reason they are a named primitive and not scattered `asm volatile`:
|
|
//!
|
|
//! x86_64 aarch64
|
|
//! memoryBarrier() mfence dsb sy
|
|
//! readMemoryBarrier() lfence dsb ld
|
|
//! writeMemoryBarrier() sfence dsb st
|
|
//!
|
|
//! x86 is forgiving (TSO + strong-uncacheable MMIO), so a compiler barrier usually
|
|
//! suffices; ARM is not, and ARM is the win condition (docs/vision.md) — so the
|
|
//! abstraction exists now, while there is one caller (hpet) to get right. See
|
|
//! docs/driver-model.md (M14) for the full ordering contract.
|
|
|
|
const builtin = @import("builtin");
|
|
|
|
/// Read a register of type `T` at absolute virtual address `addr` — a location inside
|
|
/// a device's `mmio_map` grant. `volatile`: never elided, never reordered against
|
|
/// another volatile access.
|
|
pub inline fn readRegister(comptime T: type, addr: usize) T {
|
|
return @as(*const volatile T, @ptrFromInt(addr)).*;
|
|
}
|
|
|
|
/// Write `value` of type `T` to the register at absolute virtual address `addr`.
|
|
pub inline fn writeRegister(comptime T: type, addr: usize, value: T) void {
|
|
@as(*volatile T, @ptrFromInt(addr)).* = value;
|
|
}
|
|
|
|
/// Full memory barrier: all loads and stores before it are globally visible before any
|
|
/// after it. Use when an MMIO write must complete before a following read.
|
|
pub inline fn memoryBarrier() void {
|
|
switch (builtin.target.cpu.arch) {
|
|
.x86_64 => asm volatile ("mfence" ::: .{ .memory = true }),
|
|
.aarch64 => asm volatile ("dsb sy" ::: .{ .memory = true }),
|
|
else => @compileError("mmio.memoryBarrier: unsupported architecture"),
|
|
}
|
|
}
|
|
|
|
/// Read memory barrier: loads before it complete before loads after it. Use after an IRQ
|
|
/// wake, before reading what the device wrote to shared memory.
|
|
pub inline fn readMemoryBarrier() void {
|
|
switch (builtin.target.cpu.arch) {
|
|
.x86_64 => asm volatile ("lfence" ::: .{ .memory = true }),
|
|
.aarch64 => asm volatile ("dsb ld" ::: .{ .memory = true }),
|
|
else => @compileError("mmio.readMemoryBarrier: unsupported architecture"),
|
|
}
|
|
}
|
|
|
|
/// Write memory barrier: stores before it become visible before stores after it. Use
|
|
/// between filling a DMA descriptor in RAM and ringing the device's doorbell.
|
|
pub inline fn writeMemoryBarrier() void {
|
|
switch (builtin.target.cpu.arch) {
|
|
.x86_64 => asm volatile ("sfence" ::: .{ .memory = true }),
|
|
.aarch64 => asm volatile ("dsb st" ::: .{ .memory = true }),
|
|
else => @compileError("mmio.writeMemoryBarrier: unsupported architecture"),
|
|
}
|
|
}
|
|
|
|
test "barriers emit and registers round-trip through a RAM cell" {
|
|
// The barriers must at least assemble for the host arch; ordering can't be unit
|
|
// tested, but a missing/mistyped mnemonic is caught here.
|
|
writeMemoryBarrier();
|
|
readMemoryBarrier();
|
|
memoryBarrier();
|
|
var cell: u64 = 0;
|
|
writeRegister(u64, @intFromPtr(&cell), 0xDEAD_BEEF);
|
|
try @import("std").testing.expectEqual(@as(u64, 0xDEAD_BEEF), readRegister(u64, @intFromPtr(&cell)));
|
|
}
|