M14a: memory-ordering / MMIO layer (library/mmio)
The tree had zero memory barriers — correct-by-accident on x86 (TSO + strong- uncacheable MMIO), but a landmine for the first DMA driver and for ARM, which is the win condition. Add /lib/mmio: typed volatile register access (read/write) plus mb / rmb / wmb, lowered per-architecture (mfence/lfence/sfence on x86_64, dsb sy/ld/st on aarch64) so the ordering rules are a named primitive, not scattered `asm volatile`. `volatile` is not a barrier — it says nothing about ordinary stores (a DMA descriptor in WB RAM) relative to a volatile doorbell write; wmb() between them is the fix. Prove it on the one existing caller: hpet now does its register access through mmio.read/write. It needs no barriers itself (pure MMIO, no DMA, UC grant on x86) — the point is the typed, arch-portable access every driver should use; the barriers are there for the DMA drivers to come. New `mmio` module injected into addUserBinary; host test asserts the barriers assemble and a register round-trips. Suite 37/37 plus host tests.
This commit is contained in:
Daniel Samson
parent
a581712b09
commit
e7c7e7b94c
23
build.zig
23
build.zig
|
|
@ -59,6 +59,7 @@ fn addUserBinary(
|
|||
target: std.Build.ResolvedTarget,
|
||||
runtime_module: *std.Build.Module,
|
||||
posix_module: *std.Build.Module,
|
||||
mmio_module: *std.Build.Module,
|
||||
name: []const u8,
|
||||
root: []const u8,
|
||||
) *std.Build.Step.Compile {
|
||||
|
|
@ -78,6 +79,8 @@ fn addUserBinary(
|
|||
// POSIX/C compatibility layer, available to any program that wants it
|
||||
// (danos-native code uses `runtime` directly). See library/posix/.
|
||||
.{ .name = "posix", .module = posix_module },
|
||||
// Typed volatile MMIO + memory barriers, for drivers. See library/mmio/.
|
||||
.{ .name = "mmio", .module = mmio_module },
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
|
@ -181,6 +184,13 @@ pub fn build(b: *std.Build) void {
|
|||
},
|
||||
});
|
||||
|
||||
// Typed volatile MMIO register access + memory-ordering barriers, for drivers on
|
||||
// top of an mmio_map grant. Depends only on `builtin` (arch-conditional barriers);
|
||||
// no target set, so it inherits each driver's. See library/mmio/mmio.zig.
|
||||
const mmio_module = b.addModule("mmio", .{
|
||||
.root_source_file = b.path("library/mmio/mmio.zig"),
|
||||
});
|
||||
|
||||
// The POSIX / C compatibility layer, a separate library layered strictly over the
|
||||
// runtime (it calls the runtime's IPC/heap, never system calls directly). This is
|
||||
// the one place POSIX/C spellings are allowed verbatim — see docs/coding-standards.md
|
||||
|
|
@ -264,7 +274,7 @@ pub fn build(b: *std.Build) void {
|
|||
// Built by the shared user-binary recipe (see addUserBinary): freestanding,
|
||||
// linked into the kernel's user region against the `runtime` runtime library, and
|
||||
// started in ring 3 by the kernel's user-ELF loader.
|
||||
const init_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, "init", "system/services/init/init.zig");
|
||||
const init_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, mmio_module, "init", "system/services/init/init.zig");
|
||||
const init_install = b.addInstallArtifact(init_exe, .{ .dest_dir = .{ .override = .{ .custom = "system/services" } } });
|
||||
b.getInstallStep().dependOn(&init_install.step);
|
||||
|
||||
|
|
@ -272,11 +282,11 @@ pub fn build(b: *std.Build) void {
|
|||
// Each is built by the same user-binary recipe, then packed into one image by
|
||||
// the host-side make-initial-ramdisk tool. The bootloader ferries the image to the kernel,
|
||||
// which unpacks it and spawns each program (system/initial-ramdisk.zig).
|
||||
const vfs_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, "vfs", "system/services/vfs/vfs.zig");
|
||||
const vfstest_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, "vfs-test", "system/services/vfs/vfs-test.zig");
|
||||
const hpet_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, "hpet", "system/drivers/hpet/hpet.zig");
|
||||
const bus_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, "bus", "system/drivers/bus/bus.zig");
|
||||
const device_manager_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, "device-manager", "system/services/device-manager/device-manager.zig");
|
||||
const vfs_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, mmio_module, "vfs", "system/services/vfs/vfs.zig");
|
||||
const vfstest_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, mmio_module, "vfs-test", "system/services/vfs/vfs-test.zig");
|
||||
const hpet_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, mmio_module, "hpet", "system/drivers/hpet/hpet.zig");
|
||||
const bus_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, mmio_module, "bus", "system/drivers/bus/bus.zig");
|
||||
const device_manager_exe = addUserBinary(b, kernel_target, runtime_module, posix_module, mmio_module, "device-manager", "system/services/device-manager/device-manager.zig");
|
||||
|
||||
// Pack the user binaries into the initial_ramdisk image with the host-side Python tool
|
||||
// (the container format is trivial, and Python sidesteps std API churn). Args:
|
||||
|
|
@ -429,6 +439,7 @@ pub fn build(b: *std.Build) void {
|
|||
"system/boot-handoff.zig",
|
||||
"system/abi.zig",
|
||||
"system/devices/device-abi.zig",
|
||||
"library/mmio/mmio.zig", // barriers assemble + registers round-trip
|
||||
}) |root| {
|
||||
const mod_tests = b.addTest(.{
|
||||
.root_module = b.createModule(.{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
//! /lib/mmio — typed volatile MMIO register access, plus the memory-ordering
|
||||
//! barriers a device driver needs. Used by drivers on top of an `mmio_map` grant.
|
||||
//!
|
||||
//! **`volatile` is not a barrier.** In Zig it means only: don't elide this access, and
|
||||
//! don't reorder it against *other volatile* accesses. It says nothing about ordinary
|
||||
//! stores — the DMA descriptor you just filled in write-back RAM — which the compiler
|
||||
//! (and, on weakly-ordered hardware, the CPU) may freely move past a volatile MMIO
|
||||
//! write. The canonical bug:
|
||||
//!
|
||||
//! ring[i] = descriptor; // ordinary store to WB RAM
|
||||
//! doorbell.* = i; // volatile store to UC MMIO
|
||||
//! // nothing orders these; the device can read a stale descriptor
|
||||
//!
|
||||
//! Put a `wmb()` between them. The barriers lower per-architecture — which is the whole
|
||||
//! reason they are a named primitive and not scattered `asm volatile`:
|
||||
//!
|
||||
//! x86_64 aarch64
|
||||
//! mb() mfence dsb sy
|
||||
//! rmb() lfence dsb ld
|
||||
//! wmb() sfence dsb st
|
||||
//!
|
||||
//! x86 is forgiving (TSO + strong-uncacheable MMIO), so a compiler barrier usually
|
||||
//! suffices; ARM is not, and ARM is the win condition (docs/vision.md) — so the
|
||||
//! abstraction exists now, while there is one caller (hpet) to get right. See
|
||||
//! docs/driver-model.md (M14) for the full ordering contract.
|
||||
|
||||
const builtin = @import("builtin");
|
||||
|
||||
/// Read a register of type `T` at absolute virtual address `addr` — a location inside
|
||||
/// a device's `mmio_map` grant. `volatile`: never elided, never reordered against
|
||||
/// another volatile access.
|
||||
pub inline fn read(comptime T: type, addr: usize) T {
|
||||
return @as(*const volatile T, @ptrFromInt(addr)).*;
|
||||
}
|
||||
|
||||
/// Write `value` of type `T` to the register at absolute virtual address `addr`.
|
||||
pub inline fn write(comptime T: type, addr: usize, value: T) void {
|
||||
@as(*volatile T, @ptrFromInt(addr)).* = value;
|
||||
}
|
||||
|
||||
/// Full barrier: all loads and stores before it are globally visible before any after
|
||||
/// it. Use when an MMIO write must complete before a following read.
|
||||
pub inline fn mb() void {
|
||||
switch (builtin.target.cpu.arch) {
|
||||
.x86_64 => asm volatile ("mfence" ::: .{ .memory = true }),
|
||||
.aarch64 => asm volatile ("dsb sy" ::: .{ .memory = true }),
|
||||
else => @compileError("mmio.mb: unsupported architecture"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read barrier: loads before it complete before loads after it. Use after an IRQ
|
||||
/// wake, before reading what the device wrote to shared memory.
|
||||
pub inline fn rmb() void {
|
||||
switch (builtin.target.cpu.arch) {
|
||||
.x86_64 => asm volatile ("lfence" ::: .{ .memory = true }),
|
||||
.aarch64 => asm volatile ("dsb ld" ::: .{ .memory = true }),
|
||||
else => @compileError("mmio.rmb: unsupported architecture"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write barrier: stores before it become visible before stores after it. Use between
|
||||
/// filling a DMA descriptor in RAM and ringing the device's doorbell.
|
||||
pub inline fn wmb() void {
|
||||
switch (builtin.target.cpu.arch) {
|
||||
.x86_64 => asm volatile ("sfence" ::: .{ .memory = true }),
|
||||
.aarch64 => asm volatile ("dsb st" ::: .{ .memory = true }),
|
||||
else => @compileError("mmio.wmb: unsupported architecture"),
|
||||
}
|
||||
}
|
||||
|
||||
test "barriers emit and registers round-trip through a RAM cell" {
|
||||
// The barriers must at least assemble for the host arch; ordering can't be unit
|
||||
// tested, but a missing/mistyped mnemonic is caught here.
|
||||
wmb();
|
||||
rmb();
|
||||
mb();
|
||||
var cell: u64 = 0;
|
||||
write(u64, @intFromPtr(&cell), 0xDEAD_BEEF);
|
||||
try @import("std").testing.expectEqual(@as(u64, 0xDEAD_BEEF), read(u64, @intFromPtr(&cell)));
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@
|
|||
//! 0x108 TIMER0_COMPARATOR
|
||||
|
||||
const runtime = @import("runtime");
|
||||
const mmio = @import("mmio");
|
||||
const device = runtime.device;
|
||||
const ipc = runtime.ipc;
|
||||
|
||||
|
|
@ -50,8 +51,15 @@ const tn_route_mask: u64 = 0x1F << tn_route_shift;
|
|||
/// Interrupts to observe before declaring victory.
|
||||
const target_ticks = 5;
|
||||
|
||||
fn register(base: usize, off: usize) *volatile u64 {
|
||||
return @ptrFromInt(base + off);
|
||||
/// Read/write a 64-bit HPET register through the typed volatile MMIO layer (/lib/mmio).
|
||||
/// The HPET is pure MMIO with no DMA, and on x86 its grant is strong-uncacheable (so
|
||||
/// UC writes are already ordered) — no barriers are needed here; the point is the
|
||||
/// typed, arch-portable access every driver should use.
|
||||
inline fn rd(base: usize, off: usize) u64 {
|
||||
return mmio.read(u64, base + off);
|
||||
}
|
||||
inline fn wr(base: usize, off: usize, value: u64) void {
|
||||
mmio.write(u64, base + off, value);
|
||||
}
|
||||
|
||||
/// A timer-class device exposing both an MMIO window and an IRQ: its id, the two
|
||||
|
|
@ -66,16 +74,16 @@ fn findHpet(buffer: []device.DeviceDescriptor) ?Found {
|
|||
// Skip comparator children a bus driver may have published below the block
|
||||
// (see system/drivers/bus/bus.zig) — we want the register block itself.
|
||||
if (d.parent != device.no_parent) continue;
|
||||
var mmio: ?u64 = null;
|
||||
var mmio_index: ?u64 = null;
|
||||
var irq: ?u64 = null;
|
||||
for (0..d.resource_count) |j| {
|
||||
switch (d.resources[j].kind) {
|
||||
@intFromEnum(device.ResourceKind.memory) => mmio = mmio orelse j,
|
||||
@intFromEnum(device.ResourceKind.memory) => mmio_index = mmio_index orelse j,
|
||||
@intFromEnum(device.ResourceKind.irq) => irq = irq orelse j,
|
||||
else => {},
|
||||
}
|
||||
}
|
||||
if (mmio) |m| if (irq) |i| {
|
||||
if (mmio_index) |m| if (irq) |i| {
|
||||
return .{ .device_id = d.id, .mmio = m, .irq = i, .gsi = d.resources[i].start };
|
||||
};
|
||||
}
|
||||
|
|
@ -114,7 +122,7 @@ pub fn main() void {
|
|||
|
||||
// --- program the hardware ------------------------------------------------
|
||||
// Counter period, so we can arm the comparator a fixed wall-clock distance out.
|
||||
const femtos_per_tick = register(base, register_general_cap).* >> 32;
|
||||
const femtos_per_tick = rd(base, register_general_cap) >> 32;
|
||||
if (femtos_per_tick == 0) {
|
||||
_ = runtime.system.write("hpet: bad HPET period\n");
|
||||
return;
|
||||
|
|
@ -122,21 +130,21 @@ pub fn main() void {
|
|||
const ticks_per_ms = 1_000_000_000_000 / femtos_per_tick;
|
||||
|
||||
// Stop the counter and take the legacy route off while we reconfigure.
|
||||
register(base, register_general_configuration).* &= ~(configuration_enable | configuration_leg_rt);
|
||||
wr(base, register_general_configuration, rd(base, register_general_configuration) & ~(configuration_enable | configuration_leg_rt));
|
||||
|
||||
// Timer 0: one-shot, level-triggered, routed to our GSI, interrupt enabled.
|
||||
// One-shot (not periodic) sidesteps the HPET's Tn_value_SET accumulator quirk —
|
||||
// we simply re-arm from the driver on each interrupt, which is what a tickless
|
||||
// timer driver does anyway.
|
||||
var t0 = register(base, register_timer0_configuration).*;
|
||||
var t0 = rd(base, register_timer0_configuration);
|
||||
t0 &= ~(tn_route_mask | tn_type_periodic);
|
||||
t0 |= tn_int_type_level | tn_int_enb | (gsi << tn_route_shift);
|
||||
register(base, register_timer0_configuration).* = t0;
|
||||
wr(base, register_timer0_configuration, t0);
|
||||
|
||||
// Clear any stale assertion, then arm ~100 ms out and start the counter.
|
||||
register(base, register_int_status).* = 1;
|
||||
register(base, register_timer0_comparator).* = register(base, register_main_counter).* + ticks_per_ms * 100;
|
||||
register(base, register_general_configuration).* |= configuration_enable;
|
||||
wr(base, register_int_status, 1);
|
||||
wr(base, register_timer0_comparator, rd(base, register_main_counter) + ticks_per_ms * 100);
|
||||
wr(base, register_general_configuration, rd(base, register_general_configuration) | configuration_enable);
|
||||
|
||||
if (!device.irqBind(hpet.device_id, hpet.irq, endpoint)) {
|
||||
_ = runtime.system.write("hpet: irq_bind failed\n");
|
||||
|
|
@ -157,17 +165,17 @@ pub fn main() void {
|
|||
|
||||
// Quiet the device: write 1 to timer 0's status bit. Until this lands, the
|
||||
// line is still asserted and unmasking would refire immediately.
|
||||
register(base, register_int_status).* = 1;
|
||||
wr(base, register_int_status, 1);
|
||||
count += 1;
|
||||
|
||||
if (count < target_ticks) {
|
||||
register(base, register_timer0_comparator).* = register(base, register_main_counter).* + ticks_per_ms * 100;
|
||||
wr(base, register_timer0_comparator, rd(base, register_main_counter) + ticks_per_ms * 100);
|
||||
} else {
|
||||
// Last one: stop the source rather than re-arming, so the line is left
|
||||
// both quiet *and* unmasked by the ack below. Re-arming here would leave
|
||||
// a pending interrupt that nobody is waiting for, and the ISR would mask
|
||||
// the line again a moment later.
|
||||
register(base, register_timer0_configuration).* &= ~tn_int_enb;
|
||||
wr(base, register_timer0_configuration, rd(base, register_timer0_configuration) & ~tn_int_enb);
|
||||
}
|
||||
|
||||
_ = runtime.system.write("hpet: irq\n");
|
||||
|
|
|
|||
Loading…
Reference in New Issue