boot: the capsule — one-file system image first, manifest and walk as fallbacks

Real-firmware finding: the per-file /system tree walk boots in seconds
under OVMF but stalls for MINUTES on real firmware — the cost is not
bytes (USB 3 moves the ~4 MB instantly) but firmware filesystem
OPERATIONS: ~30 opens, each an uncached directory-chain walk in an
unoptimized firmware FAT driver. This is why every real OS loader
(winload, GRUB) reads many files through its own filesystem code over
Block I/O rather than the firmware's file protocol.

The loader now reads boot/system.img — the bundled binaries packed into
ONE v2 initial_ramdisk (tools/pack-system-image.py, derived from the
same bundled list in the same build graph, so tree and capsule cannot
drift) — with a single open + sequential read, the one firmware file
I/O shape that is fast everywhere. The manifest (open each listed path
by name) and the tree walk remain as fallbacks, so a hand-assembled
stick without the capsule still boots. The running system is identical
in all three cases: the kernel receives the same in-RAM table.

The load phase now brackets itself with unconditional on-screen
breadcrumbs ('EFI: loading the system...' / '...starting the kernel'),
because this phase stalling behind a silent black screen — kernel
status is serial-only by design — already cost a real-hardware
debugging session.

Direction (settled with the user): this capsule becomes the BOOTSTRAP
capsule — kernel + init + the storage-bring-up set — once a
spawn-from-memory syscall lets init and the device manager load
everything else from the stick's real file tree at runtime through
danos's own storage stack: file-granular updates (rebuild one binary,
copy one file), the initramfs shape.

boot/efi.zig

@@ -380,11 +380,19 @@ const Bundled = struct {
     data: []align(8) u8,
 };
 
-/// Walk the boot volume's /system tree and pack every regular file (except the
+/// Gather the boot volume's user binaries into an in-RAM v2 initial_ramdisk
-/// kernel image itself — the only top-level file) into an in-RAM v2
+/// image, entries named by full FHS path — the volume's file structure is the
-/// initial_ramdisk image, entries named by full FHS path. This is what makes the
+/// single source of truth (no packed ramdisk artifact; init travels in the
-/// volume's file structure the single source of truth: there is no packed
+/// table like everything else).
-/// ramdisk artifact on disk, and init travels in the table like everything else.
+///
+/// Two strategies, most portable first:
+///   1. /system/manifest (written by the build): each listed path is opened BY
+///      NAME — the case-insensitive lookup every firmware FAT driver gets
+///      right, and the only file access the pre-tree loader ever used.
+///   2. No manifest: ENUMERATE the /system tree. Portable in principle, but
+///      firmware differs in what names enumeration returns (bare 8.3 entries
+///      come back uppercase on some drivers), so this is the fallback for
+///      hand-assembled sticks, not the primary path.
 fn loadSystemTree(bs: *uefi.tables.BootServices, boot_information: *BootInformation) !void {
     const loaded = (try bs.handleProtocol(uefi.protocol.LoadedImage, uefi.handle)) orelse
         return error.NoLoadedImage;
@@ -395,12 +403,30 @@ fn loadSystemTree(bs: *uefi.tables.BootServices, boot_information: *BootInformat
     const root = try fs.openVolume();
     defer _ = root.close() catch {};
 
-    const system_directory = try root.open(system_directory_name, .read, .{});
+    // Unconditional breadcrumb (con_out, independent of -Dserial): this phase
-    defer _ = system_directory.close() catch {};
+    // is where a slow firmware stalls, and a silent black screen here already
+    // cost a real-hardware debugging session.
+    log("EFI: loading the system...\r\n");
+
+    // The capsule (boot\system.img) first: one open + one sequential read is
+    // the only firmware file I/O shape that is fast everywhere. It is already
+    // the kernel's wire format — hand it over as-is.
+    if (loadCapsule(bs, root, boot_information)) {
+        log("EFI: system image loaded, starting the kernel\r\n");
+        return;
+    }
 
     var list: [maximum_bundled]Bundled = undefined;
     var count: usize = 0;
-    try walkDirectory(bs, system_directory, "/system", 0, &list, &count);
+
+    loadByManifest(bs, root, &list, &count) catch {
+        count = 0; // a torn manifest read leaves partial entries; start over
+    };
+    if (count == 0) {
+        const system_directory = try root.open(system_directory_name, .read, .{});
+        defer _ = system_directory.close() catch {};
+        try walkDirectory(bs, system_directory, "/system", 0, &list, &count);
+    }
     if (count == 0) return error.NoBinaries;
 
     // Assemble the v2 image: header, entry table, then the blobs.
@@ -426,7 +452,72 @@ fn loadSystemTree(bs: *uefi.tables.BootServices, boot_information: *BootInformat
 
     boot_information.initial_ramdisk_base = @intFromPtr(image.ptr);
     boot_information.initial_ramdisk_len = total;
-    progress("EFI: /system tree loaded\r\n");
+    log("EFI: /system tree loaded, starting the kernel\r\n");
+}
+
+/// The boot capsule: the bundled binaries as one v2 initial_ramdisk image.
+const capsule_file_name = std.unicode.utf8ToUtf16LeStringLiteral("boot\\system.img");
+
+/// Load boot\system.img whole and hand it to the kernel unmodified — it is
+/// already the initial_ramdisk wire format. Returns false (capsule absent or
+/// unreadable or wrong magic) to let the caller fall back to per-file loading.
+fn loadCapsule(bs: *uefi.tables.BootServices, root: *uefi.protocol.File, boot_information: *BootInformation) bool {
+    const file = root.open(capsule_file_name, .read, .{}) catch return false;
+    defer _ = file.close() catch {};
+    const image = readWholeFile(bs, file) catch return false;
+    if (image.len < @sizeOf(initial_ramdisk.Header) or
+        std.mem.bytesToValue(initial_ramdisk.Header, image[0..@sizeOf(initial_ramdisk.Header)]).magic != initial_ramdisk.magic)
+    {
+        _ = bs.freePool(image.ptr) catch {};
+        return false;
+    }
+    boot_information.initial_ramdisk_base = @intFromPtr(image.ptr);
+    boot_information.initial_ramdisk_len = image.len;
+    return true;
+}
+
+/// The manifest path, and a scratch limit for its UTF-16 conversion.
+const manifest_file_name = std.unicode.utf8ToUtf16LeStringLiteral("system\\manifest");
+
+/// Load every binary the manifest lists, opening each path by name from the
+/// volume root. A listed-but-unopenable file is skipped (the kernel reports the
+/// absence); a missing manifest errors so the caller falls back to the walk.
+fn loadByManifest(bs: *uefi.tables.BootServices, root: *uefi.protocol.File, list: *[maximum_bundled]Bundled, count: *usize) !void {
+    const manifest_handle = try root.open(manifest_file_name, .read, .{});
+    var manifest_open = true;
+    defer if (manifest_open) {
+        _ = manifest_handle.close() catch {};
+    };
+    const manifest = try readWholeFile(bs, manifest_handle);
+    _ = manifest_handle.close() catch {};
+    manifest_open = false;
+    defer _ = bs.freePool(manifest.ptr) catch {};
+
+    var lines = std.mem.tokenizeAny(u8, manifest, "\r\n");
+    while (lines.next()) |line| {
+        if (line.len < 2 or line[0] != '/') continue;
+        if (line.len >= initial_ramdisk.maximum_name) continue;
+        if (count.* == maximum_bundled) return;
+
+        // "/system/services/init" -> UTF-16 "system\services\init".
+        var name16: [initial_ramdisk.maximum_name]u16 = undefined;
+        var i: usize = 0;
+        for (line[1..]) |c| {
+            name16[i] = if (c == '/') '\\' else c;
+            i += 1;
+        }
+        name16[i] = 0;
+
+        const file = root.open(@ptrCast(name16[0..i :0]), .read, .{}) catch continue;
+        defer _ = file.close() catch {};
+        const data = readWholeFile(bs, file) catch continue;
+
+        var entry: *Bundled = &list[count.*];
+        @memcpy(entry.path[0..line.len], line);
+        entry.path_len = line.len;
+        entry.data = data;
+        count.* += 1;
+    }
 }
 
 /// Recursively collect the regular files below `directory` into `list`. Top-level

build.zig

@@ -235,6 +235,8 @@ fn addBootImage(
     b: *std.Build,
     kernel_bin: std.Build.LazyPath,
     efi_bin: std.Build.LazyPath,
+    manifest: std.Build.LazyPath,
+    capsule: std.Build.LazyPath,
     bundled: []const BundledBinary,
 ) std.Build.LazyPath {
     const mk_fat = b.addSystemCommand(&.{"python3"});
@@ -245,6 +247,10 @@ fn addBootImage(
     mk_fat.addFileArg(efi_bin);
     mk_fat.addArg("system/kernel");
     mk_fat.addFileArg(kernel_bin);
+    mk_fat.addArg("system/manifest");
+    mk_fat.addFileArg(manifest);
+    mk_fat.addArg("boot/system.img");
+    mk_fat.addFileArg(capsule);
     for (bundled) |item| {
         mk_fat.addArg(item.path);
         mk_fat.addFileArg(item.binary);
@@ -622,6 +628,40 @@ pub fn build(b: *std.Build) void {
         .{ .path = "system/tests/thread-test", .binary = thread_test_exe.getEmittedBin() },
     };
 
+    // The boot manifest: the FHS path of every bundled binary, one per line. The
+    // EFI loader reads THIS by name and opens each listed path by name — FAT
+    // name lookup is case-insensitive and firmware-portable, unlike directory
+    // ENUMERATION, whose returned names vary by firmware (bare 8.3 entries come
+    // back uppercase on some FAT drivers). The tree walk remains only as the
+    // loader's fallback for hand-assembled sticks without a manifest.
+    var manifest_text: std.ArrayListUnmanaged(u8) = .empty;
+    for (bundled) |item| {
+        manifest_text.append(b.allocator, '/') catch @panic("OOM");
+        manifest_text.appendSlice(b.allocator, item.path) catch @panic("OOM");
+        manifest_text.append(b.allocator, '\n') catch @panic("OOM");
+    }
+    const manifest_files = b.addWriteFiles();
+    const manifest_file = manifest_files.add("manifest", manifest_text.items);
+    const manifest_install = b.addInstallFileWithDir(manifest_file, .prefix, "system/manifest");
+    b.getInstallStep().dependOn(&manifest_install.step);
+
+    // The boot capsule: the same bundled list packed into ONE file (v2
+    // initial_ramdisk format), because a single open + sequential read is the
+    // only firmware file I/O shape that is fast everywhere — a per-file tree
+    // walk measured MINUTES on real firmware. The loader tries this first,
+    // then the manifest, then the walk; the running system cannot tell the
+    // difference (it always receives the same in-RAM table). Derived from the
+    // tree in the same build graph, so the two cannot drift.
+    const mk_capsule = b.addSystemCommand(&.{"python3"});
+    mk_capsule.addFileArg(b.path("tools/pack-system-image.py"));
+    const capsule_img = mk_capsule.addOutputFileArg("system.img");
+    for (bundled) |item| {
+        mk_capsule.addArg(item.path);
+        mk_capsule.addFileArg(item.binary);
+    }
+    const capsule_install = b.addInstallFile(capsule_img, "boot/system.img");
+    b.getInstallStep().dependOn(&capsule_install.step);
+
     // Install every bundled binary to its FHS home, so zig-out is a true image of
     // the filesystem — the same tree make-fat-image.py lays out on the boot volume.
     for (bundled) |item| {
@@ -669,7 +709,7 @@ pub fn build(b: *std.Build) void {
     // binaries at their FHS paths. QEMU presents this image as a USB mass-storage
     // device the guest boots from (see run-x86-64 and the test harness), and the
     // danos fat driver mounts the same image at /mnt/usb.
-    const fat_image = addBootImage(b, exe.getEmittedBin(), efiexe.getEmittedBin(), &bundled);
+    const fat_image = addBootImage(b, exe.getEmittedBin(), efiexe.getEmittedBin(), manifest_file, capsule_img, &bundled);
     const fat_image_install = b.addInstallFile(fat_image, "danos-usb.img");
     b.getInstallStep().dependOn(&fat_image_install.step);
 
@@ -678,7 +718,7 @@ pub fn build(b: *std.Build) void {
     // log captured to serial0 — without baking serial into the image users flash.
     // Built lazily (only when `run-x86-64` is requested), and never installed.
     const exe_serial = addKernel(b, kernel_target, optimize, kernel_modules, test_case, true);
-    const fat_image_serial = addBootImage(b, exe_serial.getEmittedBin(), efiexe.getEmittedBin(), &bundled);
+    const fat_image_serial = addBootImage(b, exe_serial.getEmittedBin(), efiexe.getEmittedBin(), manifest_file, capsule_img, &bundled);
 
     // `zig build check-fat-image` — validate the produced image is a real FAT32
     // with the EFI stub present (the builder's own --verify, no external tools).

tools/pack-system-image.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Pack the bundled binaries into the boot capsule (boot/system.img) — the
+single file the EFI loader reads in one sequential pass, which is the only
+shape firmware file I/O is fast at (a per-file tree walk measured minutes on
+real firmware). The format is the v2 initial_ramdisk (system/initial-ramdisk.zig):
+entries named by full FHS path, so the running system is identical whether the
+loader read the capsule or walked the tree.
+
+Usage: pack-system-image.py <out.img> [<path> <file>]...
+Layout (little-endian): Header{magic "DNR2", count}, Entry{name[64], offset, len}*N, blobs.
+"""
+import struct
+import sys
+
+MAGIC = 0x32524E44  # "DNR2"
+HEADER = struct.Struct("<II")
+ENTRY = struct.Struct("<64sQQ")
+
+
+def main() -> int:
+    out_path = sys.argv[1]
+    rest = sys.argv[2:]
+    if len(rest) % 2 != 0:
+        sys.stderr.write("usage: pack-system-image.py <out.img> [<path> <file>]...\n")
+        return 2
+    items = [(rest[i], rest[i + 1]) for i in range(0, len(rest), 2)]
+
+    table_end = HEADER.size + len(items) * ENTRY.size
+    entries = b""
+    blobs = []
+    offset = table_end
+    for path, source in items:
+        name = path if path.startswith("/") else "/" + path
+        encoded = name.encode("ascii")
+        if len(encoded) > 63:
+            sys.stderr.write(f"pack-system-image: path too long (>63): {name}\n")
+            return 2
+        with open(source, "rb") as f:
+            data = f.read()
+        entries += ENTRY.pack(encoded, offset, len(data))
+        blobs.append(data)
+        offset += len(data)
+
+    with open(out_path, "wb") as f:
+        f.write(HEADER.pack(MAGIC, len(items)))
+        f.write(entries)
+        for blob in blobs:
+            f.write(blob)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())