danos/boot/efi.zig

const std = @import("std");
const uefi = std.os.uefi;
const elf = std.elf;
const boot_handoff = @import("boot-handoff");
const initial_ramdisk = @import("initial-ramdisk");
const build_options = @import("build_options");
const BootInformation = boot_handoff.BootInformation;
const GraphicsOutput = uefi.protocol.GraphicsOutput;
const EdidActive = uefi.protocol.edid.Active;
const MemoryMapSlice = uefi.tables.MemoryMapSlice;

// The boot volume is the FHS-shaped zig-out (see build.zig / docs/README.md), so the
// loader reads each artifact from its addressed FHS path. UEFI paths use backslashes;
// the FAT driver walks the components itself, so no per-directory dance is needed.

/// The kernel image: /system/kernel.
const kernel_file_name = std.unicode.utf8ToUtf16LeStringLiteral("system\\kernel");

/// The user binaries: everything under /system except the kernel itself, plus
/// the test fixtures under /test. The loader walks both trees and packs them
/// into the in-RAM initial_ramdisk image — the volume's file structure is the
/// single source of truth (no packed image artifact on disk).
const system_directory_name = std.unicode.utf8ToUtf16LeStringLiteral("system");
const test_directory_name = std.unicode.utf8ToUtf16LeStringLiteral("test");

/// Physical page size, and the sentinel UEFI uses to seek to end-of-file.
const page_size = 4096;
const seek_end = 0xffff_ffff_ffff_ffff;

pub fn main() uefi.Status {
    // `boot` never returns on success — it jumps into the kernel. If it fails,
    // report the reason (boot services are still up) and park the machine so the
    // message stays on screen.
    boot() catch |err| {
        log("\r\nEFI: boot failed: ");
        logBytes(@errorName(err));
        log("\r\n");
        while (true) asm volatile ("hlt");
    };
    unreachable;
}

fn boot() !noreturn {
    const bs = uefi.system_table.boot_services orelse return error.NoBootServices;

    // Everything the kernel needs must be gathered *before* we exit boot
    // services, since afterwards none of these calls are usable.
    var boot_information: BootInformation = .{
        // A missing GOP (a headless machine) is not fatal — hand the kernel a
        // "no framebuffer" descriptor (base 0) and let it log to serial instead.
        .framebuffer = queryFramebuffer(bs) catch boot_handoff.Framebuffer{
            .base = 0,
            .width = 0,
            .height = 0,
            .pitch = 0,
            .format = .bgrx,
        },
        .memory_map = undefined, // filled by exitBootServices, just below
        .kernel_segments = undefined, // filled by loadKernel
        .kernel_segment_count = 0,
        // Read the ACPI RSDP from the UEFI configuration table now, while boot
        // services are still up. The pointer lives in ACPI reclaim memory, which
        // the kernel identity-maps, so the physical address stays valid afterward.
        .acpi_rsdp = if (acpiRootSystemDescriptorPointer()) |p| @intFromPtr(p) else 0,
    };

    const entry = try loadKernel(bs, &boot_information);

    // Best effort: a volume without a /system tree of user binaries still boots
    // (kernel-only). The tree — init included — becomes the initial_ramdisk.
    loadSystemTree(bs, &boot_information) catch |err| {
        log("EFI: no /system binaries (");
        logBytes(@errorName(err));
        log(") - booting without user space\r\n");
    };

    // Build the page tables the kernel starts life on: identity + a physmap of
    // low RAM, plus the higher-half kernel image once it links high. Allocated
    // now, while boot services (and the memory map) are still stable — nothing
    // is allocatable after ExitBootServices, and any allocation between fetching
    // the map and exiting would invalidate the map key.
    const cr3 = try buildBootstrapTables(bs, &boot_information);

    progress("EFI: kernel loaded, exiting boot services\r\n");
    boot_information.memory_map = try exitBootServices(bs);

    // Switch onto our tables and jump to the kernel in one uninterruptible step.
    // We load RDI explicitly (SystemV first arg) rather than trusting this UEFI
    // binary's Microsoft-x64 default, and jump straight to the (possibly
    // higher-half) entry — the bootstrap tables map both the low loader code
    // executing this and the kernel's link address.
    handoff(cr3, entry, &boot_information);
}

/// A display resolution in pixels.
const Resolution = struct { width: u32, height: u32, refresh_hz: u32 };

/// Switch the GPU to the monitor's native resolution (when we can determine it)
/// and read the resulting graphics mode into our own framebuffer description.
fn queryFramebuffer(bs: *uefi.tables.BootServices) !boot_handoff.Framebuffer {
    // Enumerate the handles carrying the Graphics Output Protocol. We go through
    // handles (rather than locateProtocol) so we can also ask them for their EDID,
    // which is what tells us the panel's native resolution.
    const handles = (try bs.locateHandleBuffer(.{ .by_protocol = &GraphicsOutput.guid })) orelse
        return error.NoGraphicsOutput;
    defer _ = bs.freePool(@ptrCast(handles.ptr)) catch {};

    const gop = (try bs.handleProtocol(GraphicsOutput, handles[0])) orelse
        return error.NoGraphicsOutput;

    // Best effort: the monitor's preferred (native) timing from its EDID.
    const native = nativeResolution(bs, handles);

    // Select the mode and switch to it if it isn't already current. setMode
    // updates gop.mode (info and frame_buffer_base) to describe the new mode.
    const target = pickMode(gop, native);
    if (target != gop.mode.mode) try gop.setMode(target);

    const info = gop.mode.info;
    return .{
        .base = @intCast(gop.mode.frame_buffer_base),
        .width = info.horizontal_resolution,
        .height = info.vertical_resolution,
        // Each pixel is 32 bits, so the byte pitch is 4 * pixels-per-row.
        .pitch = info.pixels_per_scan_line * 4,
        .format = try pixelFormat(info.pixel_format),
        // The refresh rate rides the EDID preferred timing. If the firmware kept a
        // non-native mode it may not describe that mode exactly — but it is the panel's
        // own clock, a far better frame-clock seed than a hardcoded 60 Hz.
        .refresh_hz = if (native) |n| n.refresh_hz else 0,
    };
}

/// Map a GOP pixel format to ours. bit_mask / blt_only have no linear 32bpp
/// layout we can paint into, so they're rejected.
fn pixelFormat(fmt: GraphicsOutput.PixelFormat) !boot_handoff.PixelFormat {
    return switch (fmt) {
        .red_green_blue_reserved_8_bit_per_color => .rgbx,
        .blue_green_red_reserved_8_bit_per_color => .bgrx,
        else => error.UnsupportedPixelFormat,
    };
}

/// Choose the graphics mode to boot with. If we learned the monitor's native
/// resolution from EDID and a mode offers it (with a layout we can paint into),
/// use that. Otherwise keep whatever mode the firmware already selected: with a
/// valid EDID present the firmware normally defaults to the native mode itself,
/// so its default is a far safer bet than second-guessing it with, say, the
/// largest advertised mode (which is often a huge non-native surface).
fn pickMode(gop: *GraphicsOutput, native: ?Resolution) u32 {
    const n = native orelse return gop.mode.mode;
    var id: u32 = 0;
    while (id < gop.mode.max_mode) : (id += 1) {
        const info = gop.queryMode(id) catch continue;
        _ = pixelFormat(info.pixel_format) catch continue; // must be paintable
        if (info.horizontal_resolution == n.width and
            info.vertical_resolution == n.height) return id;
    }
    return gop.mode.mode; // native not on offer; trust the firmware's default
}

/// The monitor's native resolution, read from an EDID's preferred timing. We try
/// every GOP handle and both EDID protocols (Active first, then Discovered),
/// since firmware installs them inconsistently — and many, including OVMF with
/// QEMU's stdvga, don't expose them at all. Returns null when none is found, in
/// which case pickMode keeps the firmware's default mode.
fn nativeResolution(bs: *uefi.tables.BootServices, handles: []uefi.Handle) ?Resolution {
    for (handles) |h| {
        if (bs.handleProtocol(EdidActive, h) catch null) |e| {
            if (e.edid) |p| if (edidNative(p[0..e.size_of_edid])) |r| return r;
        }
        if (bs.handleProtocol(uefi.protocol.edid.Discovered, h) catch null) |e| {
            if (e.edid) |p| if (edidNative(p[0..e.size_of_edid])) |r| return r;
        }
    }
    return null;
}

/// Parse the native resolution and refresh rate from a raw EDID block. The first
/// Detailed Timing Descriptor (at byte 54) is the preferred — i.e. native — mode by
/// convention; its active pixel counts are split across low bytes and the high nibbles
/// of later bytes. The refresh rate is derived, not stored: the descriptor carries the
/// pixel clock (10 kHz units) and the active+blanking extents, and
/// refresh = clock / (horizontal total × vertical total).
fn edidNative(edid: []const u8) ?Resolution {
    if (edid.len < 128) return null;
    // Every EDID begins with this fixed 8-byte header.
    const header = [_]u8{ 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
    if (!std.mem.eql(u8, edid[0..8], &header)) return null;

    const dtd = edid[54..][0..18];
    // A zero pixel clock marks a display (not timing) descriptor: no resolution.
    if (dtd[0] == 0 and dtd[1] == 0) return null;

    const w = @as(u32, dtd[2]) | (@as(u32, dtd[4] & 0xf0) << 4);
    const h = @as(u32, dtd[5]) | (@as(u32, dtd[7] & 0xf0) << 4);
    if (w == 0 or h == 0) return null;

    const clock_hz = (@as(u64, dtd[0]) | (@as(u64, dtd[1]) << 8)) * 10_000;
    const h_blank = @as(u64, dtd[3]) | (@as(u64, dtd[4] & 0x0f) << 8);
    const v_blank = @as(u64, dtd[6]) | (@as(u64, dtd[7] & 0x0f) << 8);
    const total = (@as(u64, w) + h_blank) * (@as(u64, h) + v_blank);
    const refresh: u32 = if (total == 0) 0 else @intCast((clock_hz + total / 2) / total);
    return .{ .width = w, .height = h, .refresh_hz = refresh };
}

/// Open the kernel on the volume we booted from, read it into a pool buffer,
/// load its segments, and return the physical entry-point address.
fn loadKernel(bs: *uefi.tables.BootServices, boot_information: *BootInformation) !usize {
    const loaded = (try bs.handleProtocol(uefi.protocol.LoadedImage, uefi.handle)) orelse
        return error.NoLoadedImage;
    const device = loaded.device_handle orelse return error.NoBootDevice;
    const fs = (try bs.handleProtocol(uefi.protocol.SimpleFileSystem, device)) orelse
        return error.NoFileSystem;

    const root = try fs.openVolume();
    defer _ = root.close() catch {};

    const file = try root.open(kernel_file_name, .read, .{});
    defer _ = file.close() catch {};

    // Seek to the end to learn the size, then rewind.
    try file.setPosition(seek_end);
    const size: usize = @intCast(try file.getPosition());
    try file.setPosition(0);

    const image = try bs.allocatePool(.loader_data, size);
    defer _ = bs.freePool(image.ptr) catch {};

    // `read` may return short; loop until the whole file is in memory.
    var read_total: usize = 0;
    while (read_total < size) {
        const n = try file.read(image[read_total..]);
        if (n == 0) return error.UnexpectedEof;
        read_total += n;
    }

    return loadElf(bs, image, boot_information);
}

// --- bootstrap page tables -------------------------------------------------
// The kernel is (or will be) linked in the higher half but loaded low; the
// firmware's identity map doesn't cover the higher half, so the loader builds
// the first set of real page tables and switches CR3 before jumping in. They
// carry: an identity map of low RAM (so the loader's own code/stack executing
// the switch stays valid, and the low-linked kernel keeps working during the
// staged move), a physmap at boot_handoff.physmap_base (the kernel's permanent way to
// reach physical memory), and 4 KiB mappings of any higher-half kernel segment.
// The kernel later builds its own precise tables (paging.init) and abandons
// these; they leak as reserved LoaderData (~a handful of frames).

const pte_present: u64 = 1 << 0;
const pte_write: u64 = 1 << 1;
const pte_ps: u64 = 1 << 7; // page-size: a 2 MiB leaf at the PD level
const pte_address: u64 = 0x000F_FFFF_FFFF_F000;
const gib: u64 = 1 << 30;

/// A bump allocator over a pre-reserved block of zeroed frames, for page tables.
const TablePool = struct {
    base: usize,
    next: usize,
    cap: usize,

    fn alloc(self: *TablePool) !u64 {
        if (self.next >= self.cap) return error.OutOfBootstrapFrames;
        const frame = self.base + self.next * page_size;
        self.next += 1;
        @memset(@as(*[512]u64, @ptrFromInt(frame)), 0);
        return frame;
    }

    fn table(physical: u64) *[512]u64 {
        return @ptrFromInt(physical);
    }

    /// Return the next-level table an entry points at, creating it if absent.
    fn descend(self: *TablePool, entry: *u64) !u64 {
        if (entry.* & pte_present != 0) return entry.* & pte_address;
        const frame = try self.alloc();
        entry.* = frame | pte_present | pte_write;
        return frame;
    }

    fn map2M(self: *TablePool, pml4: u64, virtual: u64, physical: u64) !void {
        const pml4e = &table(pml4)[(virtual >> 39) & 0x1FF];
        const pdpt = try self.descend(pml4e);
        const pdpte = &table(pdpt)[(virtual >> 30) & 0x1FF];
        const pd = try self.descend(pdpte);
        table(pd)[(virtual >> 21) & 0x1FF] = (physical & ~@as(u64, 0x1F_FFFF)) | pte_present | pte_write | pte_ps;
    }

    fn map4K(self: *TablePool, pml4: u64, virtual: u64, physical: u64) !void {
        const pml4e = &table(pml4)[(virtual >> 39) & 0x1FF];
        const pdpt = try self.descend(pml4e);
        const pdpte = &table(pdpt)[(virtual >> 30) & 0x1FF];
        const pd = try self.descend(pdpte);
        const pde = &table(pd)[(virtual >> 21) & 0x1FF];
        const pt = try self.descend(pde);
        table(pt)[(virtual >> 12) & 0x1FF] = (physical & pte_address) | pte_present | pte_write;
    }
};

/// Build the bootstrap tables and return the physical PML4 address (for CR3).
/// No NX bits are set anywhere, so EFER.NXE (still off here) is irrelevant.
fn buildBootstrapTables(bs: *uefi.tables.BootServices, boot_information: *const BootInformation) !u64 {
    // 64 frames (256 KiB) — comfortably covers a PML4, two PDPTs, eight PDs for
    // the 4 GiB identity+physmap ranges, plus the kernel image's PTs.
    const pool_pages = 64;
    const block = try bs.allocatePages(.any, .loader_data, pool_pages);
    var pool = TablePool{ .base = @intFromPtr(block.ptr), .next = 0, .cap = pool_pages };

    const pml4 = try pool.alloc();

    // Identity + physmap for low RAM. 4 GiB covers all of QEMU's RAM and MMIO
    // (LAPIC/IOAPIC/HPET/ECAM/framebuffer under q35); a machine with RAM or a
    // framebuffer above 4 GiB would extend this — see the fb window below.
    var address: u64 = 0;
    while (address < 4 * gib) : (address += 2 << 20) {
        try pool.map2M(pml4, address, address); // identity
        try pool.map2M(pml4, boot_handoff.physicalToVirtual(address), address); // physmap
    }

    // A framebuffer above the 4 GiB window needs its own identity + physmap
    // pages (the kernel touches fb.base before it builds its own tables).
    const fb = boot_information.framebuffer;
    if (fb.present() and fb.base + @as(u64, fb.pitch) * fb.height > 4 * gib) {
        var p: u64 = fb.base & ~@as(u64, 0x1F_FFFF);
        const fb_end = fb.base + @as(u64, fb.pitch) * fb.height;
        while (p < fb_end) : (p += 2 << 20) {
            try pool.map2M(pml4, p, p);
            try pool.map2M(pml4, boot_handoff.physicalToVirtual(p), p);
        }
    }

    // Higher-half kernel segments (virtual != physical). While the kernel still links
    // low its segments sit in the identity range and need no separate mapping
    // (and 4 KiB-mapping them would collide with the 2 MiB identity leaves), so
    // only map segments that actually live in the higher half.
    for (boot_information.kernel_segments[0..boot_information.kernel_segment_count]) |seg| {
        if (seg.virtual < boot_handoff.kernel_virt_base) continue;
        var off: u64 = 0;
        while (off < seg.pages * page_size) : (off += page_size) {
            try pool.map4K(pml4, seg.virtual + off, seg.physical + off);
        }
    }

    return pml4;
}

/// Switch onto `cr3` and jump to the kernel `entry` with `boot_information` in RDI,
/// interrupts off, in one block so nothing runs between the CR3 load and the
/// jump. The identity mapping keeps this low loader code valid across the CR3
/// load; the jump target is mapped (identity while low, higher-half once high).
fn handoff(cr3: u64, entry: usize, boot_information: *const BootInformation) noreturn {
    asm volatile (
        \\cli
        \\movq %[cr3], %%cr3
        \\movq %[bi], %%rdi
        \\callq *%[entry]
        :
        : [cr3] "r" (cr3),
          [bi] "r" (boot_information),
          [entry] "r" (entry),
        : .{ .memory = true });
    unreachable;
}

// --- the /system and /test trees -> initial_ramdisk --------------------------

/// Cap on bundled binaries. Generous: the tree carries ~30 today.
const maximum_bundled = 64;

/// How deep the walk goes below a tree root ("/system/services/x" is depth 1,
/// "/test/system/services/x" is depth 2).
const maximum_tree_depth = 3;

/// One binary discovered under a walked tree: its FHS path (UTF-8,
/// '/'-separated, NUL-free) and its contents in a transient pool buffer.
const Bundled = struct {
    path: [initial_ramdisk.maximum_name]u8,
    path_len: usize,
    data: []align(8) u8,
};

/// Gather the boot volume's user binaries into an in-RAM v2 initial_ramdisk
/// image, entries named by full FHS path — the volume's file structure is the
/// single source of truth (no packed ramdisk artifact; init travels in the
/// table like everything else).
///
/// Two strategies, most portable first:
///   1. /system/manifest (written by the build): each listed path is opened BY
///      NAME — the case-insensitive lookup every firmware FAT driver gets
///      right, and the only file access the pre-tree loader ever used.
///   2. No manifest: ENUMERATE the /system and /test trees. Portable in
///      principle, but firmware differs in what names enumeration returns
///      (bare 8.3 entries come back uppercase on some drivers), so this is
///      the fallback for hand-assembled sticks, not the primary path.
fn loadSystemTree(bs: *uefi.tables.BootServices, boot_information: *BootInformation) !void {
    const loaded = (try bs.handleProtocol(uefi.protocol.LoadedImage, uefi.handle)) orelse
        return error.NoLoadedImage;
    const device = loaded.device_handle orelse return error.NoBootDevice;
    const fs = (try bs.handleProtocol(uefi.protocol.SimpleFileSystem, device)) orelse
        return error.NoFileSystem;

    const root = try fs.openVolume();
    defer _ = root.close() catch {};

    // Unconditional breadcrumb (con_out, independent of -Dserial): this phase
    // is where a slow firmware stalls, and a silent black screen here already
    // cost a real-hardware debugging session.
    log("EFI: loading the system...\r\n");

    // The capsule (boot\system.img) first: one open + one sequential read is
    // the only firmware file I/O shape that is fast everywhere. It is already
    // the kernel's wire format — hand it over as-is.
    if (loadCapsule(bs, root, boot_information)) {
        log("EFI: system image loaded, starting the kernel\r\n");
        return;
    }

    var list: [maximum_bundled]Bundled = undefined;
    var count: usize = 0;

    loadByManifest(bs, root, &list, &count) catch {
        count = 0; // a torn manifest read leaves partial entries; start over
    };
    if (count == 0) {
        const system_directory = try root.open(system_directory_name, .read, .{});
        defer _ = system_directory.close() catch {};
        try walkDirectory(bs, system_directory, "/system", 0, &list, &count);
        // The /test tree is optional: a stick without fixtures still boots.
        if (root.open(test_directory_name, .read, .{})) |test_directory| {
            defer _ = test_directory.close() catch {};
            try walkDirectory(bs, test_directory, "/test", 0, &list, &count);
        } else |_| {}
    }
    if (count == 0) return error.NoBinaries;

    // Assemble the v2 image: header, entry table, then the blobs.
    const table_end = @sizeOf(initial_ramdisk.Header) + count * @sizeOf(initial_ramdisk.Entry);
    var total: usize = table_end;
    for (list[0..count]) |e| total += e.data.len;

    const image = try bs.allocatePool(.loader_data, total); // survives the handoff
    std.mem.bytesAsValue(initial_ramdisk.Header, image[0..@sizeOf(initial_ramdisk.Header)]).* = .{
        .magic = initial_ramdisk.magic,
        .count = @intCast(count),
    };
    var offset: usize = table_end;
    for (list[0..count], 0..) |e, i| {
        var record = initial_ramdisk.Entry{ .name = @splat(0), .offset = offset, .len = e.data.len };
        @memcpy(record.name[0..e.path_len], e.path[0..e.path_len]);
        const slot = image[@sizeOf(initial_ramdisk.Header) + i * @sizeOf(initial_ramdisk.Entry) ..][0..@sizeOf(initial_ramdisk.Entry)];
        std.mem.bytesAsValue(initial_ramdisk.Entry, slot).* = record;
        @memcpy(image[offset..][0..e.data.len], e.data);
        offset += e.data.len;
        _ = bs.freePool(e.data.ptr) catch {};
    }

    boot_information.initial_ramdisk_base = @intFromPtr(image.ptr);
    boot_information.initial_ramdisk_len = total;
    log("EFI: boot tree loaded, starting the kernel\r\n");
}

/// The boot capsule: the bundled binaries as one v2 initial_ramdisk image.
const capsule_file_name = std.unicode.utf8ToUtf16LeStringLiteral("boot\\system.img");

/// Load boot\system.img whole and hand it to the kernel unmodified — it is
/// already the initial_ramdisk wire format. Returns false (capsule absent or
/// unreadable or wrong magic) to let the caller fall back to per-file loading.
fn loadCapsule(bs: *uefi.tables.BootServices, root: *uefi.protocol.File, boot_information: *BootInformation) bool {
    const file = root.open(capsule_file_name, .read, .{}) catch return false;
    defer _ = file.close() catch {};
    const image = readWholeFile(bs, file) catch return false;
    if (image.len < @sizeOf(initial_ramdisk.Header) or
        std.mem.bytesToValue(initial_ramdisk.Header, image[0..@sizeOf(initial_ramdisk.Header)]).magic != initial_ramdisk.magic)
    {
        _ = bs.freePool(image.ptr) catch {};
        return false;
    }
    boot_information.initial_ramdisk_base = @intFromPtr(image.ptr);
    boot_information.initial_ramdisk_len = image.len;
    return true;
}

/// The manifest path, and a scratch limit for its UTF-16 conversion.
const manifest_file_name = std.unicode.utf8ToUtf16LeStringLiteral("system\\manifest");

/// Load every binary the manifest lists, opening each path by name from the
/// volume root. A listed-but-unopenable file is skipped (the kernel reports the
/// absence); a missing manifest errors so the caller falls back to the walk.
fn loadByManifest(bs: *uefi.tables.BootServices, root: *uefi.protocol.File, list: *[maximum_bundled]Bundled, count: *usize) !void {
    const manifest_handle = try root.open(manifest_file_name, .read, .{});
    var manifest_open = true;
    defer if (manifest_open) {
        _ = manifest_handle.close() catch {};
    };
    const manifest = try readWholeFile(bs, manifest_handle);
    _ = manifest_handle.close() catch {};
    manifest_open = false;
    defer _ = bs.freePool(manifest.ptr) catch {};

    var lines = std.mem.tokenizeAny(u8, manifest, "\r\n");
    while (lines.next()) |line| {
        if (line.len < 2 or line[0] != '/') continue;
        if (line.len >= initial_ramdisk.maximum_name) continue;
        if (count.* == maximum_bundled) return;

        // "/system/services/init" -> UTF-16 "system\services\init".
        var name16: [initial_ramdisk.maximum_name]u16 = undefined;
        var i: usize = 0;
        for (line[1..]) |c| {
            name16[i] = if (c == '/') '\\' else c;
            i += 1;
        }
        name16[i] = 0;

        const file = root.open(@ptrCast(name16[0..i :0]), .read, .{}) catch continue;
        defer _ = file.close() catch {};
        const data = readWholeFile(bs, file) catch continue;

        var entry: *Bundled = &list[count.*];
        @memcpy(entry.path[0..line.len], line);
        entry.path_len = line.len;
        entry.data = data;
        count.* += 1;
    }
}

/// Recursively collect the regular files below `directory` into `list`. Top-level
/// files (depth 0) are skipped: the only one is /system/kernel, which loadKernel
/// has already consumed and which is not a spawnable user binary (/test has no
/// top-level files, so the skip is a no-op there).
fn walkDirectory(
    bs: *uefi.tables.BootServices,
    directory: *uefi.protocol.File,
    prefix: []const u8,
    depth: usize,
    list: *[maximum_bundled]Bundled,
    count: *usize,
) !void {
    // Each read() on a directory yields one EFI_FILE_INFO; zero bytes means done.
    var info_buffer: [1024]u8 align(8) = undefined;
    while (true) {
        const n = try directory.read(&info_buffer);
        if (n == 0) return;
        const info: *const uefi.protocol.File.Info.File = @ptrCast(@alignCast(&info_buffer));
        const name16 = info.getFileName();

        // Convert the (ASCII in practice) UTF-16 name. A hostile-shaped entry
        // (too long, non-ASCII) is SKIPPED, never fatal — one odd file on a
        // hand-written stick must not cost the whole boot. Names are lowered:
        // the danos tree is canonically lowercase and FAT lookups are
        // case-insensitive, but firmware ENUMERATION returns whatever the
        // directory stores — an 8.3 short entry comes back uppercase ("INIT"),
        // which would otherwise poison every path comparison downstream.
        var name_buffer: [initial_ramdisk.maximum_name]u8 = undefined;
        var name_length: usize = 0;
        var name_ok = true;
        while (name16[name_length] != 0) : (name_length += 1) {
            if (name_length == name_buffer.len) {
                name_ok = false;
                break;
            }
            const c = name16[name_length];
            if (c > 0x7F) {
                name_ok = false;
                break;
            }
            name_buffer[name_length] = std.ascii.toLower(@intCast(c));
        }
        if (!name_ok) continue;
        const name = name_buffer[0..name_length];
        // Skip dot entries: "." / ".." and host-OS litter (macOS "._*" AppleDouble
        // resource forks, ".fseventsd", ".Spotlight-V100") a copied-onto stick
        // accumulates — none of it is a danos binary.
        if (name.len == 0 or name[0] == '.') continue;

        if (info.attribute.directory) {
            if (depth == maximum_tree_depth) continue;
            var child_prefix: [initial_ramdisk.maximum_name]u8 = undefined;
            const child = try std.fmt.bufPrint(&child_prefix, "{s}/{s}", .{ prefix, name });
            const child_directory = try directory.open(name16, .read, .{});
            defer _ = child_directory.close() catch {};
            try walkDirectory(bs, child_directory, child, depth + 1, list, count);
            continue;
        }

        if (depth == 0) continue; // /system/kernel — already loaded, not bundled
        if (count.* == maximum_bundled) return error.TooManyBinaries;

        var entry: *Bundled = &list[count.*];
        const path = std.fmt.bufPrint(&entry.path, "{s}/{s}", .{ prefix, name }) catch continue; // path too long: skip the file, keep the boot
        entry.path_len = path.len;

        const file = directory.open(name16, .read, .{}) catch continue;
        defer _ = file.close() catch {};
        entry.data = readWholeFile(bs, file) catch continue; // unreadable/empty: skip
        count.* += 1;
    }
}

/// Read an open file completely into a fresh pool buffer that survives the
/// handoff (LoaderData is classified reserved, so the kernel identity-maps it).
fn readWholeFile(bs: *uefi.tables.BootServices, file: *uefi.protocol.File) ![]align(8) u8 {
    try file.setPosition(seek_end);
    const size: usize = @intCast(try file.getPosition());
    try file.setPosition(0);
    if (size == 0) return error.EmptyFile;

    const buffer = try bs.allocatePool(.loader_data, size);
    var read_total: usize = 0;
    while (read_total < size) {
        const n = try file.read(buffer[read_total..]);
        if (n == 0) return error.UnexpectedEof;
        read_total += n;
    }
    return buffer[0..size];
}

/// Validate the ELF, copy every PT_LOAD segment to its physical address, and
/// record each segment's layout so the kernel can re-map itself with the right
/// permissions.
fn loadElf(bs: *uefi.tables.BootServices, image: []u8, boot_information: *BootInformation) !usize {
    if (image.len < @sizeOf(elf.Elf64_Ehdr)) return error.NotElf;
    const ehdr: *const elf.Elf64_Ehdr = @ptrCast(@alignCast(image.ptr));

    if (ehdr.e_ident[0] != 0x7f or ehdr.e_ident[1] != 'E' or
        ehdr.e_ident[2] != 'L' or ehdr.e_ident[3] != 'F') return error.NotElf;
    if (ehdr.e_machine != .X86_64) return error.WrongArchitecture;

    var i: usize = 0;
    while (i < ehdr.e_phnum) : (i += 1) {
        const phdr: *const elf.Elf64_Phdr = @ptrCast(@alignCast(
            image.ptr + ehdr.e_phoff + i * ehdr.e_phentsize,
        ));
        if (phdr.p_type != elf.PT_LOAD) continue;

        // Reserve the exact physical pages this segment is linked at. This
        // requires the segment's p_paddr to be free in the firmware memory map;
        // if it collides, adjust `image_base` in build.zig. (Once the kernel
        // links high — M2 step 4 — p_paddr becomes a separate low load address
        // via the linker's AT(), and this stays a valid physical allocation.)
        const mem_sz: usize = @intCast(phdr.p_memsz);
        const pages = (mem_sz + page_size - 1) / page_size;
        const dest: [*]align(page_size) uefi.Page = @ptrFromInt(phdr.p_paddr);
        _ = try bs.allocatePages(.{ .address = dest }, .loader_data, pages);

        // Copy the file-backed part, then zero the .bss tail (memsz > filesz).
        const bytes: [*]u8 = @ptrFromInt(phdr.p_paddr);
        const file_sz: usize = @intCast(phdr.p_filesz);
        const off: usize = @intCast(phdr.p_offset);
        @memcpy(bytes[0..file_sz], image[off..][0..file_sz]);
        @memset(bytes[file_sz..mem_sz], 0);

        // Record the virtual link address and the physical load address so the
        // kernel can map itself with the right permissions post-switch. They're
        // equal while the kernel links low; they diverge once it links high.
        const n = boot_information.kernel_segment_count;
        if (n < boot_information.kernel_segments.len) {
            boot_information.kernel_segments[n] = .{
                .virtual = phdr.p_vaddr,
                .physical = phdr.p_paddr,
                .pages = pages,
                .flags = phdr.p_flags,
            };
            boot_information.kernel_segment_count = n + 1;
        }
    }

    return @intCast(ehdr.e_entry);
}

/// Fetch the memory map, exit boot services, and hand back the map in danos's
/// neutral form. Allocating the buffers can itself change the map (invalidating
/// the key), so retry until it takes. Both buffers are LoaderData, which survives
/// the exit, so the returned map stays valid for the kernel.
fn exitBootServices(bs: *uefi.tables.BootServices) !boot_handoff.MemoryMap {
    var attempts: usize = 0;
    while (attempts < 8) : (attempts += 1) {
        const info = try bs.getMemoryMapInfo();
        // Spare descriptors to absorb the growth from the allocations below.
        const cap = info.len + 8;
        const map_buffer = try bs.allocatePool(.loader_data, cap * info.descriptor_size);
        const regions_buffer = try bs.allocatePool(.loader_data, cap * @sizeOf(boot_handoff.MemoryRegion));
        const map = bs.getMemoryMap(map_buffer) catch {
            _ = bs.freePool(map_buffer.ptr) catch {};
            _ = bs.freePool(regions_buffer.ptr) catch {};
            continue;
        };
        bs.exitBootServices(uefi.handle, map.info.key) catch {
            _ = bs.freePool(map_buffer.ptr) catch {};
            _ = bs.freePool(regions_buffer.ptr) catch {};
            continue;
        };
        // Boot services are gone; do not touch `bs` again. Converting the map is
        // pure computation on memory we already hold, so it's safe here.
        return convertMemoryMap(map, regions_buffer);
    }
    return error.ExitBootServicesFailed;
}

/// Translate UEFI's memory map into danos's neutral `MemoryRegion` array, written
/// into `out` (sized for at least `map.info.len` regions). Adjacent regions of
/// the same kind are coalesced. This is the loader's job precisely so the kernel
/// never sees UEFI's vocabulary — the same seam the framebuffer already uses.
fn convertMemoryMap(map: MemoryMapSlice, out: []u8) boot_handoff.MemoryMap {
    const regions: [*]boot_handoff.MemoryRegion = @ptrCast(@alignCast(out.ptr));
    // We're about to call boot-services memory `usable`, but our own stack lives
    // in it and the kernel starts out running on it. Keep the region holding the
    // current stack pointer reserved so it's never handed out.
    const rsp = asm volatile ("mov %%rsp, %[out]"
        : [out] "=r" (-> usize),
    );
    var count: usize = 0;
    var i: usize = 0;
    while (i < map.info.len) : (i += 1) {
        // Stride by descriptor_size, NOT @sizeOf — firmware descriptors may be
        // larger than the struct.
        const d: *const uefi.tables.MemoryDescriptor =
            @ptrCast(@alignCast(map.ptr + i * map.info.descriptor_size));
        if (d.number_of_pages == 0) continue;
        var kind = classify(d);
        // The descriptor we're executing on stays reserved (see rsp above).
        const region_end = d.physical_start + d.number_of_pages * page_size;
        if (kind == .usable and rsp >= d.physical_start and rsp < region_end) kind = .reserved;

        // Coalesce with the previous region if it's the same kind and contiguous.
        if (count > 0) {
            const previous = &regions[count - 1];
            if (previous.kind == kind and
                previous.base + previous.pages * page_size == d.physical_start)
            {
                previous.pages += d.number_of_pages;
                continue;
            }
        }
        regions[count] = .{
            .base = d.physical_start,
            .pages = d.number_of_pages,
            .kind = kind,
        };
        count += 1;
    }
    return .{ .regions = @intFromPtr(regions), .len = count };
}

/// Map a UEFI descriptor to danos's neutral kind. A region that isn't
/// writeback-cacheable (`wb`) isn't backed by real RAM — it's device registers or
/// a reserved address-space window (e.g. PCIe configuration space) — so it's `mmio`
/// regardless of type. UEFI overloads `reserved_memory_type` for both reserved RAM
/// and such holes, and the cache attribute is what actually tells them apart.
///
/// Boot-services memory is folded straight into `usable`: we've already called
/// ExitBootServices, so it's free RAM now — the kernel never needs to know it was
/// ever the firmware's (the one live piece, our stack, is reserved by the caller).
/// Anything unrecognised is `reserved` — the safe default; our own LoaderData (the
/// kernel image and these buffers) lands there and stays reserved.
fn classify(d: *const uefi.tables.MemoryDescriptor) boot_handoff.MemoryKind {
    if (!d.attribute.wb) return .mmio;
    return switch (d.type) {
        .conventional_memory, .boot_services_code, .boot_services_data => .usable,
        .acpi_reclaim_memory => .acpi_tables,
        .acpi_memory_nvs => .acpi_nvs,
        .memory_mapped_io, .memory_mapped_io_port_space => .mmio,
        else => .reserved,
    };
}

/// Write a compile-time string to the console (best effort).
fn log(comptime message: []const u8) void {
    const out = uefi.system_table.con_out orelse return;
    _ = out.outputString(std.unicode.utf8ToUtf16LeStringLiteral(message)) catch {};
}

/// A boot-progress breadcrumb: like `log`, but compiled out unless `-Dserial`
/// (off by default), so a real-hardware boot stays silent. Fatal errors use
/// `log` directly and always show, so a failed boot still explains itself.
fn progress(comptime message: []const u8) void {
    if (!build_options.serial) return;
    log(message);
}

/// Write a runtime ASCII byte string (e.g. an @errorName) by widening to UTF-16.
fn logBytes(bytes: []const u8) void {
    const out = uefi.system_table.con_out orelse return;
    var buffer: [128]u16 = undefined;
    var i: usize = 0;
    for (bytes) |b| {
        if (i + 1 >= buffer.len) break;
        buffer[i] = b;
        i += 1;
    }
    buffer[i] = 0;
    _ = out.outputString(buffer[0..i :0].ptr) catch {};
}

fn acpiRootSystemDescriptorPointer() ?*const anyopaque {
    const table_entries = uefi.system_table.number_of_table_entries;
    const configuration_tables = uefi.system_table.configuration_table;
    const acpi2 = uefi.tables.ConfigurationTable.acpi_20_table_guid;
    const acpi1 = uefi.tables.ConfigurationTable.acpi_10_table_guid;

    for (0..table_entries) |i| {
        const entry = configuration_tables[i];
        if (entry.vendor_guid.eql(acpi2) or entry.vendor_guid.eql(acpi1)) {
            return entry.vendor_table;
        }
    }

    return null;
}