From 28b4dabbaa2deeab9ae39a3417676f2fb2b412d3 Mon Sep 17 00:00:00 2001 From: Daniel Samson <12231216+daniel-samson@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:05:08 +0100 Subject: [PATCH] serial: make the log sink a build option, off by default Serial is now a QEMU/dev aid, not a real-hardware necessity: a legacy-free board often has no live COM1, and the boot log is kept in RAM (klog) and flushed to disk. So the serial sink is compiled in only under -Dserial (default false). - kernel.zig gates serialInit + the log sink on build_options.serial - boot/efi.zig gates its EFI: progress breadcrumbs (con_out) via progress(); fatal-error messages stay always-on so a failed boot still explains itself - run-x86-64 boots a serial-enabled image variant (factored addKernel/addBootImage helpers) so a dev boot always captures serial0, without baking serial into the flashable image - test/qemu_test.py builds -Dserial=true (it asserts on serial markers) - the loopback probe stays as a real-HW safety net for -Dserial images --- boot/efi.zig | 15 ++- build.zig | 191 ++++++++++++++++++++++++++++----------- docs/testing.md | 9 ++ system/kernel/kernel.zig | 14 ++- test/qemu_test.py | 5 +- 5 files changed, 174 insertions(+), 60 deletions(-) diff --git a/boot/efi.zig b/boot/efi.zig index f827122..6114311 100644 --- a/boot/efi.zig +++ b/boot/efi.zig @@ -2,6 +2,7 @@ const std = @import("std"); const uefi = std.os.uefi; const elf = std.elf; const boot_handoff = @import("boot-handoff"); +const build_options = @import("build_options"); const BootInformation = boot_handoff.BootInformation; const GraphicsOutput = uefi.protocol.GraphicsOutput; const EdidActive = uefi.protocol.edid.Active; @@ -84,7 +85,7 @@ fn boot() !noreturn { // the map and exiting would invalidate the map key. const cr3 = try buildBootstrapTables(bs, &boot_information); - log("EFI: kernel loaded, exiting boot services\r\n"); + progress("EFI: kernel loaded, exiting boot services\r\n"); boot_information.memory_map = try exitBootServices(bs); // Switch onto our tables and jump to the kernel in one uninterruptible step. @@ -395,7 +396,7 @@ fn loadInit(bs: *uefi.tables.BootServices, boot_information: *BootInformation) ! const image = try loadFile(bs, init_file_name); boot_information.init_base = @intFromPtr(image.ptr); boot_information.init_len = image.len; - log("EFI: /system/services/init loaded\r\n"); + progress("EFI: /system/services/init loaded\r\n"); } /// Ferry the initial_ramdisk (the VFS server + drivers) to the kernel, same as init. @@ -403,7 +404,7 @@ fn loadInitialRamdisk(bs: *uefi.tables.BootServices, boot_information: *BootInfo const image = try loadFile(bs, initial_ramdisk_file_name); boot_information.initial_ramdisk_base = @intFromPtr(image.ptr); boot_information.initial_ramdisk_len = image.len; - log("EFI: initial_ramdisk loaded\r\n"); + progress("EFI: initial_ramdisk loaded\r\n"); } /// Validate the ELF, copy every PT_LOAD segment to its physical address, and @@ -561,6 +562,14 @@ fn log(comptime message: []const u8) void { _ = out.outputString(std.unicode.utf8ToUtf16LeStringLiteral(message)) catch {}; } +/// A boot-progress breadcrumb: like `log`, but compiled out unless `-Dserial` +/// (off by default), so a real-hardware boot stays silent. Fatal errors use +/// `log` directly and always show, so a failed boot still explains itself. +fn progress(comptime message: []const u8) void { + if (!build_options.serial) return; + log(message); +} + /// Write a runtime ASCII byte string (e.g. an @errorName) by widening to UTF-16. fn logBytes(bytes: []const u8) void { const out = uefi.system_table.con_out orelse return; diff --git a/build.zig b/build.zig index 304bc9e..29a1619 100644 --- a/build.zig +++ b/build.zig @@ -96,6 +96,105 @@ fn addUserBinary( return exe; } +/// The modules the kernel imports, gathered once so both kernel variants (the +/// installed one and the serial-enabled one `run-x86-64` boots) are built from +/// the same set. `build_options` is *not* here — it carries `serial`/`test_case`, +/// which differ per variant, so `addKernel` builds it fresh each time. +const KernelModules = struct { + boot_handoff: *std.Build.Module, + abi: *std.Build.Module, + device_abi: *std.Build.Module, + architecture: *std.Build.Module, + platform: *std.Build.Module, + parameters: *std.Build.Module, + initial_ramdisk: *std.Build.Module, +}; + +/// Build the freestanding x86_64 kernel ELF. Factored so we can build it twice +/// from one recipe: the installed/flashable image (serial off by default) and the +/// serial-enabled variant `run-x86-64` boots — they differ only in the `serial` +/// build option baked into `build_options`. +fn addKernel( + b: *std.Build, + kernel_target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + modules: KernelModules, + test_case: ?[]const u8, + serial: bool, +) *std.Build.Step.Compile { + // Compile-time configuration the kernel reads as `@import("build_options")`: + // the QEMU harness's -Dtest-case, and whether the serial log sink is compiled + // in (see the -Dserial option). Built per variant since `serial` differs. + const build_options = b.addOptions(); + build_options.addOption(?[]const u8, "test_case", test_case); + build_options.addOption(bool, "serial", serial); + const build_options_module = build_options.createModule(); + + const exe = b.addExecutable(.{ + .name = "kernel", + .root_module = b.createModule(.{ + .root_source_file = b.path("system/kernel/kernel.zig"), + .target = kernel_target, + .optimize = optimize, + .code_model = .kernel, // kernel runs in the top 2 GiB (higher half) + .red_zone = false, // interrupts would corrupt the SystemV red zone + .single_threaded = false, // SMP: the big kernel lock's atomics must be real across cores + .sanitize_c = .off, // the UBSan runtime needs f128/SSE support we don't provide + .stack_check = false, // stack-probe calls have no runtime to land in + .stack_protector = false, + .imports = &.{ + .{ .name = "boot-handoff", .module = modules.boot_handoff }, + .{ .name = "abi", .module = modules.abi }, + .{ .name = "device-abi", .module = modules.device_abi }, + .{ .name = "architecture", .module = modules.architecture }, + .{ .name = "platform", .module = modules.platform }, + .{ .name = "parameters", .module = modules.parameters }, + .{ .name = "build_options", .module = build_options_module }, + .{ .name = "initial-ramdisk", .module = modules.initial_ramdisk }, + }, + }), + }); + exe.setLinkerScript(b.path("system/kernel/architecture/x86_64/linker.ld")); + exe.entry = .{ .symbol_name = "_start" }; + // The self-hosted linker ignores parts of the linker script (PHDRS, + // /DISCARD/, AT(), section order); the higher-half layout depends on the + // script being authoritative, so pin the kernel to LLVM + LLD. + exe.use_llvm = true; + exe.use_lld = true; + // Higher-half virtual base (matches KERNEL_VIRT_BASE in linker.ld); the + // linker's AT() clauses give each segment a low physical load address + // (.text at 1 MiB), which the loader allocates and copies into. + exe.image_base = 0xFFFFFFFF80100000; + return exe; +} + +/// Assemble the bootable FAT32 image (the in-repo Python builder) holding what +/// the firmware and loader need off the ESP: the EFI stub, `kernel`, `init`, and +/// the initial-ramdisk. Factored so the serial-enabled `run-x86-64` variant can +/// bundle its own kernel while sharing the (serial-independent) loader, init, and +/// ramdisk. Returns the image's LazyPath. +fn addBootImage( + b: *std.Build, + kernel_bin: std.Build.LazyPath, + efi_bin: std.Build.LazyPath, + init_bin: std.Build.LazyPath, + initial_ramdisk_img: std.Build.LazyPath, +) std.Build.LazyPath { + const mk_fat = b.addSystemCommand(&.{"python3"}); + mk_fat.addFileArg(b.path("tools/make-fat-image.py")); + const fat_image = mk_fat.addOutputFileArg("danos-usb.img"); + mk_fat.addArg("64"); // MiB + mk_fat.addArg("EFI/BOOT/BOOTX64.efi"); + mk_fat.addFileArg(efi_bin); + mk_fat.addArg("system/kernel"); + mk_fat.addFileArg(kernel_bin); + mk_fat.addArg("system/services/init"); + mk_fat.addFileArg(init_bin); + mk_fat.addArg("boot/initial-ramdisk.img"); + mk_fat.addFileArg(initial_ramdisk_img); + return fat_image; +} + pub fn build(b: *std.Build) void { ensureZigVersion(); @@ -285,9 +384,12 @@ pub fn build(b: *std.Build) void { // Compile-time configuration the kernel reads as `@import("build_options")`. The // QEMU test harness sets -Dtest-case= to run one self-test at boot. const test_case = b.option([]const u8, "test-case", "Kernel self-test case to run at boot (see system/kernel/tests.zig)"); - const build_options = b.addOptions(); - build_options.addOption(?[]const u8, "test_case", test_case); - const build_options_module = build_options.createModule(); + // The serial-console log sink. Off by default: a real machine often has no + // working legacy COM1, and the boot log is kept in RAM (klog) and flushed to + // disk instead — serial is now only a QEMU convenience. `run-x86-64` and the + // QEMU test harness (test/qemu_test.py, which asserts on serial markers) turn + // it on; a flashable `zig build` image leaves it out. See serial.zig. + const serial = b.option(bool, "serial", "Compile the serial-console log sink into the kernel (default: off; run-x86-64 and the test harness enable it)") orelse false; // --- Kernel: freestanding x86_64 ELF, jumped to by the bootloader --- // SSE2 is part of the x86_64 baseline and UEFI leaves it enabled at handoff, @@ -299,41 +401,17 @@ pub fn build(b: *std.Build) void { .abi = .none, }); - const exe = b.addExecutable(.{ - .name = "kernel", - .root_module = b.createModule(.{ - .root_source_file = b.path("system/kernel/kernel.zig"), - .target = kernel_target, - .optimize = optimize, - .code_model = .kernel, // kernel runs in the top 2 GiB (higher half) - .red_zone = false, // interrupts would corrupt the SystemV red zone - .single_threaded = false, // SMP: the big kernel lock's atomics must be real across cores - .sanitize_c = .off, // the UBSan runtime needs f128/SSE support we don't provide - .stack_check = false, // stack-probe calls have no runtime to land in - .stack_protector = false, - .imports = &.{ - .{ .name = "boot-handoff", .module = boot_handoff_module }, - .{ .name = "abi", .module = abi_module }, - .{ .name = "device-abi", .module = device_abi_module }, - .{ .name = "architecture", .module = architecture_module }, - .{ .name = "platform", .module = platform_module }, - .{ .name = "parameters", .module = parameters_module }, - .{ .name = "build_options", .module = build_options_module }, - .{ .name = "initial-ramdisk", .module = initial_ramdisk_module }, - }, - }), - }); - exe.setLinkerScript(b.path("system/kernel/architecture/x86_64/linker.ld")); - exe.entry = .{ .symbol_name = "_start" }; - // The self-hosted linker ignores parts of the linker script (PHDRS, - // /DISCARD/, AT(), section order); the higher-half layout depends on the - // script being authoritative, so pin the kernel to LLVM + LLD. - exe.use_llvm = true; - exe.use_lld = true; - // Higher-half virtual base (matches KERNEL_VIRT_BASE in linker.ld); the - // linker's AT() clauses give each segment a low physical load address - // (.text at 1 MiB), which the loader allocates and copies into. - exe.image_base = 0xFFFFFFFF80100000; + const kernel_modules = KernelModules{ + .boot_handoff = boot_handoff_module, + .abi = abi_module, + .device_abi = device_abi_module, + .architecture = architecture_module, + .platform = platform_module, + .parameters = parameters_module, + .initial_ramdisk = initial_ramdisk_module, + }; + // The installed/flashable kernel: serial follows -Dserial (off by default). + const exe = addKernel(b, kernel_target, optimize, kernel_modules, test_case, serial); // Everything installs into a FHS-shaped zig-out: it IS the danos filesystem *and* // the boot volume. Each binary lands at its addressed, leaf-collapsed path — the @@ -498,6 +576,13 @@ pub fn build(b: *std.Build) void { // Boot methods live in boot/, one per way of getting the kernel running. // Each is its own binary/entry (a loader is built for its own target); today // that's UEFI for x86-64, with room for e.g. a device-tree path for the Pis. + // The loader reads -Dserial too, so its boot-progress breadcrumbs (con_out, + // which firmware may mirror to a serial console) are silenced by default — a + // real-hardware boot stays quiet. Fatal-error messages ignore this and always + // show, so a failed boot still explains itself on screen. See boot/efi.zig. + const loader_options = b.addOptions(); + loader_options.addOption(bool, "serial", serial); + const loader_options_module = loader_options.createModule(); const efiexe = b.addExecutable(.{ .name = "BOOTX64", .root_module = b.createModule(.{ @@ -510,6 +595,7 @@ pub fn build(b: *std.Build) void { .imports = &.{ // The bootloader speaks only the handoff contract — never the user ABI. .{ .name = "boot-handoff", .module = boot_handoff_module }, + .{ .name = "build_options", .module = loader_options_module }, }, }), }); @@ -525,21 +611,17 @@ pub fn build(b: *std.Build) void { // stub, the kernel, init, and the initial-ramdisk. QEMU presents this image as // a USB mass-storage device the guest boots from (see run-x86-64 and the test // harness), and the danos fat driver mounts the same image at /mnt/usb. - const mk_fat = b.addSystemCommand(&.{"python3"}); - mk_fat.addFileArg(b.path("tools/make-fat-image.py")); - const fat_image = mk_fat.addOutputFileArg("danos-usb.img"); - mk_fat.addArg("64"); // MiB - mk_fat.addArg("EFI/BOOT/BOOTX64.efi"); - mk_fat.addFileArg(efiexe.getEmittedBin()); - mk_fat.addArg("system/kernel"); - mk_fat.addFileArg(exe.getEmittedBin()); - mk_fat.addArg("system/services/init"); - mk_fat.addFileArg(init_exe.getEmittedBin()); - mk_fat.addArg("boot/initial-ramdisk.img"); - mk_fat.addFileArg(initial_ramdisk_img); + const fat_image = addBootImage(b, exe.getEmittedBin(), efiexe.getEmittedBin(), init_exe.getEmittedBin(), initial_ramdisk_img); const fat_image_install = b.addInstallFile(fat_image, "danos-usb.img"); b.getInstallStep().dependOn(&fat_image_install.step); + // The image `run-x86-64` boots: identical to the flashable one but with the + // serial log sink compiled in, so a developer always gets the machine-readable + // log captured to serial0 — without baking serial into the image users flash. + // Built lazily (only when `run-x86-64` is requested), and never installed. + const exe_serial = addKernel(b, kernel_target, optimize, kernel_modules, test_case, true); + const fat_image_serial = addBootImage(b, exe_serial.getEmittedBin(), efiexe.getEmittedBin(), init_exe.getEmittedBin(), initial_ramdisk_img); + // `zig build check-fat-image` — validate the produced image is a real FAT32 // with the EFI stub present (the builder's own --verify, no external tools). const check_fat = b.addSystemCommand(&.{"python3"}); @@ -611,8 +693,9 @@ pub fn build(b: *std.Build) void { run_efi.addPrefixedFileArg("if=pflash,format=raw,file=", vars_out); // Boot off the FAT32 USB image: a mass-storage device on the same xHCI bus as // the keyboard and mouse. OVMF finds \EFI\BOOT\BOOTX64.efi on it and boots. + // The serial-enabled variant, so serial0 carries the log for this dev boot. run_efi.addArg("-drive"); - run_efi.addPrefixedFileArg("if=none,id=bootusb,format=raw,file=", fat_image); + run_efi.addPrefixedFileArg("if=none,id=bootusb,format=raw,file=", fat_image_serial); run_efi.addArgs(&.{ "-device", "usb-storage,bus=xhci.0,drive=bootusb,removable=on,bootindex=0", @@ -634,8 +717,10 @@ pub fn build(b: *std.Build) void { const make_log_dir = b.addSystemCommand(&.{ "mkdir", "-p", log_dir }); const serial_log = b.fmt("{s}/run-x86-64-serial0-{s}.log", .{ log_dir, timestamp(b) }); run_efi.addArgs(&.{ "-serial", b.fmt("file:{s}", .{serial_log}) }); - // The whole FHS zig-out must be installed (and the scratch dir created) before we mount it. - run_efi.step.dependOn(b.getInstallStep()); + // We boot the self-contained `fat_image_serial` (added as a file arg above, so + // it's already a dependency) — not the installed FHS zig-out — so `run-x86-64` + // builds only the serial kernel, never the flashable one. Just make the serial + // scratch dir first. run_efi.step.dependOn(&make_log_dir.step); const run_efi_step = b.step("run-x86-64", "Boot the x86-64 kernel in QEMU (UEFI/OVMF); serial0 is logged to zig-out/qemu-test/run-x86-64-serial0-.log"); diff --git a/docs/testing.md b/docs/testing.md index 6ed97f6..d3187a8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -27,6 +27,15 @@ transcript. Serial is per-architecture (x86 uses port I/O; an ARM board uses a memory-mapped UART), so it lives behind the [arch](arch.md) boundary — and adding a new architecture's UART is what makes the same tests run there. +The serial log sink is **compiled in only under `-Dserial`** (off by default). +A real machine often has no live legacy COM1 — writing to a dead one is slow — +and the boot log is kept in a RAM buffer (`klog`) and flushed to disk instead, +so serial is now purely a QEMU/dev aid. The harness (`test/qemu_test.py`) builds +every case with `-Dserial=true`, and `zig build run-x86-64` boots a serial-enabled +image variant, so both get the transcript; a flashable `zig build` image leaves +serial out. (Even with `-Dserial`, a loopback probe disables a dead port at boot, +so a serial-enabled image is still safe on real hardware.) + ## In-kernel test cases Building with `-Dtest-case=` makes the kernel, after normal bring-up, run one diff --git a/system/kernel/kernel.zig b/system/kernel/kernel.zig index 8d8408c..f51423b 100644 --- a/system/kernel/kernel.zig +++ b/system/kernel/kernel.zig @@ -60,8 +60,16 @@ fn kmain(boot_information: *const BootInformation) noreturn { // file on a ramdisk/USB/SSD), so a message survives as long as any is present. // A headless, serial-less machine still boots correctly — it just goes quiet, // with port-0x80 checkpoints as the only progress signal. - architecture.serialInit(); - log.addSink(architecture.serialWrite); + // + // Serial is compiled in only under -Dserial (build.zig): a real machine often + // has no live legacy COM1, and the log survives in the RAM buffer (below) and + // is flushed to disk — so serial is now a QEMU/dev convenience the flashable + // image leaves out. When it *is* built in, `serialInit`'s loopback probe still + // guards against a dead port (so a -Dserial image is safe on real hardware). + if (build_options.serial) { + architecture.serialInit(); + log.addSink(architecture.serialWrite); + } if (architecture.debugconPresent()) log.addSink(architecture.debugconWrite); // Retain the whole stream in a RAM buffer too, so a user program can later // read it back (klog_read) and persist the boot log to disk — the only way to @@ -87,7 +95,7 @@ fn kmain(boot_information: *const BootInformation) noreturn { "/system/kernel: framebuffer console online (bootstrap; graphics driver later)\n" else "/system/kernel: no framebuffer (headless) -> logging to serial/debugcon only\n"); - log.write(if (architecture.serialPresent()) + if (build_options.serial) log.write(if (architecture.serialPresent()) "/system/kernel: serial console online (COM1)\n" else "/system/kernel: no serial UART (COM1 absent) -> log kept in RAM/debugcon\n"); diff --git a/test/qemu_test.py b/test/qemu_test.py index 181e20e..f1b4f4d 100644 --- a/test/qemu_test.py +++ b/test/qemu_test.py @@ -504,7 +504,10 @@ TIMEOUT = 30 # seconds per case def build(arch, case): - cmd = ["zig", "build", f"-Dtest-case={case}"] + arch["zig_flags"] + # -Dserial: the harness asserts on markers the kernel writes to serial0, so the + # serial log sink must be compiled in. It is off by default (a flashed real- + # hardware image keeps its log in RAM instead; see build.zig / serial.zig). + cmd = ["zig", "build", f"-Dtest-case={case}", "-Dserial=true"] + arch["zig_flags"] r = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True) if r.returncode != 0: return r.stderr.strip() or r.stdout.strip()