const std = @import("std"); const boot_handoff = @import("boot-handoff"); const abi = @import("abi"); const parameters = @import("parameters"); const architecture = @import("architecture"); const console = @import("console.zig"); const log = @import("log.zig"); const pmm = @import("pmm.zig"); const heap = @import("heap.zig"); const scheduler = @import("scheduler.zig"); const process = @import("process.zig"); const devices_broker = @import("devices-broker.zig"); const irq = @import("irq.zig"); const platform = @import("platform"); const tests = @import("tests.zig"); const build_options = @import("build_options"); const BootInformation = boot_handoff.BootInformation; /// The calling convention used to enter the kernel. Pinned to SystemV explicitly: /// the bootloader is built for the UEFI target, whose C convention is Microsoft /// x64 (first argument in RCX), while the kernel is SystemV (first argument in /// RDI). Both sides reference this so the `boot_information` pointer lands in the /// register the other expects. `boot_handoff.kernel_abi` re-exports it to the loader. pub const kernel_abi = boot_handoff.kernel_abi; // POST/checkpoint codes emitted to I/O port 0x80 at boot milestones — the // last-resort progress signal on a machine with no text output at all. const cp_entry = 0x10; const cp_paging = 0x20; const cp_heap = 0x30; const cp_discovery = 0x40; const cp_scheduler = 0x50; const cp_timer = 0x60; const cp_running = 0x70; const cp_exception = 0xE0; const cp_panic = 0xEE; /// Physical address of the low page reserved at boot for the AP trampoline (0 = none /// was available). Claimed right after the frame allocator comes up, before paging /// and the heap consume the scarce sub-1 MiB frames. var ap_trampoline_page: u64 = 0; /// Kernel entry point. The bootloader jumps here after `ExitBootServices` with a /// pointer to the handoff data. There is no runtime, no stack unwinding, and no /// caller to return to, so this never returns. /// The real entry (`_start`, in isr.s) installs a kernel-owned stack in .bss /// then calls this with the loader's `boot_information` pointer in RDI. We can't keep /// running on the loader's stack: it's a low physical address that the identity /// map covers only transitionally, and vanishes once the kernel drops the low /// half. `boot_information` (also low) is reached through the physmap — its base is the /// same under the loader's bootstrap tables and the kernel's own. export fn kmainEntry(boot_information: *const BootInformation) callconv(kernel_abi) noreturn { kmain(@ptrFromInt(boot_handoff.physicalToVirtual(@intFromPtr(boot_information)))); } fn kmain(boot_information: *const BootInformation) noreturn { // The **log** is the machine-readable diagnostic stream: it fans out to every // *diagnostic* channel that exists (serial, the 0xE9 debug console, and later a // file on a ramdisk/USB/SSD), so a message survives as long as any is present. // A headless, serial-less machine still boots correctly — it just goes quiet, // with port-0x80 checkpoints as the only progress signal. architecture.serialInit(); log.addSink(architecture.serialWrite); if (architecture.debugconPresent()) log.addSink(architecture.debugconWrite); // The **framebuffer** is deliberately *not* a log sink. It's a separate output // surface — a bootstrap text console today, a graphics device driver later — so // we never assume the OS is text-based. Only a few user-facing status lines // (via `status`) and panics are mirrored to it; the verbose log stays out. const fb = boot_information.framebuffer; console.init(fb); log.checkpoint(cp_entry); // Catch CPU exceptions before doing anything that might fault: install our // reporter, then bring up the GDT + IDT. architecture.setFaultHandler(onException); architecture.init(); status("danos: initialising kernel...\n"); log.write(if (console.present()) "danos: framebuffer console online (bootstrap; graphics driver later)\n" else "danos: no framebuffer (headless) -> logging to serial/debugcon only\n"); log.write("danos: cpu tables online (GDT, IDT, TSS)\n"); log.print(" resolution : {d}x{d}\n", .{ fb.width, fb.height }); log.print(" pitch : {d} bytes\n", .{fb.pitch}); log.print(" format : {s}\n", .{@tagName(fb.format)}); log.print(" framebuffer: 0x{x:0>16}\n", .{fb.base}); log.print(" footprint : {d} MiB\n", .{(fb.pitch * fb.height) / (1024 * 1024)}); // Summarise the physical memory the loader handed us. The array is danos's // own MemoryRegion, so this is a plain slice — no firmware layout in sight. const regions = @as([*]const boot_handoff.MemoryRegion, @ptrFromInt(boot_handoff.physicalToVirtual(boot_information.memory_map.regions)))[0..boot_information.memory_map.len]; var usable_pages: u64 = 0; var reserved_pages: u64 = 0; // reserved RAM only — MMIO is device space, not RAM for (regions) |r| { switch (r.kind) { .usable => usable_pages += r.pages, .reserved, .acpi_tables, .acpi_nvs => reserved_pages += r.pages, .mmio => {}, } } const total_pages = usable_pages + reserved_pages; const total_bytes = total_pages * abi.page_size; const gib = 1 << 30; log.write("\ndanos: physical memory\n"); log.print(" total RAM : {d}.{d:0>2} GiB ({d} MiB) - RAM the firmware reported\n", .{ total_bytes / gib, (total_bytes % gib) * 100 / gib, mib(total_pages) }); log.print(" usable : {d} MiB - free RAM (incl. reclaimed boot-services memory)\n", .{mib(usable_pages)}); log.print(" reserved : {d} MiB - kernel image, boot stack, ACPI, runtime services\n", .{mib(reserved_pages)}); log.print(" regions : {d} - entries in the firmware memory map\n", .{regions.len}); // Bring up the physical frame allocator over that map, and prove it works: // allocate three frames, then hand them back. pmm.init(boot_information.memory_map); // Claim the AP trampoline's low (<1 MiB) page *now*, before paging and the heap // draw down sub-1 MiB frames (the allocator scans upward from frame 0). Held // until SMP bring-up; 0 means none was available (we stay uniprocessor). ap_trampoline_page = pmm.allocBelow(0x100000) orelse 0; const s1 = pmm.stats(); log.print("\ndanos: frame allocator online\n", .{}); log.print(" free frames: {d} ({d} MiB)\n", .{ s1.free_frames, mib(s1.free_frames) }); const f0 = pmm.alloc(); const f1 = pmm.alloc(); const f2 = pmm.alloc(); log.print(" alloc x3 : 0x{x} 0x{x} 0x{x}\n", .{ f0 orelse 0, f1 orelse 0, f2 orelse 0 }); if (f0) |p| pmm.free(p); if (f1) |p| pmm.free(p); if (f2) |p| pmm.free(p); log.print(" after free : {d} frames free\n", .{pmm.stats().free_frames}); // Switch off the firmware's page tables onto our own (with real permissions). architecture.enablePaging(pmm.alloc, pmm.free, boot_information); log.checkpoint(cp_paging); log.print("\ndanos: paging enabled\n", .{}); log.print(" page tables: root = 0x{x:0>16}\n", .{architecture.activePageTable()}); log.print(" kernel segs: {d} (mapped with W^X permissions)\n", .{boot_information.kernel_segment_count}); // Bring up the kernel heap (dynamic allocation), built on the VMM. heap.init(); log.checkpoint(cp_heap); log.write("\ndanos: kernel heap online\n"); // Measure the amount of resources the kernel is actually using const s2 = pmm.stats(); log.print(" Kernel footprint: {d} KiB\n", .{kib(s1.free_frames - s2.free_frames)}); // Enumerate hardware from the firmware tables (ACPI here) into a generic // device tree, then list it. Discovery walks ACPI memory directly (identity- // mapped) and maps PCIe configuration space on demand via the VMM. A failure here is // not fatal yet — log it and carry on. const hal = platform.Hal{ .mapMmio = architecture.mapMmio, .pioRead = architecture.pioRead, .pioWrite = architecture.pioWrite, }; if (platform.discover(boot_information, heap.allocator(), hal)) |devtree| { var device_tree = devtree; log.write("\ndanos: device discovery online\n"); device_tree.dump(log.write); // Snapshot the device tree for user-space drivers (device_enumerate/claim/ // mmio_map operate on this flat, id-indexed table + claim map). devices_broker.init(&device_tree); if (devices_broker.dropped > 0) { // Otherwise entirely silent: drivers would just never see that hardware. log.print("danos: WARNING {d} device(s) dropped — table full\n", .{devices_broker.dropped}); } // Install the device-IRQ trampolines, so a driver's irq_bind has vectors to // land on. Every line stays masked until something binds it (ioapic.init). irq.init(); // Power register map extracted from the FADT + AML, for confidence it parsed. const pw = platform.powerInformation(); log.write("danos: power\n"); log.print(" pm1a_cnt : {s} 0x{x} (width {d})\n", .{ if (pw.pm1a_cnt.mmio) "mmio" else "io", pw.pm1a_cnt.address, pw.pm1a_cnt.width }); if (pw.s5) |s| { log.print(" S5 slp_typ : a={d} b={d}\n", .{ s.slp_typ_a, s.slp_typ_b }); } else { log.write(" S5 slp_typ : (not found)\n"); } log.print(" reset : supported={} {s} 0x{x} val 0x{x}\n", .{ pw.reset_supported, if (pw.reset.mmio) "mmio" else "io", pw.reset.address, pw.reset_value }); // AML namespace parse integrity: consumed should equal total. const am = platform.amlStats(); log.print(" aml : {d} namespace nodes, parsed {d}/{d} bytes\n", .{ am.nodes, am.consumed, am.total }); // Feed the architecture layer the discovered addresses/facts so it makes no legacy // assumptions — the point of all this on UEFI Class 3 firmware. MMIO bases // (HPET, I/O APIC) come from the device tree; scalar facts from ACPI. const pinfo = platform.platformInformation(); const hpet_base: u64 = if (device_tree.firstOfClass(.timer)) |t| (if (t.firstResource(.memory)) |r| r.start else 0) else 0; var ioapic_base: u64 = 0; var ioapic_gsi: u32 = 0; if (device_tree.firstOfClass(.interrupt_controller)) |ic| { if (ic.firstResource(.memory)) |r| ioapic_base = r.start; if (ic.firstResource(.irq)) |r| ioapic_gsi = @intCast(r.start); } var isos: [16]architecture.IsoEntry = undefined; const iso_n = @min(pinfo.override_count, isos.len); for (0..iso_n) |i| isos[i] = .{ .source = pinfo.overrides[i].source, .gsi = pinfo.overrides[i].gsi, .flags = pinfo.overrides[i].flags, }; const pm_timer: ?architecture.PmTimer = if (pinfo.pm_timer.present()) .{ .mmio = pinfo.pm_timer.mmio, .address = pinfo.pm_timer.address, .is_32bit = pinfo.pm_timer_32bit } else null; architecture.configurePlatform(.{ .pic_present = pinfo.pic_present, .hpet_base = hpet_base, .pm_timer = pm_timer, .ioapic_base = ioapic_base, .ioapic_gsi_base = ioapic_gsi, .overrides = isos[0..iso_n], }); if (pinfo.spcr_uart) |u| architecture.serialReconfigure(u.mmio, u.address); log.write("danos: platform\n"); log.print(" 8259 PIC : {s}\n", .{if (pinfo.pic_present) "present" else "absent"}); log.print(" lapic base : 0x{x}\n", .{pinfo.lapic_base}); log.print(" hpet base : 0x{x}\n", .{hpet_base}); log.print(" pm timer : {s} 0x{x} ({s})\n", .{ if (pinfo.pm_timer.mmio) "mmio" else "io", pinfo.pm_timer.address, if (pinfo.pm_timer_32bit) "32-bit" else "24-bit" }); if (pinfo.spcr_uart) |u| { log.print(" console UART: {s} 0x{x} (SPCR type {d})\n", .{ if (u.mmio) "mmio" else "io", u.address, pinfo.spcr_kind }); } else { log.write(" console UART: none in SPCR -> legacy COM1\n"); } log.print(" ioapic : base 0x{x}, {d} inputs (masked); route0 raw 0x{x}\n", .{ ioapic_base, architecture.irqRouteCount(), architecture.irqRouteRaw(0) }); const cores = platform.cpus(); log.print(" cpus : {d} usable core(s); 1 running (BSP), {d} AP(s) parked (SMP bring-up pending)\n", .{ cores.len, if (cores.len > 0) cores.len - 1 else 0 }); if (platform.cpusDropped() > 0) log.print(" cpus : WARNING {d} core(s) beyond pool cap dropped\n", .{platform.cpusDropped()}); } else |err| { log.print("\ndanos: device discovery failed: {s}\n", .{@errorName(err)}); } log.checkpoint(cp_discovery); // Install the system_call handler (int 0x80 gate + system_call stub) once, before any // user code runs. process.init(); // Register the current context as the first task before enabling preemption. scheduler.init(4); log.checkpoint(cp_scheduler); log.write("\ndanos: scheduler online\n"); // Start the timer and unmask interrupts — the kernel now has a heartbeat, and // the timer preempts among tasks. architecture.startTimer(); architecture.enableInterrupts(); log.checkpoint(cp_timer); log.print("danos: timer online ({d} Hz tick; timer clock {d} MHz, clock {d} MHz; calibrated via {s})\n", .{ architecture.timer_hz, architecture.timerClockHz() / 1_000_000, architecture.clockHz() / 1_000_000, architecture.timerCalibrationSource() }); // Wake the other cores (application processors). A no-op on a single-core // machine; on SMP each AP climbs to long mode and reports in (docs/smp.md). bringUpSecondaries(); // In a test build (`zig build -Dtest-case=`), run that case and stop. // Normal builds fall through to the idle halt. if (build_options.test_case) |case| { tests.run(case, boot_information); architecture.halt(); } log.checkpoint(cp_running); status("kernel initialised.\n"); // Publish the initial-ramdisk so user space can `system_spawn` its bundled // binaries by name. The kernel no longer launches them itself: init is the // service supervisor and the device manager spawns the drivers it discovers. publishInitialRamdisk(boot_information); // Hand over to user space: load /system/services/init (read off the boot volume by // the loader) and spawn it as a real ring-3 process, PID 1. As the supervisor it // brings up the system services (the VFS server, the device manager); the device // manager then discovers the hardware and spawns each driver. init runs on its own // address space, preemptively — this boot context becomes the BSP's idle loop. if (boot_information.init_len != 0) { status("starting /system/services/init...\n"); const image = @as([*]const u8, @ptrFromInt(boot_handoff.physicalToVirtual(boot_information.init_base)))[0..boot_information.init_len]; process.spawnProcess(image, 4, &.{"/system/services/init"}) catch |err| { statusPrint("/system/services/init failed to load: {s}\n", .{@errorName(err)}); }; } else { status("no /system/services/init on the boot volume.\n"); } // Become the idle task: drop below every real task and halt until an // interrupt. The timer keeps preempting into init and any other work. scheduler.setPriority(0); status("\nkernel idle; user space is running.\n"); architecture.halt(); } /// Publish the initial-ramdisk image to the process layer so user space can /// `system_spawn` its bundled binaries by name. The kernel used to spawn every /// bundled program here; now init (the service supervisor) and the device manager /// (drivers) own that, so this only hands the image over — nothing is launched from /// the kernel. fn publishInitialRamdisk(boot_information: *const boot_handoff.BootInformation) void { if (boot_information.initial_ramdisk_len == 0) return; const image = @as([*]const u8, @ptrFromInt(boot_handoff.physicalToVirtual(boot_information.initial_ramdisk_base)))[0..boot_information.initial_ramdisk_len]; process.setInitialRamdisk(image); } /// Wake the application processors the firmware left parked. Allocates the low /// trampoline page (and makes it executable), then wakes each non-boot core in turn, /// handing it a fresh kernel stack and its per-CPU slot. Cores that don't report in /// are left parked — the running system is unaffected. See docs/smp.md. fn bringUpSecondaries() void { const cores = platform.cpus(); if (cores.len <= 1) return; // A low (<1 MiB) frame was reserved at boot for the real-mode trampoline (a SIPI // vector addresses it). It's kept for the system's life — armed only during a // wake, inert (zeroed, non-executable) otherwise — so cores can be re-woken later. if (ap_trampoline_page == 0) { log.write("danos: smp: no low page for the AP trampoline; staying uniprocessor\n"); return; } architecture.setTrampolinePage(ap_trampoline_page); architecture.setSecondaryEntry(scheduler.secondaryMain); // where a woken core joins the run loop // Test hook: the smp-retry case forces the first wake to fail, so the retry below // must still bring every core online. Inert in a normal build (test_case is null). if (build_options.test_case) |tc| { if (std.mem.eql(u8, tc, "smp-retry")) architecture.testFailNextWakes(1); } log.print("\ndanos: bringing up {d} application processor(s)\n", .{cores.len - 1}); const maximum_wake_attempts = 3; // a core that misses the first INIT-SIPI-SIPI gets retried for (cores[1..], 1..) |core, index| { const stack = heap.allocator().alloc(u8, parameters.kernel_stack_size) catch { log.print(" cpu apic_id {d}: no stack; skipped\n", .{core.apic_id}); continue; }; const stack_top = (@intFromPtr(stack.ptr) + stack.len) & ~@as(usize, 15); // This core's dedicated fault stack — allocated only now that the core is // real, rather than reserved statically for every possible core. const fault_stack = heap.allocator().alloc(u8, architecture.fault_stack_size) catch { log.print(" cpu apic_id {d}: no fault stack; skipped\n", .{core.apic_id}); continue; }; architecture.setFaultStack(index, (@intFromPtr(fault_stack.ptr) + fault_stack.len) & ~@as(usize, 15)); const pc = scheduler.prepareSecondary(index, core.apic_id); var attempt: u32 = 1; while (attempt <= maximum_wake_attempts) : (attempt += 1) { if (architecture.startSecondary(core.apic_id, stack_top, @intFromPtr(pc), index)) { pc.online = true; log.print(" cpu apic_id {d}: online (attempt {d})\n", .{ core.apic_id, attempt }); break; } if (attempt == maximum_wake_attempts) log.print(" cpu apic_id {d}: no response after {d} attempts (parked)\n", .{ core.apic_id, maximum_wake_attempts }); } } log.print("danos: {d}/{d} cores online\n", .{ scheduler.onlineCount(), cores.len }); } /// A user-facing status line: to the diagnostic `log` *and* the on-screen console /// (if a framebuffer is present). The verbose log uses `log.*` directly and never /// touches the framebuffer. fn status(message: []const u8) void { log.write(message); console.write(message); } fn statusPrint(comptime fmt: []const u8, args: anytype) void { var buffer: [256]u8 = undefined; status(std.fmt.bufPrint(&buffer, fmt, args) catch return); } /// Frames (4 KiB pages) to whole MiB. fn mib(pages: u64) u64 { return pages * abi.page_size / (1024 * 1024); } fn kib(frames: u64) u64 { return frames * abi.page_size / (1024); } /// Whether a ring-3 exception is attributable to the process that raised it — and /// therefore recoverable by killing that process. NMI (2), double fault (8), and /// machine check (18) report machine or kernel trouble even when they arrive with a /// user CS (an NMI interrupts whatever happens to be running), so they stay terminal. fn recoverableFault(vector: u64) bool { return switch (vector) { 2, 8, 18 => false, else => true, }; } /// Report a CPU exception. Two outcomes (docs/resilience.md): /// /// **A fault taken in user mode kills the faulting process, not the machine.** The /// kernel is intact — the CPU trapped onto the task's kernel stack — so the process /// is killed, everything it held (address space, IRQ bindings, IPC handles, device /// grants' frames) is reclaimed, and the core reschedules. A crashing driver takes /// itself down, never the OS. /// /// **Everything else halts this core.** A kernel-mode fault means the trusted base /// itself is broken — there is nothing safe to kill — and NMI/#DF/#MC report machine /// trouble regardless of CS (`recoverableFault`). Even then the fault is *contained*: /// on an application processor only that core stops and the rest keep running. The /// report names the core so an AP fault is attributed, and goes to every output sink /// plus a POST code and a persistent breadcrumb. (A ring-3 fault on a *borrowed* /// kernel thread — process.run, the user-pf isolation probe — also lands here: there /// is no scheduled process to kill.) /// Classify a CPU exception vector as the ExitReason a supervisor reads — the /// fault classes of docs/process-lifecycle.md. Faults are exit reasons, never /// signals delivered to the faulting process: recovery is restart, not a handler. fn exitReasonForVector(vector: u64) abi.ExitReason { return switch (vector) { 14 => .segmentation_fault, // page fault 6 => .illegal_instruction, // invalid opcode 0, 16, 19 => .arithmetic_fault, // divide error, x87, SIMD 13 => .protection_fault, // general protection else => .fault, }; } fn onException(state: *const architecture.CpuState) noreturn { if (architecture.fromUser(state) and scheduler.currentIsUserProcess() and recoverableFault(state.vector)) { statusPrint("\ndanos: process {d} ({s}) killed by {s} (vector {d}) on core {d}\n", .{ scheduler.currentId(), scheduler.current().name(), architecture.exceptionName(state.vector), state.vector, scheduler.currentCpuIndex() }); statusPrint(" error code : 0x{x}\n", .{state.error_code}); statusPrint(" IP : 0x{x:0>16}\n", .{architecture.instructionPointer(state)}); if (architecture.faultAddress(state)) |address| statusPrint(" fault addr : 0x{x:0>16}\n", .{address}); process.killCurrentProcess(exitReasonForVector(state.vector)); // reclaims everything, reschedules; never returns } log.checkpoint(cp_exception); const core = scheduler.currentCpuIndex(); // A fault is user-facing enough to paint on screen too (via statusPrint), on // top of the diagnostic log. statusPrint("\nCPU EXCEPTION on core {d}: {s} (vector {d})\n", .{ core, architecture.exceptionName(state.vector), state.vector }); statusPrint(" error code : 0x{x}\n", .{state.error_code}); statusPrint(" IP : 0x{x:0>16}\n", .{architecture.instructionPointer(state)}); statusPrint(" SP : 0x{x:0>16}\n", .{architecture.stackPointer(state)}); if (architecture.faultAddress(state)) |address| statusPrint(" fault addr : 0x{x:0>16}\n", .{address}); var buffer: [128]u8 = undefined; log.recordPanic(std.fmt.bufPrint(&buffer, "CPU exception {s} (vector {d}) on core {d} at IP 0x{x}", .{ architecture.exceptionName(state.vector), state.vector, core, architecture.instructionPointer(state) }) catch "cpu exception"); architecture.halt(); } /// Freestanding has no OS to receive a panic. Emit it to every output sink, drop a /// POST code + a persistent breadcrumb (so a post-mortem can recover it even with /// no live console), then halt. Assumes no console — the sinks self-guard. pub const panic = std.debug.FullPanic(struct { fn panic(message: []const u8, first_trace_address: ?usize) noreturn { _ = first_trace_address; log.checkpoint(cp_panic); log.recordPanic(message); status("\nKERNEL PANIC: "); status(message); status("\n"); architecture.halt(); } }.panic);