init: restart a crashed boot service

init supervises its boot services (spawned against an exit endpoint) but the
child-exit handler was `if (got.isNotification()) continue;` — it silently
dropped a dead service. Now init restarts it: on a child-exit notification it
finds the service, and unless it exited cleanly (chose to stop) or has hit the
crash-loop cap (maximum_restarts), respawns it and logs the death + reason. The
reincarnation half of resilience (docs/resilience.md) at the service level, the
counterpart to the device manager's driver restarts. A `shutting_down` flag
skips restarts during the orderly stop sequence, whose child deaths are expected.

system/kernel/console.zig

@@ -2,12 +2,15 @@
 //! into the linear framebuffer the bootloader handed us. No firmware, no driver
 //! — just pixels.
 //!
-//! This is a **bootstrap** console — a stop-gap so early boot has something on
-//! screen. The framebuffer is a general graphics surface, *not* inherently a text
-//! terminal; once the driver machinery exists it becomes a proper graphics device
-//! driver and this text-grid crutch goes away. It is therefore kept **separate
-//! from the diagnostic [log](log.zig)** — the log fans out to serial/debugcon/file,
-//! while this only paints the handful of user-facing status lines and panics.
+//! This is a **bootstrap / fatal-fallback** console. The driver machinery now exists — the
+//! user-space **display service** ([../services/display](../services/display/display.zig),
+//! docs/display.md) owns the framebuffer in normal operation — so this no longer paints
+//! routine status. It exists for the two cases the display service can't cover: **early
+//! boot**, before the service has claimed the framebuffer, and **fatal errors** (a kernel
+//! panic or a kernel-mode fault), which force it back on (`setSuppressed`) so a dying
+//! machine's last words reach the screen even over a live display. It is kept **separate
+//! from the diagnostic [log](log.zig)** — the log fans out to serial/debugcon/file and
+//! carries all routine kernel output; this only paints those fatal cases.
 //!
 //! The module owns a single console and a `present` flag; `write` is a no-op when
 //! the firmware handed over no framebuffer (a headless machine), so the kernel

system/kernel/kernel.zig

@@ -9,6 +9,7 @@ const wall_clock = @import("wall-clock.zig");
 const pmm = @import("pmm.zig");
 const heap = @import("heap.zig");
 const scheduler = @import("scheduler.zig");
+const sync = @import("sync.zig");
 const process = @import("process.zig");
 const devices_broker = @import("devices-broker.zig");
 const irq = @import("irq.zig");
@@ -156,12 +157,13 @@ fn kmain(boot_information: *const BootInformation) noreturn {
     log.print("  page tables: root = 0x{x:0>16}\n", .{architecture.activePageTable()});
     log.print("  kernel segs: {d} (mapped with W^X permissions)\n", .{boot_information.kernel_segment_count});
 
-    // Now on our own tables, the framebuffer window is write-combining: bring up
-    // the on-screen console and clear it (a fast burst here, not the loader's
-    // uncached crawl). From here `status` reaches the screen as well as the log.
+    // Now on our own tables, the framebuffer window is write-combining: bring up the
+    // on-screen console and clear it to a blank canvas (a fast burst here, not the loader's
+    // uncached crawl). Routine boot output goes only to the log; this console now exists for
+    // early-boot and fatal (`fatal`/panic) output, until the display service takes over.
     console.init(fb);
     log.write(if (console.present())
-        "/system/kernel: framebuffer console online (bootstrap; graphics driver later)\n"
+        "/system/kernel: framebuffer ready (early-boot + fatal fallback; the display service drives it in normal operation)\n"
     else
         "/system/kernel: no framebuffer (headless) -> logging to serial/debugcon only\n");
 
@@ -414,11 +416,22 @@ fn bringUpSecondaries() void {
     log.print("/system/kernel: {d}/{d} cores online\n", .{ scheduler.onlineCount(), cores.len });
 }
 
-/// A user-facing status line: to the diagnostic `log` *and* the on-screen console
-/// (if a framebuffer is present). The verbose log uses `log.*` directly and never
-/// touches the framebuffer.
+/// A user-facing status line. Now that the user-space **display service** owns the
+/// framebuffer in normal operation (docs/display.md), routine kernel output goes to the
+/// diagnostic `log` (serial/debugcon/RAM) *only* — never to the on-screen console, which
+/// the compositor is about to paint over. For a message that must reach the screen even so
+/// — a panic or a fatal fault, when the machine is going down — use `fatal`.
 fn status(message: []const u8) void {
     log.write(message);
+}
+
+/// A fatal, user-facing message: to the diagnostic log *and* the on-screen console, forcing
+/// the console back on (`setSuppressed(false)`) first — a dying machine's last words outrank
+/// any display service holding the framebuffer. The console is otherwise silent in normal
+/// operation (see `status`); it exists now only for early-boot and fatal output.
+fn fatal(message: []const u8) void {
+    log.write(message);
+    console.setSuppressed(false);
     console.write(message);
 }
 
@@ -427,6 +440,11 @@ fn statusPrint(comptime fmt: []const u8, args: anytype) void {
     status(std.fmt.bufPrint(&buffer, fmt, args) catch return);
 }
 
+fn fatalPrint(comptime fmt: []const u8, args: anytype) void {
+    var buffer: [256]u8 = undefined;
+    fatal(std.fmt.bufPrint(&buffer, fmt, args) catch return);
+}
+
 /// Frames (4 KiB pages) to whole MiB.
 fn mib(pages: u64) u64 {
     return pages * abi.page_size / (1024 * 1024);
@@ -486,20 +504,26 @@ fn onException(state: *const architecture.CpuState) noreturn {
     }
 
     log.checkpoint(cp_exception);
-    // The machine is going down: force the console back on even if a display service
-    // was holding the framebuffer, so the exception actually reaches the screen.
-    console.setSuppressed(false);
     const core = scheduler.currentCpuIndex();
-    // A fault is user-facing enough to paint on screen too (via statusPrint), on
-    // top of the diagnostic log.
-    statusPrint("\nCPU EXCEPTION on core {d}: {s} (vector {d})\n", .{ core, architecture.exceptionName(state.vector), state.vector });
-    statusPrint("  error code : 0x{x}\n", .{state.error_code});
-    statusPrint("  IP         : 0x{x:0>16}\n", .{architecture.instructionPointer(state)});
-    statusPrint("  SP         : 0x{x:0>16}\n", .{architecture.stackPointer(state)});
-    if (architecture.faultAddress(state)) |address| statusPrint("  fault addr : 0x{x:0>16}\n", .{address});
+    // The machine is going down: paint the exception on screen too — `fatalPrint` forces the
+    // console back on even if a display service was holding the framebuffer — on top of the
+    // diagnostic log.
+    fatalPrint("\nCPU EXCEPTION on core {d}: {s} (vector {d})\n", .{ core, architecture.exceptionName(state.vector), state.vector });
+    // Name the culprit: which task, and whether it faulted in ring 3 (a process the
+    // kernel would normally kill — landing here means it had no address space) or ring 0
+    // (the trusted base itself). Without this the fatal report is anonymous.
+    fatalPrint("  task       : {d} ({s}), {s}\n", .{ scheduler.currentIdSafe(), scheduler.currentNameSafe(), if (architecture.fromUser(state)) "ring 3 (user)" else "ring 0 (kernel)" });
+    fatalPrint("  error code : 0x{x}\n", .{state.error_code});
+    fatalPrint("  IP         : 0x{x:0>16}\n", .{architecture.instructionPointer(state)});
+    fatalPrint("  SP         : 0x{x:0>16}\n", .{architecture.stackPointer(state)});
+    if (architecture.faultAddress(state)) |address| fatalPrint("  fault addr : 0x{x:0>16}\n", .{address});
 
     var buffer: [128]u8 = undefined;
     log.recordPanic(std.fmt.bufPrint(&buffer, "CPU exception {s} (vector {d}) on core {d} at IP 0x{x}", .{ architecture.exceptionName(state.vector), state.vector, core, architecture.instructionPointer(state) }) catch "cpu exception");
+    // Free the BKL if this core held it (a kernel-mode fault, or a nested fault in the
+    // recovery teardown), so halting this one core doesn't deadlock every other core on
+    // the lock. Only that core stops; the rest — and the supervisor — keep running.
+    sync.releaseIfHeldHere();
     architecture.halt();
 }
 
@@ -511,10 +535,11 @@ pub const panic = std.debug.FullPanic(struct {
         _ = first_trace_address;
         log.checkpoint(cp_panic);
         log.recordPanic(message);
-        console.setSuppressed(false); // a panic outranks any display service holding the screen
-        status("\nKERNEL PANIC: ");
-        status(message);
-        status("\n");
+        fatal("\nKERNEL PANIC: "); // a panic outranks any display service holding the screen
+        fatal(message);
+        fatal("\n");
+        fatalPrint("  task       : {d} ({s})\n", .{ scheduler.currentIdSafe(), scheduler.currentNameSafe() });
+        sync.releaseIfHeldHere(); // don't deadlock the other cores on the lock we may hold
         architecture.halt();
     }
 }.panic);

system/kernel/scheduler.zig

@@ -791,6 +791,21 @@ pub fn currentCpuIndex() u32 {
     return thisCpu().index;
 }
 
+/// The running task's id, or 0 if this core's scheduler isn't up yet (early boot, no GS
+/// base). Safe for a fault reporter to call unconditionally — like `currentCpuIndex`,
+/// it never dereferences an unpublished per-CPU pointer and so can't fault a second time.
+pub fn currentIdSafe() u32 {
+    if (architecture.cpuLocal() == 0) return 0;
+    return thisCpu().current.id;
+}
+
+/// The running task's name (argv[0]), or "" if this core's scheduler isn't up yet.
+/// The companion to `currentIdSafe` for naming the culprit in a fatal fault report.
+pub fn currentNameSafe() []const u8 {
+    if (architecture.cpuLocal() == 0) return "";
+    return thisCpu().current.name();
+}
+
 /// Change the running task's priority (takes effect next time it's enqueued).
 pub fn setPriority(p: Priority) void {
     current().priority = p;

system/kernel/sync.zig

@@ -33,6 +33,13 @@ const architecture = @import("architecture");
 /// 0 = free, 1 = held. A single global lock for the whole kernel.
 var held = std.atomic.Value(u32).init(0);
 
+/// The per-CPU base pointer (`architecture.cpuLocal()`) of the core currently holding
+/// the lock, or 0 when free. Metadata only — `held` is what enforces exclusion — read
+/// solely by `releaseIfHeldHere` on the fatal-fault path. `cpuLocal()` is a unique,
+/// architecture-level token per core (0 before this core's GS base is published, which
+/// is fine: that window is single-core early boot, where no other core can deadlock).
+var owner = std.atomic.Value(usize).init(0);
+
 /// Enter the kernel: disable interrupts on this core, then spin until we own the
 /// lock. Returns the caller's prior interrupt flags for `leave` to restore.
 /// Interrupts stay off for the whole critical section so this core's timer tick
@@ -67,14 +74,29 @@ export fn releaseForFreshTask() callconv(.c) void {
     release();
 }
 
+/// Release the big kernel lock **only if this core is the one holding it** — a no-op
+/// otherwise. For the fatal-fault path (a kernel-mode fault, or a nested fault inside the
+/// recovery teardown, both of which run under the lock): a core that dies holding the BKL
+/// must free it, or every other core spins forever in `acquire` and the whole machine
+/// deadlocks instead of just that core stopping. It must NOT free a lock another core
+/// owns, hence the owner check. Caveat: if we held it mid-mutation the shared state may be
+/// inconsistent — but letting the other cores (and the supervisor) run on possibly-degraded
+/// state is strictly more recoverable than a guaranteed total hang.
+pub fn releaseIfHeldHere() void {
+    const me = architecture.cpuLocal();
+    if (me != 0 and owner.load(.monotonic) == me) release();
+}
+
 fn acquire() void {
     // Test-and-test-and-set: try once, then spin read-only until the lock looks
     // free before retrying the (bus-locked) swap — cheaper on the coherency fabric.
     while (held.swap(1, .acquire) != 0) {
         while (held.load(.monotonic) != 0) architecture.cpuRelax();
     }
+    owner.store(architecture.cpuLocal(), .monotonic);
 }
 
 fn release() void {
+    owner.store(0, .monotonic);
     held.store(0, .release);
 }

system/services/init/init.zig

@@ -30,12 +30,22 @@ const log_path = "/mnt/usb/DANOS.LOG";
 /// microkernel keeps such choices in user space, not the kernel. Drivers are absent
 /// on purpose: the device manager owns those. (A future init reads this from a
 /// manifest under /system/services instead of a hardcoded list.)
-const boot_services = [_][]const u8{ "vfs", "input", "device-manager", "fat", "display" };
+const boot_services = [_][]const u8{ "vfs", "input", "device-manager", "fat", "display", "display-demo" };
 
-var children: [boot_services.len]u32 = .{0} ** boot_services.len;
-var child_count: usize = 0;
+/// The live process id of each boot service (0 = not running), indexed by its position
+/// in `boot_services`, plus how many times init has restarted it. init supervises these:
+/// it spawns them against `supervision_endpoint` and, on a child's death, restarts it (up
+/// to `maximum_restarts`) — the reincarnation half of resilience (docs/resilience.md), the
+/// service-level counterpart to the device manager's driver restarts.
+var child_ids: [boot_services.len]u32 = .{0} ** boot_services.len;
+var restart_counts: [boot_services.len]u32 = .{0} ** boot_services.len;
+var shutting_down = false;
 var supervision_endpoint: runtime.ipc.Handle = 0;
 
+/// Give up restarting a service after this many crashes — a crash-loop cap, so a service
+/// that faults immediately on every spawn doesn't respawn forever.
+const maximum_restarts = 3;
+
 pub fn main() void {
     // Prove the heap end to end: allocate through the runtime allocator (which
     // mmaps pages from the kernel and carves them with the free list), write into
@@ -63,11 +73,8 @@ pub fn main() void {
     // Bring up the boot services, supervised so init can stop them cleanly.
     // Best-effort and silent: each service announces its own readiness, and in
     // an isolation test with no initial-ramdisk the spawns simply no-op.
-    for (boot_services) |service| {
-        if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| {
-            children[child_count] = id;
-            child_count += 1;
-        }
+    for (boot_services, 0..) |service, i| {
+        if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| child_ids[i] = id;
     }
 
     // Once the storage stack is up, a one-shot copies the boot log to the USB
@@ -105,11 +112,47 @@ pub fn main() void {
             if (receive[1] == @intFromEnum(power.Event.power_button)) shutDown();
             continue;
         }
-        // Child-exit notifications and anything else: keep waiting.
+        if (got.isChildExit()) {
+            restartChild(got.childProcessId());
+            continue;
+        }
+        // Anything else: keep waiting.
         if (got.isNotification()) continue;
     }
 }
 
+/// A supervised boot service died. Find which one and restart it — unless it exited
+/// cleanly (it chose to stop, e.g. a driver with no hardware) or has hit the crash-loop
+/// cap. Reclaiming the dead process is already the kernel's job (docs/process-lifecycle.md
+/// iron rule 1); init only decides whether to bring it back.
+fn restartChild(id: u32) void {
+    if (shutting_down) return; // deaths during the stop sequence are expected, not crashes
+    for (boot_services, 0..) |service, i| {
+        if (child_ids[i] != id) continue;
+        child_ids[i] = 0;
+        // An unknown reason (the record aged out) is treated as a crash worth restarting.
+        const reason = runtime.process.exitReason(id) orelse .fault;
+        if (reason == .exited) {
+            logLine("/system/services/init: {s} exited cleanly; not restarting\n", .{service});
+            return;
+        }
+        restart_counts[i] += 1;
+        if (restart_counts[i] > maximum_restarts) {
+            logLine("/system/services/init: {s} keeps crashing; giving up after {d} restarts\n", .{ service, maximum_restarts });
+            return;
+        }
+        logLine("/system/services/init: {s} died ({s}); restarting ({d}/{d})\n", .{ service, @tagName(reason), restart_counts[i], maximum_restarts });
+        if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |new_id| child_ids[i] = new_id;
+        return;
+    }
+    // An untracked child (e.g. the log-flush one-shot): nothing to restart.
+}
+
+fn logLine(comptime fmt: []const u8, args: anytype) void {
+    var line: [128]u8 = undefined;
+    _ = runtime.system.write(std.fmt.bufPrint(&line, fmt, args) catch return);
+}
+
 /// Look up the power service and subscribe our endpoint (handed over as the
 /// call's capability) so events arrive as buffered messages here.
 fn subscribePower() void {
@@ -153,15 +196,16 @@ fn flushKernelLog() void {
 /// it), waiting up to a deadline for each to exit before killing it, then ask the
 /// power service to enter S5.
 fn shutDown() void {
+    shutting_down = true; // the stop loop below kills children — those deaths aren't crashes
     _ = runtime.system.write("/system/services/init: shutting down\n");
     // Persist the fullest log to the USB volume BEFORE tearing anything down: the
-    // reverse-order stop loop below kills the fat server (children[3]) first, so
-    // /mnt/usb must be written while it is still mounted.
+    // reverse-order stop loop below kills the fat server first, so /mnt/usb must be
+    // written while it is still mounted.
     flushKernelLog();
-    var i = child_count;
+    var i = boot_services.len;
     while (i > 0) {
         i -= 1;
-        if (children[i] != 0) runtime.process.stop(children[i], 2000, supervision_endpoint);
+        if (child_ids[i] != 0) runtime.process.stop(child_ids[i], 2000, supervision_endpoint);
     }
     if (runtime.ipc.lookup(.power)) |h| {
         const request = power.Shutdown{};