init: restart a crashed boot service

init supervises its boot services (spawned against an exit endpoint) but the
child-exit handler was `if (got.isNotification()) continue;` — it silently
dropped a dead service. Now init restarts it: on a child-exit notification it
finds the service, and unless it exited cleanly (chose to stop) or has hit the
crash-loop cap (maximum_restarts), respawns it and logs the death + reason. The
reincarnation half of resilience (docs/resilience.md) at the service level, the
counterpart to the device manager's driver restarts. A `shutting_down` flag
skips restarts during the orderly stop sequence, whose child deaths are expected.

system/services/init/init.zig

@@ -32,10 +32,20 @@ const log_path = "/mnt/usb/DANOS.LOG";
 /// manifest under /system/services instead of a hardcoded list.)
 const boot_services = [_][]const u8{ "vfs", "input", "device-manager", "fat", "display", "display-demo" };
 
-var children: [boot_services.len]u32 = .{0} ** boot_services.len;
+/// The live process id of each boot service (0 = not running), indexed by its position
-var child_count: usize = 0;
+/// in `boot_services`, plus how many times init has restarted it. init supervises these:
+/// it spawns them against `supervision_endpoint` and, on a child's death, restarts it (up
+/// to `maximum_restarts`) — the reincarnation half of resilience (docs/resilience.md), the
+/// service-level counterpart to the device manager's driver restarts.
+var child_ids: [boot_services.len]u32 = .{0} ** boot_services.len;
+var restart_counts: [boot_services.len]u32 = .{0} ** boot_services.len;
+var shutting_down = false;
 var supervision_endpoint: runtime.ipc.Handle = 0;
 
+/// Give up restarting a service after this many crashes — a crash-loop cap, so a service
+/// that faults immediately on every spawn doesn't respawn forever.
+const maximum_restarts = 3;
+
 pub fn main() void {
     // Prove the heap end to end: allocate through the runtime allocator (which
     // mmaps pages from the kernel and carves them with the free list), write into
@@ -63,11 +73,8 @@ pub fn main() void {
     // Bring up the boot services, supervised so init can stop them cleanly.
     // Best-effort and silent: each service announces its own readiness, and in
     // an isolation test with no initial-ramdisk the spawns simply no-op.
-    for (boot_services) |service| {
+    for (boot_services, 0..) |service, i| {
-        if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| {
+        if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| child_ids[i] = id;
-            children[child_count] = id;
-            child_count += 1;
-        }
     }
 
     // Once the storage stack is up, a one-shot copies the boot log to the USB
@@ -105,11 +112,47 @@ pub fn main() void {
             if (receive[1] == @intFromEnum(power.Event.power_button)) shutDown();
             continue;
         }
-        // Child-exit notifications and anything else: keep waiting.
+        if (got.isChildExit()) {
+            restartChild(got.childProcessId());
+            continue;
+        }
+        // Anything else: keep waiting.
         if (got.isNotification()) continue;
     }
 }
 
+/// A supervised boot service died. Find which one and restart it — unless it exited
+/// cleanly (it chose to stop, e.g. a driver with no hardware) or has hit the crash-loop
+/// cap. Reclaiming the dead process is already the kernel's job (docs/process-lifecycle.md
+/// iron rule 1); init only decides whether to bring it back.
+fn restartChild(id: u32) void {
+    if (shutting_down) return; // deaths during the stop sequence are expected, not crashes
+    for (boot_services, 0..) |service, i| {
+        if (child_ids[i] != id) continue;
+        child_ids[i] = 0;
+        // An unknown reason (the record aged out) is treated as a crash worth restarting.
+        const reason = runtime.process.exitReason(id) orelse .fault;
+        if (reason == .exited) {
+            logLine("/system/services/init: {s} exited cleanly; not restarting\n", .{service});
+            return;
+        }
+        restart_counts[i] += 1;
+        if (restart_counts[i] > maximum_restarts) {
+            logLine("/system/services/init: {s} keeps crashing; giving up after {d} restarts\n", .{ service, maximum_restarts });
+            return;
+        }
+        logLine("/system/services/init: {s} died ({s}); restarting ({d}/{d})\n", .{ service, @tagName(reason), restart_counts[i], maximum_restarts });
+        if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |new_id| child_ids[i] = new_id;
+        return;
+    }
+    // An untracked child (e.g. the log-flush one-shot): nothing to restart.
+}
+
+fn logLine(comptime fmt: []const u8, args: anytype) void {
+    var line: [128]u8 = undefined;
+    _ = runtime.system.write(std.fmt.bufPrint(&line, fmt, args) catch return);
+}
+
 /// Look up the power service and subscribe our endpoint (handed over as the
 /// call's capability) so events arrive as buffered messages here.
 fn subscribePower() void {
@@ -153,15 +196,16 @@ fn flushKernelLog() void {
 /// it), waiting up to a deadline for each to exit before killing it, then ask the
 /// power service to enter S5.
 fn shutDown() void {
+    shutting_down = true; // the stop loop below kills children — those deaths aren't crashes
     _ = runtime.system.write("/system/services/init: shutting down\n");
     // Persist the fullest log to the USB volume BEFORE tearing anything down: the
-    // reverse-order stop loop below kills the fat server (children[3]) first, so
+    // reverse-order stop loop below kills the fat server first, so /mnt/usb must be
-    // /mnt/usb must be written while it is still mounted.
+    // written while it is still mounted.
     flushKernelLog();
-    var i = child_count;
+    var i = boot_services.len;
     while (i > 0) {
         i -= 1;
-        if (children[i] != 0) runtime.process.stop(children[i], 2000, supervision_endpoint);
+        if (child_ids[i] != 0) runtime.process.stop(child_ids[i], 2000, supervision_endpoint);
     }
     if (runtime.ipc.lookup(.power)) |h| {
         const request = power.Shutdown{};