init: restart a crashed boot service

init supervises its boot services (spawned against an exit endpoint) but the
child-exit handler was `if (got.isNotification()) continue;` — it silently
dropped a dead service. Now init restarts it: on a child-exit notification it
finds the service, and unless it exited cleanly (chose to stop) or has hit the
crash-loop cap (maximum_restarts), respawns it and logs the death + reason. The
reincarnation half of resilience (docs/resilience.md) at the service level, the
counterpart to the device manager's driver restarts. A `shutting_down` flag
skips restarts during the orderly stop sequence, whose child deaths are expected.
This commit is contained in:
Daniel Samson 2026-07-14 09:05:10 +01:00
parent e1605e3235
commit a01a4f3b3d
No known key found for this signature in database
GPG Key ID: A9EB2589C60F7268
1 changed files with 56 additions and 12 deletions

View File

@ -32,10 +32,20 @@ const log_path = "/mnt/usb/DANOS.LOG";
/// manifest under /system/services instead of a hardcoded list.)
const boot_services = [_][]const u8{ "vfs", "input", "device-manager", "fat", "display", "display-demo" };
var children: [boot_services.len]u32 = .{0} ** boot_services.len;
var child_count: usize = 0;
/// The live process id of each boot service (0 = not running), indexed by its position
/// in `boot_services`, plus how many times init has restarted it. init supervises these:
/// it spawns them against `supervision_endpoint` and, on a child's death, restarts it (up
/// to `maximum_restarts`) the reincarnation half of resilience (docs/resilience.md), the
/// service-level counterpart to the device manager's driver restarts.
var child_ids: [boot_services.len]u32 = .{0} ** boot_services.len;
var restart_counts: [boot_services.len]u32 = .{0} ** boot_services.len;
var shutting_down = false;
var supervision_endpoint: runtime.ipc.Handle = 0;
/// Give up restarting a service after this many crashes a crash-loop cap, so a service
/// that faults immediately on every spawn doesn't respawn forever.
const maximum_restarts = 3;
pub fn main() void {
// Prove the heap end to end: allocate through the runtime allocator (which
// mmaps pages from the kernel and carves them with the free list), write into
@ -63,11 +73,8 @@ pub fn main() void {
// Bring up the boot services, supervised so init can stop them cleanly.
// Best-effort and silent: each service announces its own readiness, and in
// an isolation test with no initial-ramdisk the spawns simply no-op.
for (boot_services) |service| {
if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| {
children[child_count] = id;
child_count += 1;
}
for (boot_services, 0..) |service, i| {
if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| child_ids[i] = id;
}
// Once the storage stack is up, a one-shot copies the boot log to the USB
@ -105,11 +112,47 @@ pub fn main() void {
if (receive[1] == @intFromEnum(power.Event.power_button)) shutDown();
continue;
}
// Child-exit notifications and anything else: keep waiting.
if (got.isChildExit()) {
restartChild(got.childProcessId());
continue;
}
// Anything else: keep waiting.
if (got.isNotification()) continue;
}
}
/// A supervised boot service died. Find which one and restart it unless it exited
/// cleanly (it chose to stop, e.g. a driver with no hardware) or has hit the crash-loop
/// cap. Reclaiming the dead process is already the kernel's job (docs/process-lifecycle.md
/// iron rule 1); init only decides whether to bring it back.
fn restartChild(id: u32) void {
if (shutting_down) return; // deaths during the stop sequence are expected, not crashes
for (boot_services, 0..) |service, i| {
if (child_ids[i] != id) continue;
child_ids[i] = 0;
// An unknown reason (the record aged out) is treated as a crash worth restarting.
const reason = runtime.process.exitReason(id) orelse .fault;
if (reason == .exited) {
logLine("/system/services/init: {s} exited cleanly; not restarting\n", .{service});
return;
}
restart_counts[i] += 1;
if (restart_counts[i] > maximum_restarts) {
logLine("/system/services/init: {s} keeps crashing; giving up after {d} restarts\n", .{ service, maximum_restarts });
return;
}
logLine("/system/services/init: {s} died ({s}); restarting ({d}/{d})\n", .{ service, @tagName(reason), restart_counts[i], maximum_restarts });
if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |new_id| child_ids[i] = new_id;
return;
}
// An untracked child (e.g. the log-flush one-shot): nothing to restart.
}
fn logLine(comptime fmt: []const u8, args: anytype) void {
var line: [128]u8 = undefined;
_ = runtime.system.write(std.fmt.bufPrint(&line, fmt, args) catch return);
}
/// Look up the power service and subscribe our endpoint (handed over as the
/// call's capability) so events arrive as buffered messages here.
fn subscribePower() void {
@ -153,15 +196,16 @@ fn flushKernelLog() void {
/// it), waiting up to a deadline for each to exit before killing it, then ask the
/// power service to enter S5.
fn shutDown() void {
shutting_down = true; // the stop loop below kills children those deaths aren't crashes
_ = runtime.system.write("/system/services/init: shutting down\n");
// Persist the fullest log to the USB volume BEFORE tearing anything down: the
// reverse-order stop loop below kills the fat server (children[3]) first, so
// /mnt/usb must be written while it is still mounted.
// reverse-order stop loop below kills the fat server first, so /mnt/usb must be
// written while it is still mounted.
flushKernelLog();
var i = child_count;
var i = boot_services.len;
while (i > 0) {
i -= 1;
if (children[i] != 0) runtime.process.stop(children[i], 2000, supervision_endpoint);
if (child_ids[i] != 0) runtime.process.stop(child_ids[i], 2000, supervision_endpoint);
}
if (runtime.ipc.lookup(.power)) |h| {
const request = power.Shutdown{};