init: restart a crashed boot service
init supervises its boot services (spawned against an exit endpoint) but the child-exit handler was `if (got.isNotification()) continue;` — it silently dropped a dead service. Now init restarts it: on a child-exit notification it finds the service, and unless it exited cleanly (chose to stop) or has hit the crash-loop cap (maximum_restarts), respawns it and logs the death + reason. The reincarnation half of resilience (docs/resilience.md) at the service level, the counterpart to the device manager's driver restarts. A `shutting_down` flag skips restarts during the orderly stop sequence, whose child deaths are expected.
This commit is contained in:
parent
e1605e3235
commit
a01a4f3b3d
|
|
@ -32,10 +32,20 @@ const log_path = "/mnt/usb/DANOS.LOG";
|
|||
/// manifest under /system/services instead of a hardcoded list.)
|
||||
const boot_services = [_][]const u8{ "vfs", "input", "device-manager", "fat", "display", "display-demo" };
|
||||
|
||||
var children: [boot_services.len]u32 = .{0} ** boot_services.len;
|
||||
var child_count: usize = 0;
|
||||
/// The live process id of each boot service (0 = not running), indexed by its position
|
||||
/// in `boot_services`, plus how many times init has restarted it. init supervises these:
|
||||
/// it spawns them against `supervision_endpoint` and, on a child's death, restarts it (up
|
||||
/// to `maximum_restarts`) — the reincarnation half of resilience (docs/resilience.md), the
|
||||
/// service-level counterpart to the device manager's driver restarts.
|
||||
var child_ids: [boot_services.len]u32 = .{0} ** boot_services.len;
|
||||
var restart_counts: [boot_services.len]u32 = .{0} ** boot_services.len;
|
||||
var shutting_down = false;
|
||||
var supervision_endpoint: runtime.ipc.Handle = 0;
|
||||
|
||||
/// Give up restarting a service after this many crashes — a crash-loop cap, so a service
|
||||
/// that faults immediately on every spawn doesn't respawn forever.
|
||||
const maximum_restarts = 3;
|
||||
|
||||
pub fn main() void {
|
||||
// Prove the heap end to end: allocate through the runtime allocator (which
|
||||
// mmaps pages from the kernel and carves them with the free list), write into
|
||||
|
|
@ -63,11 +73,8 @@ pub fn main() void {
|
|||
// Bring up the boot services, supervised so init can stop them cleanly.
|
||||
// Best-effort and silent: each service announces its own readiness, and in
|
||||
// an isolation test with no initial-ramdisk the spawns simply no-op.
|
||||
for (boot_services) |service| {
|
||||
if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| {
|
||||
children[child_count] = id;
|
||||
child_count += 1;
|
||||
}
|
||||
for (boot_services, 0..) |service, i| {
|
||||
if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |id| child_ids[i] = id;
|
||||
}
|
||||
|
||||
// Once the storage stack is up, a one-shot copies the boot log to the USB
|
||||
|
|
@ -105,11 +112,47 @@ pub fn main() void {
|
|||
if (receive[1] == @intFromEnum(power.Event.power_button)) shutDown();
|
||||
continue;
|
||||
}
|
||||
// Child-exit notifications and anything else: keep waiting.
|
||||
if (got.isChildExit()) {
|
||||
restartChild(got.childProcessId());
|
||||
continue;
|
||||
}
|
||||
// Anything else: keep waiting.
|
||||
if (got.isNotification()) continue;
|
||||
}
|
||||
}
|
||||
|
||||
/// A supervised boot service died. Find which one and restart it — unless it exited
|
||||
/// cleanly (it chose to stop, e.g. a driver with no hardware) or has hit the crash-loop
|
||||
/// cap. Reclaiming the dead process is already the kernel's job (docs/process-lifecycle.md
|
||||
/// iron rule 1); init only decides whether to bring it back.
|
||||
fn restartChild(id: u32) void {
|
||||
if (shutting_down) return; // deaths during the stop sequence are expected, not crashes
|
||||
for (boot_services, 0..) |service, i| {
|
||||
if (child_ids[i] != id) continue;
|
||||
child_ids[i] = 0;
|
||||
// An unknown reason (the record aged out) is treated as a crash worth restarting.
|
||||
const reason = runtime.process.exitReason(id) orelse .fault;
|
||||
if (reason == .exited) {
|
||||
logLine("/system/services/init: {s} exited cleanly; not restarting\n", .{service});
|
||||
return;
|
||||
}
|
||||
restart_counts[i] += 1;
|
||||
if (restart_counts[i] > maximum_restarts) {
|
||||
logLine("/system/services/init: {s} keeps crashing; giving up after {d} restarts\n", .{ service, maximum_restarts });
|
||||
return;
|
||||
}
|
||||
logLine("/system/services/init: {s} died ({s}); restarting ({d}/{d})\n", .{ service, @tagName(reason), restart_counts[i], maximum_restarts });
|
||||
if (runtime.system.spawnSupervised(service, &.{}, supervision_endpoint)) |new_id| child_ids[i] = new_id;
|
||||
return;
|
||||
}
|
||||
// An untracked child (e.g. the log-flush one-shot): nothing to restart.
|
||||
}
|
||||
|
||||
fn logLine(comptime fmt: []const u8, args: anytype) void {
|
||||
var line: [128]u8 = undefined;
|
||||
_ = runtime.system.write(std.fmt.bufPrint(&line, fmt, args) catch return);
|
||||
}
|
||||
|
||||
/// Look up the power service and subscribe our endpoint (handed over as the
|
||||
/// call's capability) so events arrive as buffered messages here.
|
||||
fn subscribePower() void {
|
||||
|
|
@ -153,15 +196,16 @@ fn flushKernelLog() void {
|
|||
/// it), waiting up to a deadline for each to exit before killing it, then ask the
|
||||
/// power service to enter S5.
|
||||
fn shutDown() void {
|
||||
shutting_down = true; // the stop loop below kills children — those deaths aren't crashes
|
||||
_ = runtime.system.write("/system/services/init: shutting down\n");
|
||||
// Persist the fullest log to the USB volume BEFORE tearing anything down: the
|
||||
// reverse-order stop loop below kills the fat server (children[3]) first, so
|
||||
// /mnt/usb must be written while it is still mounted.
|
||||
// reverse-order stop loop below kills the fat server first, so /mnt/usb must be
|
||||
// written while it is still mounted.
|
||||
flushKernelLog();
|
||||
var i = child_count;
|
||||
var i = boot_services.len;
|
||||
while (i > 0) {
|
||||
i -= 1;
|
||||
if (children[i] != 0) runtime.process.stop(children[i], 2000, supervision_endpoint);
|
||||
if (child_ids[i] != 0) runtime.process.stop(child_ids[i], 2000, supervision_endpoint);
|
||||
}
|
||||
if (runtime.ipc.lookup(.power)) |h| {
|
||||
const request = power.Shutdown{};
|
||||
|
|
|
|||
Loading…
Reference in New Issue