danos/system/kernel/scheduler.zig

//! The scheduler: fixed-priority preemptive multitasking.
//!
//! Tasks are kernel threads (ring 0, each with its own stack). The **highest-
//! priority ready task always runs**; within a priority level, tasks round-robin.
//! Selection is O(1) — a bitmap of non-empty priority levels plus a FIFO queue per
//! level — which keeps scheduling deterministic, as a real-time kernel needs (see
//! docs/vision.md).
//!
//! Switching happens both cooperatively (`yield`) and preemptively (the timer
//! calls `tick`). See docs/scheduling.md for the interrupt-flag discipline that
//! makes those two paths coexist.
//!
//! Cross-core safety is the **big kernel lock** (`sync.zig`): every critical
//! section here runs under it, and it is held across a context switch and released
//! by the task that resumes (see sync.zig's hand-off rule). On a single core the
//! lock is never contended, so the behaviour is exactly the old interrupt-flag
//! model; it's what lets a second core enter `schedule()` without corrupting the
//! shared queues.

const std = @import("std");
const abi = @import("abi");
const parameters = @import("parameters");
const architecture = @import("architecture");
const heap = @import("heap.zig");
const sync = @import("sync.zig");

/// Priority level: 0 (lowest) .. 7 (highest). 8 levels total.
pub const Priority = u3;
const number_priorities = 8;

const stack_size = parameters.kernel_stack_size; // each task's kernel stack
const maximum_tasks = parameters.maximum_tasks; // maximum tasks alive at once (static pool)

// `reaping` = the task has exited and is queued on its core's reap list; its slot must not
// be reused (freeSlot skips it) until the reaper has freed its kernel stack and set it
// `free` (docs/threading-plan.md M8/M9).
const State = enum { free, ready, running, blocked, reaping };

pub const Task = struct {
    id: u32 = 0,
    state: State = .free,
    priority: Priority = 0,
    sp: usize = 0, // saved stack pointer, valid while not running
    stack: []u8 = &.{},
    kstack_top: usize = 0, // top of `stack` (== TSS.rsp0 for a user task); 0 = none
    wake_at: u64 = 0, // uptime (ms) to wake a sleeping task; 0 = not sleeping
    affinity: ?u32 = null, // null = runs on any core; else the index of its pinned core
    // --- process management (process.zig) ---
    // Id of the process that spawned this one (0 = the kernel). The supervision
    // link is the kill authority: only the supervisor may process_kill a child.
    supervisor: u32 = 0,
    // Id of this task's process leader — the main task's own id, copied to every
    // thread it (transitively) spawns; 0 for kernel tasks and never followed.
    // Equal `leader` is what makes two tasks one process; the leader's id is the
    // id `system_spawn` returned, so it is the process id the supervisor speaks
    // (docs/shared-fate-plan.md).
    leader: u32 = 0,
    // Endpoint to notify when this process ends (any way: exit, fault, kill), or
    // null. Holds its own reference, dropped when the notification is posted.
    // Opaque here for the same reason as `handles` below.
    exit_endpoint: ?*anyopaque = null,
    // How this process ended — set by the death paths (exit, fault, kill) just
    // before the reap records it for `process_exit_reason`. Meaningless while
    // the task lives.
    exit_reason: abi.ExitReason = .exited,
    // Endpoint this process's signals arrive on (signal_bind), or null — same
    // ownership rules as exit_endpoint (holds a reference; opaque here).
    signal_endpoint: ?*anyopaque = null,
    // Signals posted but not yet delivered: the coalescing pending mask
    // (docs/process-lifecycle.md). Bits are abi.Signal values. Signals pend here
    // until an endpoint is bound; two pending terminates are one terminate.
    pending_signals: u32 = 0,
    // Set by process_kill (or a group fan-out) on a task that is running on
    // another core; the kernel finishes the kill at that task's next system call
    // or timer tick. Atomic because the syscall-entry check reads it without the
    // lock while another core writes it under the lock — monotonic is enough: a
    // missed read is caught at the next delivery point (x86-TSO or not).
    kill_pending: std.atomic.Value(bool) = .init(false),
    // True while this task executes its own system call — the timer tick must not
    // tear a task down in the middle of a kernel operation, only while it runs
    // user code (or sits at a block point, where teardown is safe).
    in_system_call: bool = false,
    // Where this task is parked while blocked, so a kill can unlink it: the
    // WaitQueue it waits on (maintained by waitLocked/wakeLocked), or the endpoint
    // whose sender FIFO it queues in (maintained by the IPC layer; opaque here).
    wait_queue: ?*WaitQueue = null,
    ipc_wait_endpoint: ?*anyopaque = null,
    // Physical root of this task's address space, or 0 for a kernel task (which
    // runs on the shared kernel page tables). A user task carries its own.
    address_space: u64 = 0,
    user_ip: u64 = 0, // user-mode entry point (user task only)
    user_sp: u64 = 0, // user-mode stack pointer (user task only)
    user_arg: u64 = 0, // value delivered in the user's first argument register at first entry
    // (rdi on x86_64, via architecture.jumpToUserArg): 0 for a process (its _start ignores
    // it), the closure pointer for a thread (docs/threading.md)
    // The user address this task is blocked on in futex_wait (0 = not futex-waiting).
    // Cleared to 0 by futexWakeLocked as the "woken, not timed out" signal (docs/threading.md).
    futex_addr: u64 = 0,
    // The task id this task is blocked in `thread_join` on (0 = not joining). Woken by
    // `wakeJoinersLocked` when that task exits (docs/threading-plan.md M9).
    join_target: u32 = 0,
    // This task's user-space TLS thread pointer — 0 until set via `set_thread_pointer`.
    // Architecture-neutral: the arch layer maps it to the FS base on x86_64, `TPIDR_EL0` on
    // aarch64. Restored on every context switch to this task (docs/threading-plan.md M10).
    thread_pointer: u64 = 0,
    // The mmap / MMIO grant-arena cursors moved from Task to the per-address-space object
    // (`AddressSpaceRef`, below) so threads sharing one address space hand out disjoint grants
    // — see addressSpaceMmapNextPtr / addressSpaceDeviceMapNextPtr (docs/threading-plan.md M7).
    // --- synchronous IPC (ipc_sync.zig) ---
    // Per-process handle table: a small-int handle names a kernel capability object.
    // Each entry tags its `kind` (an IPC endpoint or a shared-memory object) so the
    // close/exit and cap-passing paths reclaim the right type. Kept opaque here so the
    // scheduler and IPC modules don't import each other (ipc_sync.zig owns the kinds).
    handles: [ipc_maximum_handles]?HandleObject = .{null} ** ipc_maximum_handles,
    // A server holds the caller it currently owes a reply to (set by ReplyWait's
    // receive, cleared when it replies). A client, while blocked in Call, records
    // its message + reply buffers here and its result lands in `ipc_status`.
    ipc_client: ?*Task = null,
    ipc_send_ptr: u64 = 0, // client: outgoing message (virtual_address in this task's address space)
    ipc_send_len: u64 = 0,
    ipc_reply_ptr: u64 = 0, // client: reply buffer (virtual_address)
    ipc_reply_cap: u64 = 0,
    ipc_status: i64 = 0, // client: reply length / -errno, written by the replier
    // The DMA and shared-memory arena cursors moved to the per-address-space object
    // (`AddressSpaceRef`) like the mmap/MMIO cursors before them — per-TASK cursors made
    // sibling threads hand out overlapping windows of the one shared arena
    // (docs/shared-fate-plan.md M4 tripped exactly that).
    ipc_send_cap: u64 = ~@as(u64, 0), // handle to transfer with this message (abi.no_cap = none)
    ipc_received_cap: u64 = ~@as(u64, 0), // client: handle the reply's transferred cap landed at (abi.no_cap = none)
    next: ?*Task = null, // ready-queue link (also the endpoint sender-FIFO link)
    // The process's name — argv[0] as it was spawned (a boot-volume path for init,
    // an initial-ramdisk name for everything else); empty for kernel tasks. Fixed
    // storage, so the fault path can name the dead without touching the heap.
    // Zero-initialised (not `undefined`): an undefined default is materialised as
    // a 0xAA fill, which would move the whole static task pool out of .bss.
    name_buffer: [maximum_task_name]u8 = .{0} ** maximum_task_name,
    name_length: u8 = 0,

    /// The task's name (argv[0] at spawn), or empty for a kernel task.
    pub fn name(self: *const Task) []const u8 {
        return self.name_buffer[0..self.name_length];
    }
};

/// Capacity of `Task.name_buffer` — matches the longest name `system_spawn`
/// accepts, so a spawned name is never truncated. Shared with the ABI's
/// ProcessDescriptor, so `enumerate` copies names without clipping.
pub const maximum_task_name = abi.maximum_process_name;

/// Size of each task's IPC handle table. Kept here (not in ipc_sync.zig) because
/// it dimensions a field of `Task`; ipc_sync.zig re-exports it.
// Raised from 16 with DMA-region capabilities: a driver now holds its per-device
// channel endpoints plus received DMA-region handles (a storage driver forwards several
// buffer caps), and repeated cap-passing consumes slots until handle_close.
pub const ipc_maximum_handles = 32;

/// One handle-table entry: a capability object plus a `kind` tag saying what `ptr` points
/// at (an ipc endpoint or a shared-memory object), so a task's exit path and the
/// capability-passing path reclaim/share the right type. The `kind` values are defined by
/// ipc_sync.zig (`handle_kind_*`); kept an opaque `u8` here so the scheduler doesn't import
/// the IPC module.
pub const HandleObject = struct { kind: u8, ptr: *anyopaque };

var tasks = [_]Task{.{}} ** maximum_tasks;

/// Address-space reference counts: one live entry per address space, counting the
/// tasks that share it. An address space is 1:1 with a process today; threads
/// (docs/threading.md) will push a count above 1, and `destroyAddressSpace` must run
/// only when the **last** task on an address space exits. All access is under the big
/// kernel lock. There can be no more live address spaces than tasks, so the table is
/// sized to the task pool and never overflows in practice.
// The per-address-space kernel object: a reference count plus the grant-arena cursors.
// One live entry per address space; threads sharing an address space share this entry,
// so their mmap/mmio grants bump one cursor and never overlap (docs/threading-plan.md M7).
// `mmap_next`/`device_map_next` are 0 until process.zig seeds them to the arena base.
const AddressSpaceRef = struct {
    root: u64 = 0,
    count: u32 = 0,
    mmap_next: u64 = 0,
    device_map_next: u64 = 0,
    dma_next: u64 = 0, // bump pointer into this space's DMA arena (0 = unseeded)
    shared_memory_next: u64 = 0, // bump pointer into this space's shared-memory arena (0 = unseeded)
    // Group-death state (docs/shared-fate-plan.md), set once by the first kill
    // trigger and never cleared while the entry lives. `dying` gates
    // retainAddressSpace — no new member may join a dying group (closing the
    // thread_spawn escape) — and the stash is what `group_exit_hook` posts when
    // the last reference drops: the leader's identity, the group reason, and the
    // leader's counted exit-endpoint reference (moved off its Task under the
    // same lock hold that set the latch).
    dying: bool = false,
    group_leader: u32 = 0,
    group_supervisor: u32 = 0,
    group_reason: abi.ExitReason = .exited,
    group_exit_endpoint: ?*anyopaque = null,
    // Shared-memory objects mapped into this space (docs/shared-fate-plan.md M3).
    // Each mapping holds one reference to its object, dropped through
    // `space_mapping_release_hook` when the space is destroyed — so "last
    // reference" means no handles AND no mappings, and frames can never be freed
    // while a live space still maps them. Opaque: the object type is the IPC
    // layer's.
    mappings: [maximum_space_mappings]?*anyopaque = .{null} ** maximum_space_mappings,
};

/// Shared-memory mappings one address space can hold — matches the per-task
/// handle table's order of magnitude; `shared_memory_map` fails when full.
const maximum_space_mappings = 16;
var address_space_refs = [_]AddressSpaceRef{.{}} ** maximum_tasks;
var address_space_destroy_count: u64 = 0;

/// Total bytes of task **kernel** stacks currently allocated from the kernel heap —
/// incremented when a task is created, decremented when the reaper frees a dead task's
/// stack. A test-observable proof that the reaper reclaims every stack (docs/threading-
/// plan.md M8): with no live tasks beyond the baseline, this returns to its baseline.
var live_stack_bytes: usize = 0;

/// Test-observable: bytes of task kernel stacks currently live (see `live_stack_bytes`).
pub fn liveStackBytes() usize {
    return live_stack_bytes;
}

/// Free a dead task's kernel stack and drop it from `live_stack_bytes`. The task must be
/// off that stack already (killed while not running, or reaped after it switched away).
/// Caller holds the kernel lock.
fn reapStackLocked(t: *Task) void {
    if (t.stack.len == 0) return; // boot/idle tasks run on a static stack — nothing to free
    live_stack_bytes -= t.stack.len;
    heap.allocator().free(t.stack);
    t.stack = &.{};
    t.kstack_top = 0;
}

/// Free every `.reaping` task queued on this core's reap list and mark each `.free` (now
/// its slot may be reused). The tasks are all off their stacks (they switched away), and
/// the caller holds the lock, so freeing is safe (docs/threading-plan.md M8/M9).
fn drainReapListLocked(pc: *PerCpu) void {
    var node = pc.reap_list;
    pc.reap_list = null;
    while (node) |t| {
        node = t.next; // save the link before we clear it
        t.next = null;
        reapStackLocked(t);
        t.state = .free; // reusable only now, after the stack is freed
    }
}

/// Take a reference to address space `root` (0 = a kernel task, which owns none).
/// Returns false only if the ref table is full — bounded by `maximum_tasks`, so in
/// practice it never is. Caller holds the kernel lock.
fn retainAddressSpace(root: u64) bool {
    if (root == 0) return true;
    var free: ?*AddressSpaceRef = null;
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) {
            // A dying group admits no new member: a thread_spawn that was already
            // past its syscall-entry kill check when the group was condemned
            // fails here, under the same lock that would have created the task
            // (docs/shared-fate-plan.md).
            if (entry.dying) return false;
            entry.count += 1;
            return true;
        }
        if (entry.count == 0 and free == null) free = entry;
    }
    const slot = free orelse return false;
    slot.* = .{ .root = root, .count = 1 };
    return true;
}

/// Drop a reference to `root`; destroy the address space when the **last** one drops.
/// A `root` with no entry — never retained, e.g. a hand-built test space — is
/// destroyed directly, preserving the pre-refcount behaviour. Caller holds the lock.
fn releaseAddressSpace(root: u64) void {
    if (root == 0) return;
    for (&address_space_refs) |*entry| {
        if (entry.count == 0 or entry.root != root) continue;
        entry.count -= 1;
        if (entry.count == 0) {
            // Copy the group-death stash out, then fully reset the slot before
            // the hook runs — a stale stash must never survive into an unrelated
            // process's reused entry (docs/shared-fate-plan.md).
            const was_dying = entry.dying;
            const leader = entry.group_leader;
            const supervisor = entry.group_supervisor;
            const reason = entry.group_reason;
            const endpoint = entry.group_exit_endpoint;
            const mappings = entry.mappings;
            entry.* = .{};
            architecture.destroyAddressSpace(root);
            address_space_destroy_count += 1;
            // Release the mapping references now that no mapping exists —
            // `device_grant`-tagged leaves kept destroyAddressSpace's sweep off
            // the frames, so this drop is what may actually free them (M3).
            if (space_mapping_release_hook) |release| {
                for (mappings) |slot| if (slot) |object| release(object);
            }
            // The group-death moment: the space is gone, every member is dead.
            // process.zig posts the leader's deferred exit publication here —
            // last, so the supervisor's notification postdates every release.
            if (was_dying) if (group_exit_hook) |hook| hook(leader, supervisor, reason, endpoint);
        }
        return;
    }
    // Never retained (a hand-built test space): destroyed directly, out of the
    // group-death hook's scope, preserving the pre-refcount behaviour.
    architecture.destroyAddressSpace(root);
    address_space_destroy_count += 1;
}

/// Called (lock held) at the group-death moment — the last reference to a dying
/// group's address space dropped and the space was destroyed. Registered by
/// process.zig, which posts the leader's deferred exit notification and re-stamps
/// its exit record (docs/shared-fate-plan.md). Runs under the lock at both
/// release sites (`exitUserLocked`, `destroyTaskLocked`).
pub var group_exit_hook: ?*const fn (leader: u32, supervisor: u32, reason: abi.ExitReason, exit_endpoint: ?*anyopaque) void = null;

/// Latch `root`'s group as dying and stash the group-death payload for the hook.
/// Returns false — changing nothing — if the group is already dying (a concurrent
/// trigger lost the race) or `root` has no live entry. Caller holds the lock.
pub fn markGroupDyingLocked(root: u64, leader: u32, supervisor: u32, reason: abi.ExitReason, exit_endpoint: ?*anyopaque) bool {
    for (&address_space_refs) |*entry| {
        if (entry.count == 0 or entry.root != root) continue;
        if (entry.dying) return false;
        entry.dying = true;
        entry.group_leader = leader;
        entry.group_supervisor = supervisor;
        entry.group_reason = reason;
        entry.group_exit_endpoint = exit_endpoint;
        return true;
    }
    return false;
}

/// Called (lock held) once per recorded shared-memory mapping when an address
/// space is destroyed — drops the mapping's object reference. Registered by
/// process.zig (the object type lives in the IPC layer).
pub var space_mapping_release_hook: ?*const fn (*anyopaque) void = null;

/// Record a shared-memory mapping on `root`'s space; its reference is dropped
/// via `space_mapping_release_hook` at space destruction. Returns false —
/// recording nothing — if the space has no live entry or its mapping table is
/// full. On true, the caller has transferred one object reference to the space.
/// Caller holds the lock.
pub fn recordSpaceMappingLocked(root: u64, object: *anyopaque) bool {
    for (&address_space_refs) |*entry| {
        if (entry.count == 0 or entry.root != root) continue;
        for (&entry.mappings) |*slot| {
            if (slot.* == null) {
                slot.* = object;
                return true;
            }
        }
        return false;
    }
    return false;
}

/// Whether `root`'s group is already dying. Caller holds the lock.
pub fn groupDyingLocked(root: u64) bool {
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) return entry.dying;
    }
    return false;
}

/// The stashed supervisor of `root`'s DYING group, or null if the group is not
/// dying. Answers the process_kill authority question during the condemned
/// window, when the leader's task slot may already be reaped. Lock held.
pub fn groupSupervisorLocked(root: u64) ?u32 {
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) {
            return if (entry.dying) entry.group_supervisor else null;
        }
    }
    return null;
}

/// Undo a `recordSpaceMappingLocked` — the rollback half for a caller whose
/// later step failed. Removes one matching slot; the caller drops the reference
/// it had transferred. Lock held.
pub fn removeSpaceMappingLocked(root: u64, object: *anyopaque) void {
    for (&address_space_refs) |*entry| {
        if (entry.count == 0 or entry.root != root) continue;
        for (&entry.mappings) |*slot| {
            if (slot.* == object) {
                slot.* = null;
                return;
            }
        }
        return;
    }
}

/// The whole static task pool, for process.zig's group fan-out — which must scan
/// members under the lock it already holds. Slots may be `.free`/`.reaping`;
/// callers filter by state and must not hold pointers past the lock.
pub fn allTasksLocked() []Task {
    return tasks[0..];
}

/// Test-observable: how many address spaces are live (entries with a nonzero count).
pub fn liveAddressSpaceCount() u32 {
    var live: u32 = 0;
    for (&address_space_refs) |*entry| {
        if (entry.count != 0) live += 1;
    }
    return live;
}

/// Test-observable: total address-space destructions since boot.
pub fn addressSpaceDestroyCount() u64 {
    return address_space_destroy_count;
}

/// Pointer to the mmap grant-arena cursor for address space `root`, so the mmap syscall
/// can read-and-bump it. Per-address-space (not per-task), so sibling threads get
/// disjoint grants. **Caller holds the kernel lock** (the entry is stable while held).
/// Null only if `root` was never retained — which can't happen for a live user task.
pub fn addressSpaceMmapNextPtr(root: u64) ?*u64 {
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) return &entry.mmap_next;
    }
    return null;
}

/// Pointer to the MMIO grant-arena cursor for address space `root` (see
/// `addressSpaceMmapNextPtr`). Caller holds the kernel lock.
pub fn addressSpaceDeviceMapNextPtr(root: u64) ?*u64 {
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) return &entry.device_map_next;
    }
    return null;
}

/// Pointer to the DMA arena cursor for address space `root` (see
/// `addressSpaceMmapNextPtr`). Caller holds the kernel lock.
pub fn addressSpaceDmaNextPtr(root: u64) ?*u64 {
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) return &entry.dma_next;
    }
    return null;
}

/// Pointer to the shared-memory arena cursor for address space `root` (see
/// `addressSpaceMmapNextPtr`). Caller holds the kernel lock.
pub fn addressSpaceSharedMemoryNextPtr(root: u64) ?*u64 {
    for (&address_space_refs) |*entry| {
        if (entry.count != 0 and entry.root == root) return &entry.shared_memory_next;
    }
    return null;
}
var next_id: u32 = 1;

/// Per-CPU scheduler state: the task each core is running, its own idle task, and a
/// queue of tasks **pinned** to it. One entry per core; the architecture layer stashes a
/// pointer to the *running* core's entry in the GS base, so `thisCpu()` fetches it
/// with a single read and no lock.
///
/// Most work stays in the **global** ready queue (below), which any idle core pulls
/// from — work-conserving. A task given an *affinity* instead goes to that core's
/// `pinned_*` queue and is only ever run there (no surprise migration — the more
/// real-time-predictable model, docs/smp.md). The two queues are merged at selection
/// time. Both are still mutated only under the big kernel lock, so one core enqueuing
/// into another core's pinned queue is safe.
pub const PerCpu = struct {
    current: *Task = undefined, // the task running on this core
    idle: *Task = undefined, // this core's idle task (always ready, lowest priority)
    hw_id: u32 = 0, // the core's hardware id (Local APIC id on x86_64)
    index: u32 = 0, // dense 0-based core index
    online: bool = false, // has this core finished bring-up?
    loaded_address_space: u64 = 0, // the address-space root currently loaded on this core
    loaded_thread_pointer: u64 = 0, // the TLS thread pointer currently loaded on this core (docs/threading-plan.md M10)
    // Tasks pinned to this core (affinity == index), per priority level + bitmap.
    pinned_head: [number_priorities]?*Task = .{null} ** number_priorities,
    pinned_tail: [number_priorities]?*Task = .{null} ** number_priorities,
    pinned_bitmap: u8 = 0,
    // Tasks that ended while running on THIS core: they could not free the kernel stack
    // they were standing on, so each pushed itself onto this list (`.reaping` state, linked
    // via `Task.next`) and switched away. The next task to run on this core — or the timer
    // tick — frees their stacks from its own stack, safely (docs/threading-plan.md M8). A
    // *list* (not one slot) so a second death before the first is drained can't lose it.
    reap_list: ?*Task = null,
};

const maximum_cpus = parameters.maximum_cpus;
var cpus = [_]PerCpu{.{}} ** maximum_cpus;

/// This core's per-CPU state, via the architecture layer's GS-base pointer. Valid only once
/// this core has run its scheduler bring-up (BSP in `init`, AP in `secondaryInit`).
inline fn thisCpu() *PerCpu {
    return @ptrFromInt(architecture.cpuLocal());
}

/// The task running on this core — the per-CPU replacement for the old global
/// `current`. A convenience reader; writes go through `thisCpu().current`.
pub inline fn current() *Task {
    return thisCpu().current;
}

// Per-priority FIFO ready queues, and a bitmap of which levels are non-empty. These
// are shared across all cores and mutated only under the big kernel lock.
var ready_head: [number_priorities]?*Task = .{null} ** number_priorities;
var ready_tail: [number_priorities]?*Task = .{null} ** number_priorities;
var ready_bitmap: u8 = 0;

var preemption_enabled = true;

/// Bring up scheduling on the bootstrap processor: register the currently-running
/// kernel context as task 0, publish this core's per-CPU state (via the GS base),
/// give the core an idle task, and hook the timer for preemption. Runs once, at
/// boot, before interrupts are enabled — so no lock is needed here.
pub fn init(boot_priority: Priority) void {
    const pc = &cpus[0];
    pc.* = .{ .index = 0, .online = true, .loaded_address_space = architecture.kernelPageTable() };
    architecture.setCpuLocal(0, @intFromPtr(pc));
    tasks[0] = .{ .id = 0, .state = .running, .priority = boot_priority };
    pc.current = &tasks[0];
    pc.idle = create(idle, 0, null); // this core's idle task: always ready, lowest priority
    architecture.setTickHook(tick);
}

/// The idle task: run when every other task is blocked or sleeping. `hlt` waits
/// for the next interrupt at near-zero power (see docs/halting.md).
fn idle() void {
    while (true) asm volatile ("hlt");
}

/// Reserve and initialise the per-CPU slot for an application processor at dense
/// `index` (1-based; 0 is the BSP) with hardware id `hw_id`, and return a
/// pointer the architecture bring-up hands to the core (it publishes it in its GS base).
/// Called on the BSP before waking each AP; the AP marks itself `online`.
pub fn prepareSecondary(index: usize, hw_id: u32) *PerCpu {
    const pc = &cpus[index];
    pc.* = .{ .index = @intCast(index), .hw_id = hw_id, .online = false };
    return pc;
}

/// Entry for an application processor once the architecture layer has set up its per-CPU
/// tables, LAPIC, and timer. It turns this bring-up context into the core's idle task
/// (as task 0 is for the BSP), marks the core online, and enters the run loop: with
/// interrupts enabled the timer preempts this idle context into whatever the global
/// ready queue offers, so the core runs real work in parallel with the others. The
/// `.c` calling convention lets the architecture trampoline path jump here. Never returns.
pub fn secondaryMain() callconv(.c) noreturn {
    const flags = sync.enter();
    const pc = thisCpu();
    const t = freeSlot() orelse @panic("sched: task table full (AP idle task)");
    t.* = .{ .id = next_id, .state = .running, .priority = 0 };
    next_id += 1;
    pc.current = t;
    pc.idle = t;
    pc.online = true;
    pc.loaded_address_space = architecture.kernelPageTable(); // the AP adopted the kernel tables at bring-up
    sync.leave(flags);

    architecture.enableInterrupts(); // the timer now preempts this idle context into work
    while (true) asm volatile ("hlt"); // idle when this core has nothing ready
}

/// Number of cores that have finished bring-up (the BSP plus every online AP).
pub fn onlineCount() usize {
    var n: usize = 0;
    for (&cpus) |*pc| {
        if (pc.online) n += 1;
    }
    return n;
}

/// Make `t` ready. A pinned task (affinity set) goes to that core's pinned queue;
/// everything else goes to the shared global queue.
fn enqueue(t: *Task) void {
    if (t.affinity) |cpu| {
        const pc = &cpus[cpu];
        enqueueTo(&pc.pinned_head, &pc.pinned_tail, &pc.pinned_bitmap, t);
    } else {
        enqueueTo(&ready_head, &ready_tail, &ready_bitmap, t);
    }
}

fn enqueueTo(head: *[number_priorities]?*Task, tail: *[number_priorities]?*Task, bitmap: *u8, t: *Task) void {
    t.next = null;
    const p: usize = t.priority;
    if (tail[p]) |tl| tl.next = t else head[p] = t;
    tail[p] = t;
    bitmap.* |= @as(u8, 1) << t.priority;
}

/// The highest non-empty priority level in a bitmap, or -1 if empty.
fn topLevel(bitmap: u8) i32 {
    if (bitmap == 0) return -1;
    return @as(i32, number_priorities - 1) - @as(i32, @clz(bitmap));
}

/// Pick the highest-priority ready task for core `pc`: the better of the global queue
/// and this core's pinned queue. Still O(1) (two `clz` and a compare). A pinned task
/// wins an equal-priority tie, so it can't be starved by global work at its level.
fn dequeueHighest(pc: *PerCpu) ?*Task {
    const g = topLevel(ready_bitmap);
    const p = topLevel(pc.pinned_bitmap);
    if (g < 0 and p < 0) return null;
    if (p >= g) return dequeueFrom(&pc.pinned_head, &pc.pinned_tail, &pc.pinned_bitmap, @intCast(p));
    return dequeueFrom(&ready_head, &ready_tail, &ready_bitmap, @intCast(g));
}

fn dequeueFrom(head: *[number_priorities]?*Task, tail: *[number_priorities]?*Task, bitmap: *u8, level: usize) ?*Task {
    const t = head[level].?;
    head[level] = t.next;
    if (head[level] == null) {
        tail[level] = null;
        bitmap.* &= ~(@as(u8, 1) << @intCast(level));
    }
    t.next = null;
    return t;
}

/// Create a task that runs `entry` at `priority`, runnable on any core. It becomes
/// ready immediately. Takes the kernel lock: it mutates the shared task table and
/// ready queues and allocates from the (non-thread-safe) heap, so on SMP it must be
/// serialised.
pub fn spawn(entry: *const fn () void, priority: Priority) void {
    const flags = sync.enter();
    _ = create(entry, priority, null);
    sync.leave(flags);
}

/// Like `spawn`, but **pins** the task to core `cpu` — it will only ever run there.
/// Returns true if pinned; false if `cpu` isn't a valid, online core, in which case
/// the task is still created but left unpinned (so it runs *somewhere* rather than
/// stranding in a queue no core services). Callers that require the pin (e.g. tests)
/// should check the result.
pub fn spawnOn(entry: *const fn () void, priority: Priority, cpu: u32) bool {
    const flags = sync.enter();
    defer sync.leave(flags);
    const ok = cpu < maximum_cpus and cpus[cpu].online;
    _ = create(entry, priority, if (ok) cpu else null);
    return ok;
}

/// Spawn a **user** task: a task with its own address space (`address_space`) that starts
/// in user mode at `entry` on `user_sp`, recorded under `name` (its argv[0]).
/// `supervisor` is the id of the spawning process (0 = the kernel) — the kill
/// authority — and `exit_endpoint` (an *ipc.Endpoint whose reference the caller
/// has already taken, or null) is notified when this process ends. `leader` is
/// the process leader's id for a thread, or 0 to make the new task its own
/// leader (a process spawn).
/// It gets a fresh kernel stack for syscalls/interrupts, and its first switch-in
/// lands in `user_task_trampoline`.
/// Returns the new process id, or null (creating nothing) if the table is full or
/// out of memory.
/// **Caller must hold the kernel lock** (the loader that builds `address_space` holds it
/// across the whole spawn, so the address space and the task appear atomically).
pub fn spawnUserLocked(address_space: u64, entry: u64, user_sp: u64, user_arg: u64, priority: Priority, task_name: []const u8, supervisor: u32, exit_endpoint: ?*anyopaque, leader: u32) ?u32 {
    const t = freeSlot() orelse return null;
    const stack = heap.allocator().alloc(u8, stack_size) catch return null;
    // Take this task's reference to the address space before we commit the slot, so a
    // failure here leaves nothing to unwind (the caller still owns the raw `address_space`).
    if (!retainAddressSpace(address_space)) {
        heap.allocator().free(stack);
        return null;
    }
    live_stack_bytes += stack.len; // the reaper drops this when the task dies (M8)
    t.* = .{
        .id = next_id,
        .state = .ready,
        .priority = priority,
        .stack = stack,
        .address_space = address_space,
        .user_ip = entry,
        .user_sp = user_sp,
        .user_arg = user_arg,
        .supervisor = supervisor,
        .exit_endpoint = exit_endpoint,
        .leader = if (leader == 0) next_id else leader,
    };
    const name_length = @min(task_name.len, maximum_task_name);
    @memcpy(t.name_buffer[0..name_length], task_name[0..name_length]);
    t.name_length = @intCast(name_length);
    next_id += 1;
    const top = @intFromPtr(stack.ptr) + stack.len;
    t.kstack_top = top;
    // First switch-in lands in startUserTask (no register smuggling — it reads
    // the user entry/stack from the Task itself).
    t.sp = architecture.initTaskStack(top, @intFromPtr(&startUserTask));
    enqueue(t);
    return t.id;
}

/// The first thing a fresh user task runs (in ring 0, via task_trampoline). It
/// drops to ring 3 at the task's recorded entry/stack. Reading them from the
/// Task avoids smuggling values through callee-saved registers across the
/// context switch and lock release.
fn startUserTask() void {
    const t = current();
    // No serial chatter here: this runs on every spawn, unserialized against
    // user-space writes, and its output used to shear concurrent log lines in
    // half — the largest source of corrupted markers in the QEMU scenarios.
    architecture.jumpToUserArg(t.user_ip, t.user_sp, t.user_arg); // noreturn (arg0 = 0 for a process)
}

/// The unlocked task-creation primitive. Caller must hold the kernel lock (or be the
/// single-threaded boot path). `affinity` pins the task to a core (null = any).
/// Returns the new task so a core can keep a handle to its idle task.
fn create(entry: *const fn () void, priority: Priority, affinity: ?u32) *Task {
    const t = freeSlot() orelse @panic("sched: task table full");
    const stack = heap.allocator().alloc(u8, stack_size) catch @panic("sched: no memory for task stack");
    live_stack_bytes += stack.len; // the reaper drops this when the task dies (M8)
    t.* = .{ .id = next_id, .state = .ready, .priority = priority, .stack = stack, .affinity = affinity };
    next_id += 1;
    const top = @intFromPtr(stack.ptr) + stack.len;
    t.kstack_top = top;
    t.sp = architecture.initTaskStack(top, @intFromPtr(entry));
    enqueue(t);
    return t;
}

fn freeSlot() ?*Task {
    for (&tasks) |*t| {
        if (t.state == .free) return t;
    }
    return null;
}

/// Pick the highest-priority ready task and switch this core to it. The big kernel
/// lock must be held by the caller (which also keeps local interrupts disabled);
/// it serialises every core's scheduling, so no other core can touch the shared
/// queues while we requeue `previous` and dequeue `next`. A dequeued task is `.ready`,
/// never running elsewhere, so two cores never run the same task.
fn schedule() void {
    const pc = thisCpu();
    const previous = pc.current;
    if (previous.state == .running) {
        previous.state = .ready;
        enqueue(previous); // back of its level's queue (round-robin)
    }
    const next = dequeueHighest(pc) orelse {
        previous.state = .running; // nothing else ready — keep running
        return;
    };
    next.state = .running;
    pc.current = next;
    if (next != previous) switchTo(pc, &previous.sp, next);
}

/// Make `next` this core's running task: publish its kernel stack (TSS.rsp0, so a
/// user-mode interrupt lands on a good stack) and its address space (only when
/// it differs from what's loaded — every page-table switch is a full TLB flush),
/// then switch registers/stacks. Kernel tasks (address_space == 0, no kstack_top used
/// from user mode) resolve to the shared kernel page tables and skip the kernel-
/// stack write, so this is a no-op beyond the register switch for a pure-kernel
/// workload. The big kernel lock is held and interrupts are off throughout, so no
/// interrupt can observe a half-updated (kernel stack, address space) pair.
/// `save_sp` receives the outgoing task's stack pointer.
fn switchTo(pc: *PerCpu, save_sp: *usize, next: *Task) void {
    if (next.kstack_top != 0) architecture.setKernelStack(pc.index, next.kstack_top);
    const want = if (next.address_space != 0) next.address_space else architecture.kernelPageTable();
    if (want != pc.loaded_address_space) {
        architecture.loadPageTable(want);
        pc.loaded_address_space = want;
    }
    // Restore the next task's user TLS thread pointer — only on change, the same
    // conditional-load discipline as CR3 above (docs/threading-plan.md M10).
    if (next.thread_pointer != pc.loaded_thread_pointer) {
        architecture.setThreadPointer(next.thread_pointer);
        pc.loaded_thread_pointer = next.thread_pointer;
    }
    architecture.switchContext(save_sp, next.sp);
    // Resumed now (switchContext returned into our own switchTo frame). Re-fetch the core
    // via thisCpu(): the `pc` parameter is from *our* earlier switchTo call, so it names
    // the core we last ran on — stale if we migrated. switchContext only swaps stacks on
    // the current core, so thisCpu() is the core the just-dead task died on. If a task
    // died switching to us, free its kernel stack: we're on ours so it's safe, and the big
    // lock is still held so its slot can't have been reused (docs/threading-plan.md M8).
    drainReapListLocked(thisCpu());
}

/// Voluntarily give up the CPU to the next ready task.
pub fn yield() void {
    const flags = sync.enter();
    schedule();
    sync.leave(flags);
}

/// Block the current task for `ms` milliseconds, then let it become runnable
/// again. The idle task (or other work) runs in the meantime.
pub fn sleep(ms: u64) void {
    const flags = sync.enter();
    const t = current();
    t.wake_at = architecture.millis() + ms;
    t.state = .blocked;
    schedule(); // current is blocked, so schedule() won't re-enqueue it
    sync.leave(flags);
}

// --- futex: block/wake on a user address (docs/threading.md) ----------------
//
// A futex waiter is not linked into any queue — it is simply a `.blocked` task
// tagged with the address it waits on (`futex_addr`). Waking scans the task table
// (bounded) for matching waiters. A timed wait also sets `wake_at`, so the timer's
// `wakeExpired` can wake it; `futex_addr` stays non-zero in that case, which is how
// the waiter tells a timeout from a real wake.

pub const FutexResult = enum { woken, timed_out };

/// Block the current task on futex `addr` until woken, or (if `timeout_ms > 0`) the
/// deadline. **Precondition:** the big kernel lock is held and the caller has already
/// checked, under this same lock, that the futex word equals the expected value — so
/// no wake can be missed. Returns with the lock still held.
pub fn futexWaitLocked(addr: u64, timeout_ms: u64) FutexResult {
    const t = current();
    t.futex_addr = addr;
    t.wake_at = if (timeout_ms > 0) architecture.millis() + timeout_ms else 0;
    t.state = .blocked;
    schedule(); // woken by futexWakeLocked (clears futex_addr) or wakeExpired (timeout)
    const woken = t.futex_addr == 0;
    t.futex_addr = 0;
    t.wake_at = 0;
    return if (woken) .woken else .timed_out;
}

/// Wake up to `count` tasks blocked in `futex_wait` on `addr` in address space
/// `address_space`. Precondition: the big kernel lock is held. Returns how many woke.
pub fn futexWakeLocked(address_space: u64, addr: u64, count: u32) u32 {
    var woken: u32 = 0;
    for (&tasks) |*t| {
        if (woken >= count) break;
        if (t.state == .blocked and t.address_space == address_space and t.futex_addr == addr) {
            t.futex_addr = 0; // the "woken, not timed out" signal to futexWaitLocked
            t.wake_at = 0;
            t.state = .ready;
            enqueue(t);
            woken += 1;
        }
    }
    return woken;
}

// --- thread join (docs/threading-plan.md M9) --------------------------------
//
// join needs no per-thread IPC endpoint: `thread_join(tid)` blocks the caller until the
// task with id `tid` has exited, and the exit paths wake any joiner. The caller only ever
// reclaims the joined thread's *user* stack (which the thread vacated the moment it entered
// the kernel to exit), so waking at exit time — not reap time — is safe.

/// True if a task with id `tid` is still live (has not exited). Caller holds the lock.
fn aliveTid(tid: u32) bool {
    for (&tasks) |*t| {
        if (t.id == tid and t.state != .free and t.state != .reaping) return true;
    }
    return false;
}

/// Block the current task until the task with id `tid` exits (or return at once if it
/// already has / never existed). **Precondition:** the big kernel lock is held; returns
/// with it still held. Woken by `wakeJoinersLocked`.
pub fn joinThreadLocked(tid: u32) void {
    while (aliveTid(tid)) {
        const t = current();
        t.join_target = tid;
        t.state = .blocked;
        schedule(); // woken when the joined task exits; lock handed off across the switch
        t.join_target = 0;
    }
}

/// Set the calling task's user TLS thread pointer and load it now. Persisted on the Task so
/// context switches restore it (docs/threading-plan.md M10). Caller holds the kernel lock.
pub fn setThreadPointerLocked(addr: u64) void {
    const pc = thisCpu();
    pc.current.thread_pointer = addr;
    architecture.setThreadPointer(addr);
    pc.loaded_thread_pointer = addr;
}

/// Wake every task blocked in `thread_join` on `tid` — called from the exit paths once the
/// exiting task's state is `.free`. Caller holds the lock.
fn wakeJoinersLocked(tid: u32) void {
    for (&tasks) |*t| {
        if (t.state == .blocked and t.join_target == tid) {
            t.join_target = 0;
            t.state = .ready;
            enqueue(t);
        }
    }
}

// --- event-based blocking -------------------------------------------------
//
// A WaitQueue is a set of tasks blocked waiting for something (a resource, a
// message). Tasks link into it through the same `next` field the ready queues
// use — a task is in exactly one queue at a time. These are the primitive locks,
// semaphores and IPC channels are built on.

pub const WaitQueue = struct {
    head: ?*Task = null,
};

/// Block the current task on `wait_queue` and switch away. Precondition: the big kernel
/// lock is held (so a condition can be checked and the block committed atomically;
/// it also keeps local interrupts disabled). On return — when woken — the lock is
/// still held.
pub fn waitLocked(wait_queue: *WaitQueue) void {
    const t = current();
    t.state = .blocked;
    t.wait_queue = wait_queue; // so a kill can unlink a parked waiter
    t.next = wait_queue.head;
    wait_queue.head = t;
    schedule();
}

/// Move the highest-priority waiter on `wait_queue` (if any) to the ready queue.
/// Precondition: the big kernel lock is held. Does not preempt — the caller decides.
pub fn wakeLocked(wait_queue: *WaitQueue) void {
    // Find the highest-priority waiter (bounded scan) and unlink it.
    var best_previous: ?*Task = null;
    var best: ?*Task = null;
    var previous: ?*Task = null;
    var node = wait_queue.head;
    while (node) |t| : ({
        previous = t;
        node = t.next;
    }) {
        if (best == null or t.priority > best.?.priority) {
            best = t;
            best_previous = previous;
        }
    }
    const t = best orelse return;
    if (best_previous) |p| p.next = t.next else wait_queue.head = t.next;
    t.wait_queue = null;
    t.state = .ready;
    enqueue(t);
}

/// Unlink `t` from the wait queue it is parked on, if any (the kill path — a
/// killed waiter must not be woken later as a dangling pointer). Precondition:
/// the big kernel lock is held.
pub fn removeFromWaitQueueLocked(t: *Task) void {
    const wait_queue = t.wait_queue orelse return;
    t.wait_queue = null;
    var previous: ?*Task = null;
    var node = wait_queue.head;
    while (node) |n| : ({
        previous = n;
        node = n.next;
    }) {
        if (n != t) continue;
        if (previous) |p| p.next = t.next else wait_queue.head = t.next;
        t.next = null;
        return;
    }
}

/// Unlink `t` from the ready queue it sits in (global, or its affinity core's
/// pinned queue) — the kill path for a task that is runnable but not running.
/// Precondition: the big kernel lock is held.
pub fn removeFromReadyQueueLocked(t: *Task) void {
    if (t.affinity) |cpu| {
        const pc = &cpus[cpu];
        removeFrom(&pc.pinned_head, &pc.pinned_tail, &pc.pinned_bitmap, t);
    } else {
        removeFrom(&ready_head, &ready_tail, &ready_bitmap, t);
    }
}

fn removeFrom(head: *[number_priorities]?*Task, tail: *[number_priorities]?*Task, bitmap: *u8, t: *Task) void {
    const level: usize = t.priority;
    var previous: ?*Task = null;
    var node = head[level];
    while (node) |n| : ({
        previous = n;
        node = n.next;
    }) {
        if (n != t) continue;
        if (previous) |p| p.next = t.next else head[level] = t.next;
        if (tail[level] == t) tail[level] = previous;
        if (head[level] == null) bitmap.* &= ~(@as(u8, 1) << @intCast(level));
        t.next = null;
        return;
    }
}

/// Find a live task by process id, or null. Ids are monotonic and never reused,
/// so a stale id misses cleanly rather than naming a recycled slot.
/// Precondition: the big kernel lock is held.
pub fn taskByIdLocked(id: u32) ?*Task {
    for (&tasks) |*t| {
        if (t.state != .free and t.state != .reaping and t.id == id) return t;
    }
    return null;
}

/// Make every server that still holds `t` as the client it owes a reply to forget
/// it — the reply of a dead client is dropped, not delivered into freed state.
/// Precondition: the big kernel lock is held.
pub fn forgetIpcClientLocked(t: *Task) void {
    for (&tasks) |*other| {
        if (other.state != .free and other.state != .reaping and other.ipc_client == t) other.ipc_client = null;
    }
}

/// Block the current task and switch away, without putting it on any wait queue —
/// the caller has already linked it wherever it belongs (e.g. an endpoint's sender
/// FIFO). Precondition: the big kernel lock is held; still held on return (when the
/// task is made ready again). The IPC layer's counterpart to `waitLocked`.
pub fn blockCurrentLocked() void {
    current().state = .blocked;
    schedule();
}

/// Make a specific (currently blocked) task ready to run again. Precondition: the
/// big kernel lock is held. Used by the IPC layer to wake a specific caller/server
/// rather than "some waiter on a queue".
pub fn readyLocked(t: *Task) void {
    t.state = .ready;
    enqueue(t);
}

/// Block on `wait_queue` (a self-contained critical section).
pub fn wait(wait_queue: *WaitQueue) void {
    const flags = sync.enter();
    waitLocked(wait_queue);
    sync.leave(flags);
}

/// Wake the highest-priority waiter on `wait_queue`, preempting if it outranks us.
pub fn wake(wait_queue: *WaitQueue) void {
    const flags = sync.enter();
    const pc = thisCpu();
    wakeLocked(wait_queue);
    // If a task this core would now pick outranks the running one, run it at once.
    // (A waiter pinned to *another* core isn't counted — that core picks it up on its
    // next tick; this core doesn't preempt for work it can't run.)
    if (highestReadyPriority(pc)) |p| {
        if (p > pc.current.priority) schedule();
    }
    sync.leave(flags);
}

/// The highest-priority task core `pc` could run right now — the better of the global
/// queue and this core's pinned queue — or null if it would fall back to idle.
fn highestReadyPriority(pc: *PerCpu) ?Priority {
    const top = @max(topLevel(ready_bitmap), topLevel(pc.pinned_bitmap));
    if (top < 0) return null;
    return @intCast(top);
}

/// Wake any sleeping task whose deadline has passed. Bounded by the task count,
/// so it stays deterministic. Called from the timer tick (interrupts disabled).
fn wakeExpired() void {
    const now = architecture.millis();
    for (&tasks) |*t| {
        if (t.state == .blocked and t.wake_at != 0 and now >= t.wake_at) {
            t.wake_at = 0;
            t.state = .ready;
            enqueue(t);
        }
    }
}

// Process-teardown hooks, registered by process.zig at init — the scheduler sits
// below the process layer, so finishing a kill (IRQ bindings, IPC handles, exit
// notification) is called *up* through these, mirroring how the architecture
// layer calls up into `tick`.
//
// `terminate_current_hook` ends the task running on THIS core (lock held, never
// returns — it switches away like `exitUserLocked`). `reap_task_hook` tears down
// a task that is NOT running on any core (lock held).
pub var terminate_current_hook: ?*const fn () noreturn = null;
pub var reap_task_hook: ?*const fn (*Task) void = null;

/// Finish any pending kills this core can see (the deferred half of process_kill;
/// the immediate half runs in the killer's own call). Precondition: the big kernel
/// lock is held, from `tick`.
///
/// - This core's *current* task, if condemned, is terminated here — but only when
///   it is not inside one of its own system calls (`in_system_call`): the tick may
///   have interrupted kernel code mid-operation, where teardown would leak or
///   corrupt what that operation holds. User-mode execution (and the system_call
///   entry/exit stubs, which hold nothing) are safe termination points. A task
///   that *is* mid-call dies at its next block, tick, or system_call entry instead.
///   The hook never returns; abandoning the interrupt frame is fine — the LAPIC
///   was acknowledged before the tick hook ran (see apic.timerTick), exactly as on
///   the fault-kill path.
/// - Condemned tasks that are ready or blocked are not running anywhere (state
///   changes need the lock we hold), so they are reaped in place.
fn reapKillPendingLocked() void {
    const pc = thisCpu();
    const cur = pc.current;
    // Safety net: if a dying task switched to a *fresh* task (which enters via
    // task_trampoline, not switchTo's tail), its stack is still queued here. The dying
    // task switched away before this tick, so it is off its stack — drain now (M8).
    drainReapListLocked(pc);
    if (cur.kill_pending.load(.monotonic) and cur.address_space != 0 and !cur.in_system_call) {
        if (terminate_current_hook) |hook| hook(); // noreturn
    }
    if (reap_task_hook) |hook| {
        for (&tasks) |*t| {
            if (!t.kill_pending.load(.monotonic)) continue;
            if (t.state == .ready or t.state == .blocked) hook(t);
        }
    }
}

/// Called from the timer interrupt (interrupts already disabled): wake due
/// sleepers, finish pending kills, then preempt. Takes the kernel lock like any
/// other critical section, but releases it *without* touching the interrupt flag
/// — the handler's `iretq` restores the interrupted context's flags, so
/// re-enabling here would open a nested-interrupt window before the return.
/// Called from the tick with the big kernel lock held — process.zig hangs the
/// one-shot timer sweep here (timer_bind), the same call-up pattern as the
/// teardown hooks below.
pub var timer_tick_hook: ?*const fn () void = null;

pub fn tick() void {
    _ = sync.enter();
    wakeExpired();
    if (timer_tick_hook) |hook| hook();
    reapKillPendingLocked();
    if (preemption_enabled) schedule();
    sync.leaveIsr();
}

/// Enable or disable timer-driven preemption (cooperative-only when off).
pub fn setPreemption(enabled: bool) void {
    preemption_enabled = enabled;
}

/// End the current task and switch away for good; never returns. The task goes on
/// its core's reap list; the tick-time reaper frees the stack and recycles the
/// slot. Acquires the kernel lock and hands it off to
/// the task we switch into (which releases it) — this frame never returns to leave.
pub fn exit() noreturn {
    _ = sync.enter();
    const pc = thisCpu();
    // Queue this task for reaping: `.reaping` keeps its slot out of freeSlot until its
    // stack is freed; `next` links it on the core's reap list (M8/M9).
    pc.current.state = .reaping;
    pc.current.next = pc.reap_list;
    pc.reap_list = pc.current;
    const next = dequeueHighest(pc) orelse @panic("sched: no task left to run");
    next.state = .running;
    pc.current = next;
    var discard: usize = 0;
    switchTo(pc, &discard, next);
    unreachable;
}

/// End the current **user** task: free its address space, then exit. Runs on the
/// dying task's kernel stack (in the shared kernel half, so it survives the CR3
/// switch to the kernel tables that must happen before we free the process's own
/// tables — we can't free the page tables we're standing on). The kernel stack
/// itself is reaped later, as in `exit`. Never returns.
pub fn exitUser() noreturn {
    _ = sync.enter();
    exitUserLocked();
}

/// The body of `exitUser` for callers that already hold the big kernel lock (the
/// tick-time terminate path, which enters with the lock held). The lock is handed
/// off through the switch and released by the task that resumes. Never returns.
pub fn exitUserLocked() noreturn {
    const pc = thisCpu();
    const dying = pc.current;
    const as = dying.address_space;
    if (as != 0) {
        const kroot = architecture.kernelPageTable();
        architecture.loadPageTable(kroot); // off the process tables before freeing them
        pc.loaded_address_space = kroot;
        releaseAddressSpace(as); // destroys only when this was the last task on the space
    }
    dying.state = .reaping; // dead but its slot stays reserved until the stack is freed
    wakeJoinersLocked(dying.id); // let any thread_join(dying.id) return (M9)
    dying.address_space = 0;
    dying.kill_pending.store(false, .monotonic);
    dying.in_system_call = false;
    // Queue for reaping: the task we switch to (or the next tick) frees this stack (M8/M9).
    dying.next = pc.reap_list;
    pc.reap_list = dying;
    const next = dequeueHighest(pc) orelse @panic("sched: no task left to run");
    next.state = .running;
    pc.current = next;
    var discard: usize = 0;
    switchTo(pc, &discard, next);
    unreachable;
}

/// Free a task that is NOT running on any core (it is ready or blocked, and the
/// caller — the kill path — has already unlinked it from every queue and released
/// what it held). Destroys its address space: safe here because no core can have
/// it loaded (every switch away from a task loads the next task's tables, and the
/// task isn't running). The kernel stack is leaked, as in `exitUser` (no reaper
/// yet). Precondition: the big kernel lock is held.
pub fn destroyTaskLocked(t: *Task) void {
    if (t.address_space != 0) releaseAddressSpace(t.address_space); // destroys only on the last reference
    reapStackLocked(t); // safe to free now: `t` is not running on any core (M8)
    t.address_space = 0;
    t.kill_pending.store(false, .monotonic);
    t.in_system_call = false;
    t.wake_at = 0;
    t.state = .free;
    wakeJoinersLocked(t.id); // a killed thread's joiners must return too (M9)
}

/// Snapshot the task table into `out` (up to its length), returning the total
/// number of live tasks — the kernel half of `process_enumerate`, mirroring
/// devices_broker.enumerate. Kernel tasks are included (empty name, supervisor 0):
/// an honest `ps` shows the idle tasks too. `out` is always KERNEL memory: the
/// system call bounces it out to the caller through the checked copy layer
/// (system/kernel/user-memory.zig), so a bad user pointer fails the call instead
/// of faulting ring 0.
pub fn enumerate(out: []abi.ProcessDescriptor) u64 {
    var total: u64 = 0;
    var cursor: usize = 0;
    while (true) {
        // Past the buffer, keep walking with an empty chunk: the total is the
        // whole live count, however few descriptors the caller had room for.
        const room = if (total < out.len) out[@intCast(total)..] else out[out.len..];
        const chunk = enumerateFrom(&cursor, room);
        total += chunk.live;
        if (chunk.done) return total;
    }
}

/// What one chunk of the task-table walk found.
pub const TaskChunk = struct {
    /// Live tasks passed in this chunk, whether or not they fit in `out` — this
    /// is what the running total (and hence `process_enumerate`'s result) counts.
    live: usize,
    /// How many of those were written into `out` (`@min(live, out.len)`).
    filled: usize,
    /// The cursor reached the end of the table: this was the last chunk.
    done: bool,
};

/// One chunk of the task table: starting at slot `cursor` (advanced past
/// everything scanned), describe up to `out.len` live tasks into `out` — or, with
/// an empty `out`, just count the rest. `cursor == tasks.len` ends the walk.
///
/// The chunked form exists so `process_enumerate` can stage each chunk in a small
/// kernel buffer and copy it out with `user_memory.copyToUser`, rather than
/// handing a user pointer to the kernel's own stores. A *slot* cursor, rather
/// than a "skip the first N live tasks" count, keeps chunks from duplicating or
/// losing an entry when a task exits between them.
pub fn enumerateFrom(cursor: *usize, out: []abi.ProcessDescriptor) TaskChunk {
    const flags = sync.enter();
    defer sync.leave(flags);
    var live: usize = 0;
    var filled: usize = 0;
    const limit = if (out.len == 0) tasks.len else out.len; // always makes progress
    while (cursor.* < tasks.len and live < limit) {
        const t = &tasks[cursor.*];
        cursor.* += 1;
        if (t.state == .free or t.state == .reaping) continue; // reaping = already exited
        live += 1;
        if (filled == out.len) continue;
        out[filled] = .{
            .id = t.id,
            .supervisor = t.supervisor,
            .leader = t.leader,
            .state = @intFromEnum(@as(abi.ProcessState, switch (t.state) {
                .ready => .ready,
                .running => .running,
                .blocked => .blocked,
                .free, .reaping => unreachable,
            })),
            .priority = t.priority,
            .name_length = t.name_length,
            .name = t.name_buffer,
        };
        filled += 1;
    }
    return .{ .live = live, .filled = filled, .done = cursor.* >= tasks.len };
}

/// Whether the running task is a user process (has its own address space).
pub fn currentIsUserProcess() bool {
    return current().address_space != 0;
}

pub fn currentId() u32 {
    return current().id;
}

/// The dense index of the core this task is currently running on (0 = BSP). Reads
/// per-CPU state, so a task calling it on different cores sees different values —
/// which is how a test can prove work is running in parallel. Returns 0 if the GS
/// base isn't published yet (a fault in very early boot, before `init`), so a fault
/// reporter can call it unconditionally without a second fault.
pub fn currentCpuIndex() u32 {
    if (architecture.cpuLocal() == 0) return 0;
    return thisCpu().index;
}

/// The running task's id, or 0 if this core's scheduler isn't up yet (early boot, no GS
/// base). Safe for a fault reporter to call unconditionally — like `currentCpuIndex`,
/// it never dereferences an unpublished per-CPU pointer and so can't fault a second time.
pub fn currentIdSafe() u32 {
    if (architecture.cpuLocal() == 0) return 0;
    return thisCpu().current.id;
}

/// The running task's name (argv[0]), or "" if this core's scheduler isn't up yet.
/// The companion to `currentIdSafe` for naming the culprit in a fatal fault report.
pub fn currentNameSafe() []const u8 {
    if (architecture.cpuLocal() == 0) return "";
    return thisCpu().current.name();
}

/// Change the running task's priority (takes effect next time it's enqueued).
pub fn setPriority(p: Priority) void {
    current().priority = p;
}