danos/library/runtime/heap.zig

//! The user-space heap: C-convention dynamic allocation (`malloc`/`free`/…) plus
//! a `std.mem.Allocator` adapter over the same free list, so both C-style code
//! and Zig `std` containers share one heap.
//!
//! The algorithm is a straight port of the kernel's first-fit free list
//! (system/kernel/heap.zig): an address-ordered singly linked list of free blocks,
//! split on allocation and coalesced with neighbours on free. The only thing
//! that changes on this side of the system_call boundary is where memory comes from
//! — `grow` asks the kernel for pages via `mmap` instead of mapping frames
//! itself, and the kernel picks the base address.
//!
//! 16-byte maximum alignment, exactly like the kernel heap. The free list is guarded by
//! a `Thread.Mutex` **only in multi-threaded binaries** (`addThreadedUserBinary`): the
//! guard is gated on `builtin.single_threaded`, so an ordinary single-threaded binary
//! compiles it out and pays nothing, while a threaded one can allocate safely from
//! several threads at once (docs/threading-plan.md M7). The lock lives at the two
//! free-list mutators — `rawAlloc`/`rawFree` — which every entry point funnels through.

const std = @import("std");
const builtin = @import("builtin");
const abi = @import("abi");
const system_calls = @import("system.zig");
const Mutex = @import("thread.zig").Thread.Mutex;

const page_size = abi.page_size;

/// Guards `free_list`. A no-op in single-threaded builds (compiled out); a real futex
/// mutex in threaded ones. Uncontended acquisition is a single CAS — no syscall.
var heap_mutex: Mutex = .{};

inline fn lockHeap() void {
    if (comptime !builtin.single_threaded) heap_mutex.lock();
}
inline fn unlockHeap() void {
    if (comptime !builtin.single_threaded) heap_mutex.unlock();
}

/// A block header, at the start of every block; while free it also links the
/// free list via `next`.
const Block = extern struct {
    size: usize, // total block size in bytes, including this header; a multiple of 16
    next: ?*Block, // free-list link (only meaningful while free)
};

const header_size = @sizeOf(Block); // 16
const minimum_block = header_size + 16; // smallest block worth splitting off
/// Grow granularity: one `mmap` per 64 KiB amortises the system_call.
const chunk = 64 * 1024;

var free_list: ?*Block = null;

fn alignUp(value: usize, alignment: usize) usize {
    return (value + alignment - 1) & ~(alignment - 1);
}

fn payloadOf(block: *Block) [*]u8 {
    return @ptrFromInt(@intFromPtr(block) + header_size);
}

/// Ask the kernel for more pages and add them as a free block. Because each
/// `mmap` is an independent grant, cross-grant coalescing happens only when the
/// kernel returns adjacent bases (its arena is a bump allocator, so consecutive
/// grants usually are adjacent). Returns false if the kernel is out of memory.
fn grow(minimum_bytes: usize) bool {
    const bytes = alignUp(@max(minimum_bytes, chunk), page_size);
    const ret = system_calls.mmap(bytes, system_calls.PROT_READ | system_calls.PROT_WRITE);
    if (system_calls.mmapFailed(ret)) return false;

    const block: *Block = @ptrFromInt(ret);
    block.size = bytes;
    insertFree(block); // coalesces if this grant is adjacent to a prior one
    return true;
}

/// Insert a block into the address-ordered free list, coalescing with the
/// physically adjacent free blocks on either side.
fn insertFree(block: *Block) void {
    var previous: ?*Block = null;
    var current = free_list;
    while (current) |c| : (current = c.next) {
        if (@intFromPtr(c) > @intFromPtr(block)) break;
        previous = c;
    }

    block.next = current;
    if (previous) |p| p.next = block else free_list = block;

    // Merge forward into `current` if they're contiguous.
    if (current) |c| {
        if (@intFromPtr(block) + block.size == @intFromPtr(c)) {
            block.size += c.size;
            block.next = c.next;
        }
    }
    // Merge `previous` forward into `block` if they're contiguous.
    if (previous) |p| {
        if (@intFromPtr(p) + p.size == @intFromPtr(block)) {
            p.size += block.size;
            p.next = block.next;
        }
    }
}

/// Allocate `len` bytes (16-byte aligned), or null if out of memory. Holds the heap lock
/// across the free-list search and any `grow` (which also touches the free list).
fn rawAlloc(len: usize) ?[*]u8 {
    lockHeap();
    defer unlockHeap();
    const need = alignUp(header_size + len, 16);

    var attempts: u32 = 0;
    while (attempts < 2) : (attempts += 1) {
        var previous: ?*Block = null;
        var current = free_list;
        while (current) |block| : ({
            previous = block;
            current = block.next;
        }) {
            if (block.size < need) continue;

            if (block.size >= need + minimum_block) {
                // Split: carve `need` off the front, leave the rest free.
                const rest: *Block = @ptrFromInt(@intFromPtr(block) + need);
                rest.size = block.size - need;
                rest.next = block.next;
                if (previous) |p| p.next = rest else free_list = rest;
                block.size = need;
            } else {
                // Take the whole block.
                if (previous) |p| p.next = block.next else free_list = block.next;
            }
            return payloadOf(block);
        }

        // Nothing fit: grow and try once more.
        if (!grow(need)) return null;
    }
    return null;
}

fn rawFree(ptr: [*]u8) void {
    lockHeap();
    defer unlockHeap();
    const block: *Block = @ptrFromInt(@intFromPtr(ptr) - header_size);
    insertFree(block);
}

// --- C ABI: the global implicit heap ---------------------------------------
// `extern "C"` symbols so future C code links the same malloc/free directly.

export fn malloc(size: usize) callconv(.c) ?*anyopaque {
    if (size == 0) return null;
    const p = rawAlloc(size) orelse return null;
    return @ptrCast(p);
}

export fn free(ptr: ?*anyopaque) callconv(.c) void {
    const p = ptr orelse return;
    rawFree(@ptrCast(p));
}

export fn calloc(nmemb: usize, size: usize) callconv(.c) ?*anyopaque {
    const total = std.math.mul(usize, nmemb, size) catch return null; // overflow-safe
    if (total == 0) return null;
    const p = rawAlloc(total) orelse return null;
    @memset(p[0..total], 0);
    return @ptrCast(p);
}

export fn realloc(ptr: ?*anyopaque, size: usize) callconv(.c) ?*anyopaque {
    const p = ptr orelse return malloc(size);
    if (size == 0) {
        rawFree(@ptrCast(p));
        return null;
    }
    const block: *Block = @ptrFromInt(@intFromPtr(p) - header_size);
    const old_payload = block.size - header_size;
    if (size <= old_payload) return p; // shrink/same: keep the block
    const np = rawAlloc(size) orelse return null; // grow: alloc + copy + free
    @memcpy(np[0..old_payload], @as([*]u8, @ptrCast(p))[0..old_payload]);
    rawFree(@ptrCast(p));
    return @ptrCast(np);
}

// --- std.mem.Allocator interface (same free list) --------------------------

pub fn allocator() std.mem.Allocator {
    return .{ .ptr = undefined, .vtable = &vtable };
}

const vtable = std.mem.Allocator.VTable{
    .alloc = allocImpl,
    .resize = resizeImpl,
    .remap = remapImpl,
    .free = freeImpl,
};

fn allocImpl(_: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 {
    if (alignment.toByteUnits() > 16) return null; // blocks are 16-byte aligned
    return rawAlloc(len);
}

fn resizeImpl(_: *anyopaque, memory: []u8, _: std.mem.Alignment, new_len: usize, _: usize) bool {
    // In-place iff the new payload still fits the current block.
    const block: *Block = @ptrFromInt(@intFromPtr(memory.ptr) - header_size);
    return new_len + header_size <= block.size;
}

fn remapImpl(_: *anyopaque, _: []u8, _: std.mem.Alignment, _: usize, _: usize) ?[*]u8 {
    return null;
}

fn freeImpl(_: *anyopaque, memory: []u8, _: std.mem.Alignment, _: usize) void {
    rawFree(memory.ptr);
}