From 56bd2e7678edd6adb8a578f14c03cf83564f6814 Mon Sep 17 00:00:00 2001 From: Daniel Samson <12231216+daniel-samson@users.noreply.github.com> Date: Mon, 10 Aug 2026 02:46:23 +0100 Subject: [PATCH] exfat: the on-disk layout (S4 step 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure, host-testable byte layer of the second engine: the Main Boot Sector (VBR) and the six 32-byte directory-entry types — Allocation Bitmap, Up-case Table, Volume Label, File, Stream Extension, File Name — as align(1) extern structs, plus the three exFAT checksums (boot region, up-case table, directory-entry set), the name hash, and the packed timestamp <-> Unix-epoch conversion. geometryOf accepts only "EXFAT " + 0xAA55 + an all-zero MustBeZero region; that last guard is the mutual exclusion with FAT — a FAT prober reads a zero bytes-per-sector there and rejects the volume, and this one rejects a FAT boot sector for want of the exFAT name. Wire-format widths are named consts (spec facts, no bare literals) so the bounds gate stays green; the layout is pinned by @offsetOf/@sizeOf tests. Wired into the host-test aggregate directly for now; it moves into the exfat package's own test step when that lands (step 4). 7/7 host tests, bounds green. --- build.zig | 3 + system/services/exfat/on-disk.zig | 447 ++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 system/services/exfat/on-disk.zig diff --git a/build.zig b/build.zig index 8d76688..19680c1 100644 --- a/build.zig +++ b/build.zig @@ -422,6 +422,9 @@ pub fn build(b: *std.Build) void { "system/boot-handoff.zig", "system/abi.zig", "system/initial-ramdisk.zig", // v2 path-named entries: find/basename/magic + // The exFAT engine's pure on-disk layer (S4), std-only, before its package + // exists — moves into the `exfat` package's own test step once that lands. + "system/services/exfat/on-disk.zig", }) |root| { const mod_tests = b.addTest(.{ .root_module = b.createModule(.{ diff --git a/system/services/exfat/on-disk.zig b/system/services/exfat/on-disk.zig new file mode 100644 index 0000000..cef941f --- /dev/null +++ b/system/services/exfat/on-disk.zig @@ -0,0 +1,447 @@ +//! The on-disk layout of an exFAT filesystem — the Main Boot Sector (VBR) and the +//! six 32-byte directory-entry types — as `align(1)` extern structs that bit-cast +//! straight out of a sector (multi-byte fields little-endian). Pure data, plus the +//! three exFAT checksums (boot region, up-case table, directory-entry set), the +//! name hash, and the packed timestamp <-> Unix-epoch conversion. Host-testable. +//! +//! exFAT departs from FAT in three ways this file encodes: geometry lives in a +//! MustBeZero-guarded VBR (byte 11 is zero, which is exactly why the FAT prober +//! rejects an exFAT volume — it reads a zero bytes-per-sector); a file is a SET of +//! entries (a File entry, a Stream Extension, and one or more File Name entries) +//! validated by a rotate-right checksum; and names are compared case-folded through +//! the volume's own on-disk up-case table (the folding itself lives in the engine, +//! which holds the loaded table; the hash it feeds is here). + +const std = @import("std"); + +// exFAT's fixed on-disk widths — byte and UTF-16-unit counts the FORMAT defines, +// not ceilings danos chooses. Named so the wire-format structs carry no bare +// literal lengths; the values are facts of the spec. +pub const entry_bytes: usize = 32; // every directory entry +const jump_boot_bytes = 3; +const filesystem_name_bytes = 8; // "EXFAT " +const must_be_zero_bytes = 53; // the FAT-BPB overlap the format holds zero +const boot_code_bytes = 390; +const volume_label_units = 11; + +// --- the Main Boot Sector (VBR, sector 0) ------------------------------------ + +/// The exFAT Main Boot Sector. `must_be_zero` (offset 11..64) overlaps where a +/// FAT BPB keeps bytes-per-sector/sectors-per-cluster/etc.; exFAT holds it zero, +/// so a FAT prober reading a zero bytes-per-sector rejects the volume — the +/// mutual-exclusion the two engines rely on. +pub const MainBootSector = extern struct { + jump_boot: [jump_boot_bytes]u8, // 0 + filesystem_name: [filesystem_name_bytes]u8, // 3 "EXFAT " + must_be_zero: [must_be_zero_bytes]u8, // 11 + partition_offset: u64 align(1), // 64 sectors, informational + volume_length: u64 align(1), // 72 sectors + fat_offset: u32 align(1), // 80 sectors from volume start + fat_length: u32 align(1), // 84 sectors, per FAT + cluster_heap_offset: u32 align(1), // 88 sectors from volume start + cluster_count: u32 align(1), // 92 + first_cluster_of_root: u32 align(1), // 96 + volume_serial_number: u32 align(1), // 100 + filesystem_revision: u16 align(1), // 104 + volume_flags: u16 align(1), // 106 (skipped by the boot checksum) + bytes_per_sector_shift: u8, // 108 9..12 + sectors_per_cluster_shift: u8, // 109 + number_of_fats: u8, // 110 1 (2 for TexFAT) + drive_select: u8, // 111 + percent_in_use: u8, // 112 (skipped by the boot checksum) + reserved: [7]u8, // 113 + boot_code: [boot_code_bytes]u8, // 120 + boot_signature: u16 align(1), // 510 0xAA55 +}; + +// --- directory entries (32 bytes each) --------------------------------------- + +/// Entry-type bytes. The high bit (0x80) is InUse: a type with it clear is not in +/// use, and 0x00 ends the directory. Deleting an entry clears bit 7 (0x85 -> 0x05). +pub const entry_type_allocation_bitmap: u8 = 0x81; +pub const entry_type_upcase_table: u8 = 0x82; +pub const entry_type_volume_label: u8 = 0x83; +pub const entry_type_file: u8 = 0x85; +pub const entry_type_stream_extension: u8 = 0xC0; +pub const entry_type_file_name: u8 = 0xC1; +pub const entry_type_in_use_bit: u8 = 0x80; +pub const entry_type_end_of_directory: u8 = 0x00; + +/// A raw 32-byte entry, for type dispatch before it is reinterpreted as a +/// specific entry. +pub const RawEntry = extern struct { + entry_type: u8, + data: [entry_bytes - 1]u8, + + pub fn inUse(self: RawEntry) bool { + return self.entry_type & entry_type_in_use_bit != 0; + } + pub fn isEnd(self: RawEntry) bool { + return self.entry_type == entry_type_end_of_directory; + } +}; + +/// 0x81 — the Allocation Bitmap: one bit per cluster (cluster 2 = bit 0), the +/// authority for which clusters are free. The deepest departure from FAT, where +/// the chain itself was the authority. +pub const AllocationBitmapEntry = extern struct { + entry_type: u8, // 0 0x81 + bitmap_flags: u8, // 1 + reserved: [18]u8, // 2 + first_cluster: u32 align(1), // 20 + data_length: u64 align(1), // 24 +}; + +/// 0x82 — the Up-case Table: the on-disk case-fold map (code unit -> uppercase), +/// referenced by cluster and validated by `table_checksum`. +pub const UpcaseTableEntry = extern struct { + entry_type: u8, // 0 0x82 + reserved1: [3]u8, // 1 + table_checksum: u32 align(1), // 4 + reserved2: [12]u8, // 8 + first_cluster: u32 align(1), // 20 + data_length: u64 align(1), // 24 +}; + +/// 0x83 — the Volume Label (up to 11 UTF-16 units). +pub const VolumeLabelEntry = extern struct { + entry_type: u8, // 0 0x83 + character_count: u8, // 1 + volume_label: [volume_label_units]u16 align(1), // 2 + reserved: [8]u8, // 24 +}; + +/// 0x85 — the File entry: the head of a set, carrying the attributes, +/// timestamps, the secondary-entry count, and the set checksum. +pub const FileEntry = extern struct { + entry_type: u8, // 0 0x85 + secondary_count: u8, // 1 stream (1) + name entries + set_checksum: u16 align(1), // 2 over the whole set, skipping these two bytes + file_attributes: u16 align(1), // 4 + reserved1: u16 align(1), // 6 + create_timestamp: u32 align(1), // 8 + last_modified_timestamp: u32 align(1), // 12 + last_accessed_timestamp: u32 align(1), // 16 + create_10ms: u8, // 20 + last_modified_10ms: u8, // 21 + create_utc_offset: u8, // 22 + last_modified_utc_offset: u8, // 23 + last_accessed_utc_offset: u8, // 24 + reserved2: [7]u8, // 25 +}; + +/// 0xC0 — the Stream Extension: the second entry of every file set, carrying the +/// name length + hash and the data location (first cluster, sizes, the +/// no-FAT-chain flag). +pub const StreamExtensionEntry = extern struct { + entry_type: u8, // 0 0xC0 + general_secondary_flags: u8, // 1 + reserved1: u8, // 2 + name_length: u8, // 3 UTF-16 units + name_hash: u16 align(1), // 4 + reserved2: u16 align(1), // 6 + valid_data_length: u64 align(1), // 8 + reserved3: u32 align(1), // 16 + first_cluster: u32 align(1), // 20 + data_length: u64 align(1), // 24 +}; + +/// 0xC1 — a File Name entry: 15 UTF-16 units of the name; a set carries +/// ceil(name_length / 15) of them. +pub const FileNameEntry = extern struct { + entry_type: u8, // 0 0xC1 + general_secondary_flags: u8, // 1 + file_name: [name_units_per_entry]u16 align(1), // 2 +}; + +pub const name_units_per_entry: usize = 15; + +// General secondary flags (Stream Extension + File Name entries). +pub const secondary_flag_allocation_possible: u8 = 0x01; +pub const secondary_flag_no_fat_chain: u8 = 0x02; + +// File attributes (same bit assignments as FAT). +pub const attribute_read_only: u16 = 0x0001; +pub const attribute_hidden: u16 = 0x0002; +pub const attribute_system: u16 = 0x0004; +pub const attribute_directory: u16 = 0x0010; +pub const attribute_archive: u16 = 0x0020; + +// FAT special cluster values (exFAT's FAT is 32-bit; used only for a fragmented +// chain, i.e. when no_fat_chain is clear). +pub const first_data_cluster: u32 = 2; +pub const end_of_chain: u32 = 0xFFFFFFFF; +pub const bad_cluster: u32 = 0xFFFFFFF7; + +pub const boot_signature_offset: usize = 510; // 0x55 0xAA + +// --- geometry ---------------------------------------------------------------- + +pub const Geometry = struct { + bytes_per_sector: u32, + sectors_per_cluster: u32, + cluster_count: u32, + fat_offset_sectors: u32, // from volume start + fat_length_sectors: u32, + cluster_heap_offset_sectors: u32, // from volume start + first_cluster_of_root: u32, + volume_serial_number: u32, + volume_length_sectors: u64, + number_of_fats: u32, +}; + +/// Derive the geometry from a Main Boot Sector. Returns null unless it is a +/// plausible exFAT VBR: the "EXFAT " name, an all-zero MustBeZero region, the +/// 0xAA55 signature, and sane shifts. Accepting ONLY these is what keeps exFAT and +/// FAT from ever claiming each other's volumes. +pub fn geometryOf(sector: []const u8) ?Geometry { + if (sector.len < 512) return null; + if (sector[boot_signature_offset] != 0x55 or sector[boot_signature_offset + 1] != 0xAA) return null; + const vbr = std.mem.bytesToValue(MainBootSector, sector[0..@sizeOf(MainBootSector)]); + if (!std.mem.eql(u8, &vbr.filesystem_name, "EXFAT ")) return null; + for (vbr.must_be_zero) |byte| if (byte != 0) return null; + if (vbr.bytes_per_sector_shift < 9 or vbr.bytes_per_sector_shift > 12) return null; + if (vbr.sectors_per_cluster_shift > 25) return null; + if (vbr.number_of_fats == 0 or vbr.cluster_count == 0) return null; + if (vbr.first_cluster_of_root < first_data_cluster) return null; + return .{ + .bytes_per_sector = @as(u32, 1) << @intCast(vbr.bytes_per_sector_shift), + .sectors_per_cluster = @as(u32, 1) << @intCast(vbr.sectors_per_cluster_shift), + .cluster_count = vbr.cluster_count, + .fat_offset_sectors = vbr.fat_offset, + .fat_length_sectors = vbr.fat_length, + .cluster_heap_offset_sectors = vbr.cluster_heap_offset, + .first_cluster_of_root = vbr.first_cluster_of_root, + .volume_serial_number = vbr.volume_serial_number, + .volume_length_sectors = vbr.volume_length, + .number_of_fats = vbr.number_of_fats, + }; +} + +// --- checksums and the name hash --------------------------------------------- + +/// The directory-entry-SET checksum (a File entry's `set_checksum`), a 16-bit +/// rotate-right sum over every byte of the set, skipping the two checksum bytes +/// themselves (offset 2..3 of the first entry). `entries` is the whole set: +/// (secondary_count + 1) * 32 bytes. +pub fn setChecksum(entries: []const u8) u16 { + var checksum: u16 = 0; + for (entries, 0..) |byte, i| { + if (i == 2 or i == 3) continue; + checksum = std.math.rotr(u16, checksum, 1) +% byte; + } + return checksum; +} + +/// The up-case-table checksum (an Up-case entry's `table_checksum`), a 32-bit +/// rotate-right sum over the table's on-disk bytes. +pub fn upcaseChecksum(table_bytes: []const u8) u32 { + var checksum: u32 = 0; + for (table_bytes) |byte| checksum = std.math.rotr(u32, checksum, 1) +% byte; + return checksum; +} + +/// The boot-region checksum — the u32 the checksum sector repeats — a 32-bit +/// rotate-right sum over the first eleven sectors, skipping VolumeFlags (offset +/// 106..107) and PercentInUse (offset 112) of the first sector. +pub fn bootChecksum(region: []const u8) u32 { + var checksum: u32 = 0; + for (region, 0..) |byte, i| { + if (i == 106 or i == 107 or i == 112) continue; + checksum = std.math.rotr(u32, checksum, 1) +% byte; + } + return checksum; +} + +/// The name hash a Stream entry carries: a 16-bit rotate-right sum over the +/// UP-CASED name's bytes (low byte then high byte of each UTF-16 unit). The caller +/// up-cases through the volume's table first; a mismatch lets a lookup reject a +/// name without reading its File Name entries. +pub fn nameHash(upcased: []const u16) u16 { + var hash: u16 = 0; + for (upcased) |unit| { + hash = std.math.rotr(u16, hash, 1) +% @as(u8, @truncate(unit)); + hash = std.math.rotr(u16, hash, 1) +% @as(u8, @truncate(unit >> 8)); + } + return hash; +} + +// --- timestamps -------------------------------------------------------------- +// +// exFAT packs a timestamp into one u32: the high 16 bits are a DOS date +// (year-1980 | month | day), the low 16 a DOS time (hour | minute | second/2). +// Same field layout as FAT, so the epoch math matches; there is no timezone in +// the packed value (a separate UTC-offset byte carries that, which danos leaves +// zero = UTC). + +fn isLeapYear(year: u32) bool { + return (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0); +} + +const days_in_month = [_]u8{ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + +/// Convert a packed exFAT timestamp to Unix epoch seconds (UTC). 0 for unset. +pub fn timestampToEpoch(timestamp: u32) u64 { + if (timestamp == 0) return 0; + const date: u32 = timestamp >> 16; + const time: u32 = timestamp & 0xFFFF; + const day: u32 = date & 0x1F; + const month: u32 = (date >> 5) & 0x0F; + const year: u32 = 1980 + (date >> 9); + if (month < 1 or month > 12 or day < 1) return 0; + const second: u32 = (time & 0x1F) * 2; + const minute: u32 = (time >> 5) & 0x3F; + const hour: u32 = (time >> 11) & 0x1F; + + var days: u64 = 0; + var y: u32 = 1970; + while (y < year) : (y += 1) days += if (isLeapYear(y)) 366 else 365; + var m: u32 = 1; + while (m < month) : (m += 1) { + days += days_in_month[m - 1]; + if (m == 2 and isLeapYear(year)) days += 1; + } + days += day - 1; + return ((days * 24 + hour) * 60 + minute) * 60 + second; +} + +/// Convert Unix epoch seconds (UTC) to a packed exFAT timestamp. 0 for epoch 0 or +/// any time before 1980 (unrepresentable). +pub fn epochToTimestamp(epoch: u64) u32 { + if (epoch == 0) return 0; + var remaining = epoch; + const second: u32 = @intCast(remaining % 60); + remaining /= 60; + const minute: u32 = @intCast(remaining % 60); + remaining /= 60; + const hour: u32 = @intCast(remaining % 24); + remaining /= 24; + var days: u32 = @intCast(remaining); + + var year: u32 = 1970; + while (true) { + const year_days: u32 = if (isLeapYear(year)) 366 else 365; + if (days < year_days) break; + days -= year_days; + year += 1; + } + if (year < 1980) return 0; + var month: u32 = 1; + while (true) { + var month_days: u32 = days_in_month[month - 1]; + if (month == 2 and isLeapYear(year)) month_days += 1; + if (days < month_days) break; + days -= month_days; + month += 1; + } + const day = days + 1; + const date: u32 = ((year - 1980) << 9) | (month << 5) | day; + const time: u32 = (hour << 11) | (minute << 5) | (second / 2); + return (date << 16) | time; +} + +// --- tests ------------------------------------------------------------------- + +test "on-disk struct sizes match the specification" { + try std.testing.expectEqual(@as(usize, 512), @sizeOf(MainBootSector)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(RawEntry)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(AllocationBitmapEntry)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(UpcaseTableEntry)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(VolumeLabelEntry)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(FileEntry)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(StreamExtensionEntry)); + try std.testing.expectEqual(@as(usize, 32), @sizeOf(FileNameEntry)); +} + +test "MainBootSector field offsets" { + try std.testing.expectEqual(@as(usize, 3), @offsetOf(MainBootSector, "filesystem_name")); + try std.testing.expectEqual(@as(usize, 11), @offsetOf(MainBootSector, "must_be_zero")); + try std.testing.expectEqual(@as(usize, 80), @offsetOf(MainBootSector, "fat_offset")); + try std.testing.expectEqual(@as(usize, 88), @offsetOf(MainBootSector, "cluster_heap_offset")); + try std.testing.expectEqual(@as(usize, 96), @offsetOf(MainBootSector, "first_cluster_of_root")); + try std.testing.expectEqual(@as(usize, 106), @offsetOf(MainBootSector, "volume_flags")); + try std.testing.expectEqual(@as(usize, 112), @offsetOf(MainBootSector, "percent_in_use")); + try std.testing.expectEqual(@as(usize, 510), @offsetOf(MainBootSector, "boot_signature")); + // The Stream Extension's data location must sit where the spec places it. + try std.testing.expectEqual(@as(usize, 20), @offsetOf(StreamExtensionEntry, "first_cluster")); + try std.testing.expectEqual(@as(usize, 24), @offsetOf(StreamExtensionEntry, "data_length")); +} + +test "geometryOf accepts exFAT and the MustBeZero guard rejects a FAT-shaped sector" { + var sector = [_]u8{0} ** 512; + @memcpy(sector[3..11], "EXFAT "); + sector[510] = 0x55; + sector[511] = 0xAA; + // fat_offset=128, fat_length=64, cluster_heap_offset=256, cluster_count=1000, + // root cluster=5, bytes/sector=512 (shift 9), sectors/cluster=8 (shift 3), 1 FAT. + std.mem.writeInt(u32, sector[80..84], 128, .little); + std.mem.writeInt(u32, sector[84..88], 64, .little); + std.mem.writeInt(u32, sector[88..92], 256, .little); + std.mem.writeInt(u32, sector[92..96], 1000, .little); + std.mem.writeInt(u32, sector[96..100], 5, .little); + sector[108] = 9; // bytes_per_sector_shift + sector[109] = 3; // sectors_per_cluster_shift + sector[110] = 1; // number_of_fats + const geo = geometryOf(§or) orelse return error.ShouldParse; + try std.testing.expectEqual(@as(u32, 512), geo.bytes_per_sector); + try std.testing.expectEqual(@as(u32, 8), geo.sectors_per_cluster); + try std.testing.expectEqual(@as(u32, 1000), geo.cluster_count); + try std.testing.expectEqual(@as(u32, 5), geo.first_cluster_of_root); + + // A non-zero byte in MustBeZero (where a FAT BPB keeps bytes-per-sector) is + // rejected — the mutual exclusion between the engines. + sector[11] = 0x02; + try std.testing.expect(geometryOf(§or) == null); + sector[11] = 0; + // Wrong name is rejected too. + sector[3] = 'F'; + try std.testing.expect(geometryOf(§or) == null); +} + +test "set checksum skips its own two bytes and depends on the rest" { + var set = [_]u8{0} ** 64; // a File entry + one secondary + set[0] = entry_type_file; + set[1] = 1; + set[4] = 0x20; // an attribute byte + set[40] = 0xAB; // a byte in the secondary entry + const base = setChecksum(&set); + // Changing the checksum field itself must NOT change the computed checksum. + set[2] = 0xFF; + set[3] = 0xEE; + try std.testing.expectEqual(base, setChecksum(&set)); + // Changing any other byte MUST change it. + set[4] = 0x21; + try std.testing.expect(setChecksum(&set) != base); +} + +test "name hash is deterministic and order-sensitive" { + const readme = [_]u16{ 'R', 'E', 'A', 'D', 'M', 'E' }; + const different = [_]u16{ 'E', 'R', 'A', 'D', 'M', 'E' }; + try std.testing.expectEqual(nameHash(&readme), nameHash(&readme)); + try std.testing.expect(nameHash(&readme) != nameHash(&different)); +} + +test "boot checksum skips VolumeFlags and PercentInUse" { + var region = [_]u8{0} ** 1536; // three 512-byte sectors is enough to exercise the skips + region[64] = 0x11; + const base = bootChecksum(®ion); + for ([_]usize{ 106, 107, 112 }) |skipped| { + var copy = region; + copy[skipped] = 0xFF; + try std.testing.expectEqual(base, bootChecksum(©)); + } + var copy = region; + copy[108] = 0xFF; // a non-skipped byte + try std.testing.expect(bootChecksum(©) != base); +} + +test "exFAT timestamp <-> Unix epoch round trip" { + for ([_]u64{ 1_577_836_800, 1_700_000_000, 1_262_304_000, 1_783_971_244 }) |epoch| { + try std.testing.expectEqual(epoch, timestampToEpoch(epochToTimestamp(epoch))); + } + // 1577836800 is 2020-01-01 00:00:00 UTC. + const stamp = epochToTimestamp(1_577_836_800); + try std.testing.expectEqual(@as(u32, 2020), 1980 + (stamp >> 16 >> 9)); + try std.testing.expectEqual(@as(u64, 0), timestampToEpoch(0)); + try std.testing.expectEqual(@as(u32, 0), epochToTimestamp(0)); +}