# x86_64 low-level entry code: the CPU-exception stubs, plus the GDT/IDT load # helpers. Kept in a dedicated assembly file rather than inline asm because these # need real labels and cross-symbol jumps/calls (isr_common, exceptionHandler), # and because `lgdt`/`lidt` memory operands aren't expressible in Zig inline asm. # # Each exception vector normalises the stack to a uniform trap frame — a dummy # error code where the CPU pushes none, then the vector number — and jumps to the # shared tail, which saves the general registers and calls the Zig handler with a # pointer to the frame (matching src/arch/x86_64/idt.zig's CpuState). .text # _start: the kernel entry. The loader jumps here (higher-half address) with # boot_info in RDI, still on the loader's low stack. Switch to a kernel-owned # stack in .bss (the loader stack is a low address that goes away once the low # half is dropped), keeping RDI, then call the Zig entry. kmainEntry never # returns; the hlt loop is a belt-and-braces backstop. .global _start _start: leaq bootstrap_stack_top(%rip), %rsp call kmainEntry 1: hlt jmp 1b # The kernel's initial stack (used until the scheduler hands each task its own). # 64 KiB: kmain's discovery path includes the recursive AML interpreter, so it # needs more than a token stack. Lives in .bss (zeroed, higher-half). .section .bss .balign 16 bootstrap_stack: .skip 65536 bootstrap_stack_top: .text # gdt_flush(rdi = *GDT descriptor): load the GDT, reload the data segment # registers to the data selector, and reload CS to the code selector. CS can't be # set with mov, so we far-return through the caller's own return address. .global gdt_flush gdt_flush: lgdt (%rdi) mov $0x10, %ax # kernel data selector mov %ax, %ds mov %ax, %es mov %ax, %ss mov %ax, %fs mov %ax, %gs pop %rax # caller's return address push $0x08 # kernel code selector (new CS) push %rax # return address (new RIP) lretq # idt_flush(rdi = *IDT descriptor): load the IDT. .global idt_flush idt_flush: lidt (%rdi) ret # load_tr(di = TSS selector): load the task register. .global load_tr load_tr: ltr %di ret # switch_context(rdi = &old_task.rsp, rsi = new_task.rsp) # Cooperative context switch: save the callee-saved registers on the current # stack, stash the stack pointer in the old task, load the new task's stack # pointer, restore its callee-saved registers, and return into it. Caller-saved # registers are the compiler's responsibility (this looks like a normal call). .global switch_context switch_context: push %rbx push %rbp push %r12 push %r13 push %r14 push %r15 mov %rsp, (%rdi) # save old stack pointer into old_task.rsp mov %rsi, %rsp # switch to the new task's stack pop %r15 pop %r14 pop %r13 pop %r12 pop %rbp pop %rbx ret # return into the new task's saved instruction pointer # task_trampoline: the first thing a freshly-spawned task runs. init_task_stack # leaves its entry function in r15. A fresh task is switched to with the big kernel # lock held (the hand-off rule in sync.zig) but has no enter/leave frame of its own, # so it releases the lock here before running its body. r15 survives the call (it's # callee-saved). New tasks then start with interrupts enabled. .extern releaseForFreshTask .global task_trampoline task_trampoline: call releaseForFreshTask # drop the kernel lock we inherited across the switch sti call *%r15 # call the task entry (fn() void) 1: hlt # if the entry returns, idle (still preemptible) jmp 1b # user_task_trampoline: the first thing a freshly-spawned *user* task runs. # init_user_task_stack leaves the user entry in r15 and the user stack in r14 # (both callee-saved, so they survive the lock-release call). Like task_trampoline # it drops the inherited kernel lock, then — instead of calling a kernel fn — it # builds an iretq frame and drops to ring 3. The scheduler's switchTo already # loaded this task's address space (CR3) and published its kernel stack # (TSS.rsp0 + gs kernel_rsp) before switching here, so interrupts and syscalls # from ring 3 land correctly. IF is set in the pushed RFLAGS (no sti needed). # jump_to_user(rdi = user rip, rsi = user rsp): drop the current kernel context # to ring 3, never returning. The scheduler calls this from a fresh user task's # trampoline (after the lock is released and the entry/stack read from the Task). # cli guards the swapgs..iretq window: an interrupt there would run in ring 0 # with the user GS base and mis-read per-CPU data. The pushed RFLAGS (IF set) # re-enables interrupts on the drop to ring 3. .global jump_to_user jump_to_user: cli push $0x1B # user SS (0x18 | RPL 3) push %rsi # user RSP push $0x202 # RFLAGS: IF | reserved-1 push $0x23 # user CS (0x20 | RPL 3) push %rdi # user RIP swapgs # user GS base (isr_common/syscall swap back on entry) iretq # jump_to_user_arg(rdi = user rip, rsi = user rsp, rdx = user rdi/arg0): as # jump_to_user, but delivers arg0 in the user's rdi — how a fresh **thread** # receives its closure pointer (docs/threading.md). rdi carries the rip only until # it is pushed into the iretq frame, after which we overwrite it with the arg. .global jump_to_user_arg jump_to_user_arg: cli push $0x1B # user SS (0x18 | RPL 3) push %rsi # user RSP push $0x202 # RFLAGS: IF | reserved-1 push $0x23 # user CS (0x20 | RPL 3) push %rdi # user RIP (consumes rdi) mov %rdx, %rdi # user rdi = arg0 (the thread's closure pointer) swapgs # user GS base (isr_common/syscall swap back on entry) iretq # --- ring 3 entry/exit ------------------------------------------------------ # enter_user(rdi = user rip, rsi = user rsp, rdx = &TSS.rsp0) # Drop to ring 3. Saves the callee-saved registers and the kernel stack pointer # (so user_exit_to_kernel can unwind back here), publishes that stack pointer as # this core's TSS.rsp0 — everything below it is dead, so ring-3 interrupt frames # grow safely into it — then builds the 5-word iretq frame with the user # selectors (RPL 3) and drops privilege. IF is set in the pushed RFLAGS so the # timer keeps running in user mode. .global enter_user enter_user: push %rbx push %rbp push %r12 push %r13 push %r14 push %r15 mov %rsp, user_saved_rsp(%rip) # where user_exit_to_kernel unwinds to mov %rsp, (%rdx) # TSS.rsp0: ring-3 interrupts stack here mov %rsp, %gs:0 # kernel_rsp: the syscall stub stacks here too push $0x1B # user SS (0x18 | RPL 3) push %rsi # user RSP push $0x202 # RFLAGS: IF | reserved-1 push $0x23 # user CS (0x20 | RPL 3) push %rdi # user RIP swapgs # user GS base for ring 3 (isr_common swaps back) iretq # user_exit_to_kernel: abandon the in-flight ring-3 trap frame (it lives in the # dead zone below user_saved_rsp) and return as if enter_user's call completed. # Reached from the exit syscall's handler; interrupts are off (interrupt gate) # and stay off — the Zig caller re-enables. .global user_exit_to_kernel user_exit_to_kernel: mov user_saved_rsp(%rip), %rsp pop %r15 pop %r14 pop %r13 pop %r12 pop %rbp pop %rbx ret # syscall_entry: the target of the SYSCALL instruction (LSTAR). The CPU does NOT # switch stacks — it puts the return RIP in RCX, the saved RFLAGS in R11, loads # CS/SS from STAR, masks RFLAGS with SFMASK (so IF is already clear), and jumps # here with RSP still the *user* stack. We swap in the kernel GS, switch to the # task's kernel stack via the per-CPU block, build a CpuState frame identical to # the interrupt path's, and reuse interruptDispatch (vector 128) — then SYSRET. # # Hazard (acceptable while init is the only, trusted, user program): SYSRETQ #GPs # in ring 0 if the return RIP (RCX) is non-canonical. A hostile user could arrange # that; hardening (canonical check / iretq fallback) is a later security-track item. .global syscall_entry syscall_entry: swapgs # kernel GS base movq %rsp, %gs:8 # stash user rsp in the scratch slot movq %gs:0, %rsp # switch to this task's kernel stack # Build the trap frame (same field order as isr_common), highest field first. pushq $0x1B # ss (user data | 3) pushq %gs:8 # rsp (user, from scratch) pushq %r11 # rflags (saved by syscall) pushq $0x23 # cs (user code | 3) pushq %rcx # rip (saved by syscall) pushq $0 # error_code (none for a syscall) pushq $128 # vector (same as the int 0x80 gate) push %rax push %rbx push %rcx push %rdx push %rsi push %rdi push %rbp push %r8 push %r9 push %r10 push %r11 push %r12 push %r13 push %r14 push %r15 mov %rsp, %rdi # trap-frame pointer # Preserve the caller's SSE/x87 register file across the syscall — see the same # dance in isr_common. Without it a syscall (or a task the scheduler runs while # this one blocks) clobbers the caller's live XMM values, which the compiler is # free to hold across a syscall (its wrappers only clobber rcx/r11/memory). mov %rsp, %rbx and $-16, %rsp sub $512, %rsp fxsave (%rsp) call interruptDispatch fxrstor (%rsp) mov %rbx, %rsp # back to the trap frame (undo the fxsave scratch) pop %r15 pop %r14 pop %r13 pop %r12 pop %r11 pop %r10 pop %r9 pop %r8 pop %rbp pop %rdi pop %rsi pop %rdx pop %rcx pop %rbx pop %rax add $16, %rsp # drop vector + error_code -> rsp at rip popq %rcx # rip -> RCX (SYSRETQ restores RIP from RCX) addq $8, %rsp # skip the cs slot (SYSRETQ loads CS from STAR) popq %r11 # rflags -> R11 (SYSRETQ restores RFLAGS from R11) popq %rsp # user rsp (the ss slot below is abandoned) swapgs # user GS base sysretq # -> ring 3: RIP=RCX, RFLAGS=R11, CS/SS from STAR .section .bss .balign 8 user_saved_rsp: .skip 8 .text # --- user-mode test program -------------------------------------------------- # A hand-assembled ring-3 blob, copied by the kernel onto a user-mapped page and # entered via enter_user. Position-independent (immediates and short jumps only). # In .rodata: these bytes are data to the kernel — they only execute at CPL 3 # from the user mapping. (The old hello/ping blob was retired once /sbin/init # became the real ring-3 exerciser; only the isolation proof remains.) .section .rodata # The isolation-proof program: read a kernel-only page from ring 3. The LAPIC # lives in the kernel's physmap (physmap_base + 0xFEE00000) as a supervisor # page, so this must take a #PF with error code 0x5 (present | user) before any # access happens. movabs loads the full 64-bit higher-half address (a disp32 # would sign-extend and miss). .global user_pf_start .global user_pf_end user_pf_start: movabs $0xFFFF8800FEE00000, %rcx mov (%rcx), %rax 1: jmp 1b user_pf_end: .text # Stub for a vector the CPU does NOT push an error code for: push a dummy 0. .macro STUB_NOERR vec .global isr\vec isr\vec: pushq $0 pushq $\vec jmp isr_common .endm # Stub for a vector the CPU DOES push an error code for: leave it in place. .macro STUB_ERR vec .global isr\vec isr\vec: pushq $\vec jmp isr_common .endm STUB_NOERR 0 STUB_NOERR 1 STUB_NOERR 2 STUB_NOERR 3 STUB_NOERR 4 STUB_NOERR 5 STUB_NOERR 6 STUB_NOERR 7 STUB_ERR 8 STUB_NOERR 9 STUB_ERR 10 STUB_ERR 11 STUB_ERR 12 STUB_ERR 13 STUB_ERR 14 STUB_NOERR 15 STUB_NOERR 16 STUB_ERR 17 STUB_NOERR 18 STUB_NOERR 19 STUB_NOERR 20 STUB_ERR 21 STUB_NOERR 22 STUB_NOERR 23 STUB_NOERR 24 STUB_NOERR 25 STUB_NOERR 26 STUB_NOERR 27 STUB_NOERR 28 STUB_NOERR 29 STUB_NOERR 30 STUB_NOERR 31 # Device-interrupt vectors (timer, spurious, room for more). None push an error # code, so they all use the dummy-zero form. STUB_NOERR 32 STUB_NOERR 33 STUB_NOERR 34 STUB_NOERR 35 STUB_NOERR 36 STUB_NOERR 37 STUB_NOERR 38 STUB_NOERR 39 STUB_NOERR 40 STUB_NOERR 41 STUB_NOERR 42 STUB_NOERR 43 STUB_NOERR 44 STUB_NOERR 45 STUB_NOERR 46 STUB_NOERR 47 # The syscall gate (int $0x80 from ring 3). Same frame shape as every other # vector; dispatched specially in interruptDispatch. STUB_NOERR 128 .extern interruptDispatch # Shared tail. Register push order here defines the CpuState field order. # If the interrupt came from ring 3 the GS base holds the user's value, so swap # in the kernel's before anything reads per-CPU data (swapgs discipline; see # percpu.zig). CS sits at offset 24 here (vector@0, error@8, RIP@16, CS@24). isr_common: testb $3, 24(%rsp) jz 1f swapgs 1: push %rax push %rbx push %rcx push %rdx push %rsi push %rdi push %rbp push %r8 push %r9 push %r10 push %r11 push %r12 push %r13 push %r14 push %r15 mov %rsp, %rdi # first argument: pointer to the trap frame # Save the interrupted SSE/x87 register file before any kernel code runs, and # restore it on the way out — the kernel and user both keep live values in XMM # (a 16-byte struct copy is a movdqu), and the kernel never otherwise preserves # them, so an interrupt handler (and whatever the scheduler runs in its place) # would silently clobber the interrupted task's vector registers. rbx bridges the # exact rsp across the call: it is callee-saved (interruptDispatch and every # context switch preserve it), so it survives even a blocking dispatch, and the # `and`/`sub` gives fxsave its required 16-byte-aligned scratch on the kernel stack. mov %rsp, %rbx and $-16, %rsp sub $512, %rsp fxsave (%rsp) call interruptDispatch fxrstor (%rsp) mov %rbx, %rsp # back to the trap frame (undo the fxsave scratch) pop %r15 pop %r14 pop %r13 pop %r12 pop %r11 pop %r10 pop %r9 pop %r8 pop %rbp pop %rdi pop %rsi pop %rdx pop %rcx pop %rbx pop %rax add $16, %rsp # drop the vector and error code # Symmetric to entry: if returning to ring 3, restore the user GS base. CS is # now at offset 8 (RIP@0, CS@8). testb $3, 8(%rsp) jz 1f swapgs 1: iretq