danos/system/kernel/architecture/x86_64/trampoline.s

# AP trampoline: brings a waking application processor from the 16-bit real mode it
# starts in (after INIT-SIPI-SIPI) up through protected mode into 64-bit long mode,
# then jumps to the Zig AP entry (arch/x86_64/smp.zig:apEntry).
#
# A STARTUP IPI vectors a core to physical address `vector << 12` in real mode, so
# this blob is copied to a low (<1 MiB) page and started there; at entry CS = that
# page >> 4 and IP = 0. It is fully **position-independent**: it derives its own
# linear base (CS << 4) into EBX and addresses every internal datum as
# `(label - ap_trampoline_start)(%ebx)` — a difference of two symbols in the same
# section, which the assembler folds to a constant page offset no matter where the
# blob was linked or copied to. The BSP patches the parameter block (CR3, stack,
# entry, per-CPU pointer) before each wake; see arch/x86_64/smp.zig.
#
# It lives in .rodata (not .text): it is data to be copied out and executed
# elsewhere, never run at its link address, so it must not be a normal code segment.

.section .rodata
.balign 16
.code16
.global ap_trampoline_start
ap_trampoline_start:
    cli
    cld

    # Linear base of this page (CS << 4) into EBX; all data is addressed off it.
    xorl %eax, %eax
    mov %cs, %ax
    shll $4, %eax
    movl %eax, %ebx

    mov %cs, %ax            # DS = CS, so we address our data as DS:(label - start):
    mov %ax, %ds            # the segment base (CS<<4) already supplies the page base,
                            # so data operands use the page *offset*, not EBX.

    # Relocate the pointers whose absolute (linear) targets depend on where we were
    # copied: the GDT base and the two far-jump targets = EBX + their page offsets.
    # EBX supplies the base for the *value* (via leal); the store address is DS-rel.
    leal (gdt32 - ap_trampoline_start)(%ebx), %eax
    movl %eax, gdtr32_base - ap_trampoline_start
    leal (prot_entry - ap_trampoline_start)(%ebx), %eax
    movl %eax, jmp32_off - ap_trampoline_start
    leal (long_entry - ap_trampoline_start)(%ebx), %eax
    movl %eax, jmp64_off - ap_trampoline_start

    lgdtl gdtr32 - ap_trampoline_start

    movl %cr0, %eax         # enter protected mode (CR0.PE)
    orl $1, %eax
    movl %eax, %cr0

    ljmpl *(jmp32_ptr - ap_trampoline_start)   # -> prot_entry, CS = 0x08

.code32
prot_entry:
    movw $0x10, %ax         # flat 32-bit data segments
    movw %ax, %ds
    movw %ax, %es
    movw %ax, %ss
    movw %ax, %fs
    movw %ax, %gs

    # CR4: PAE (required for long mode) + OSFXSR/OSXMMEXCPT. The kernel is built with
    # SSE (part of the x86_64 baseline), and the compiler emits SSE for things as
    # ordinary as a struct copy — without OSFXSR those instructions #UD. The BSP got
    # these bits from UEFI; an AP starts fresh, so we must set them ourselves.
    movl %cr4, %eax
    orl $((1 << 5) | (1 << 9) | (1 << 10)), %eax
    movl %eax, %cr4

    # CR0: clear EM (no x87 emulation) and set MP, so SSE/x87 don't fault.
    movl %cr0, %eax
    andl $~(1 << 2), %eax   # ~EM
    orl $(1 << 1), %eax     # MP
    movl %eax, %cr0

    movl (param_cr3 - ap_trampoline_start)(%ebx), %eax  # kernel page tables
    movl %eax, %cr3

    movl $0xC0000080, %ecx  # EFER: long mode enable (LME) + NX enable (NXE, since
    rdmsr                   # the kernel's PTEs set the NX bit)
    orl $((1 << 8) | (1 << 11)), %eax
    wrmsr

    movl %cr0, %eax         # paging on (CR0.PG) — now in long mode (compat sub-mode)
    orl $(1 << 31), %eax
    movl %eax, %cr0

    ljmpl *(jmp64_ptr - ap_trampoline_start)(%ebx)   # -> long_entry, CS = 0x18 (L=1)

.code64
long_entry:
    movw $0x10, %ax         # sane flat data segments
    movw %ax, %ds
    movw %ax, %es
    movw %ax, %ss

    # RBX = EBX (zero-extended) = page base. Load our stack and per-CPU pointer, then
    # call the Zig entry — which runs from the kernel image and never returns.
    movq (param_stack - ap_trampoline_start)(%rbx), %rsp
    movq (param_percpu - ap_trampoline_start)(%rbx), %rdi   # SysV arg 0
    movq (param_entry - ap_trampoline_start)(%rbx), %rax
    callq *%rax
1:  hlt                     # unreachable; guard against a stray return
    jmp 1b

# --- data: GDT, far pointers, and the BSP-patched parameter block -----------
.balign 8
gdt32:
    .quad 0x0000000000000000    # 0x00 null
    .quad 0x00CF9A000000FFFF    # 0x08 32-bit code (G, D, present, exec/read)
    .quad 0x00CF92000000FFFF    # 0x10 data (valid in 32- and 64-bit)
    .quad 0x00AF9A000000FFFF    # 0x18 64-bit code (L=1)
gdt32_end:

gdtr32:
    .word gdt32_end - gdt32 - 1
gdtr32_base:
    .long 0                     # patched (16-bit code): linear base of gdt32

jmp32_ptr:                      # indirect far-jump operand: offset then selector
jmp32_off:
    .long 0                     # patched: linear address of prot_entry
    .word 0x08                  # 32-bit code selector

jmp64_ptr:
jmp64_off:
    .long 0                     # patched: linear address of long_entry
    .word 0x18                  # 64-bit code selector

# The parameter block, filled in by the BSP (smp.zig) before each STARTUP IPI. Global
# so the Zig side can locate each field as (symbol - ap_trampoline_start).
.global ap_tramp_cr3
.global ap_tramp_stack
.global ap_tramp_entry
.global ap_tramp_percpu
param_cr3:
ap_tramp_cr3:
    .quad 0                     # kernel PML4 physical address (CR3)
param_stack:
ap_tramp_stack:
    .quad 0                     # top of this AP's kernel stack
param_entry:
ap_tramp_entry:
    .quad 0                     # address of apEntry (the Zig AP entry)
param_percpu:
ap_tramp_percpu:
    .quad 0                     # this AP's per-CPU pointer (goes in GS base)

.global ap_trampoline_end
ap_trampoline_end: