151 lines
5.6 KiB
ArmAsm
151 lines
5.6 KiB
ArmAsm
# AP trampoline: brings a waking application processor from the 16-bit real mode it
|
|
# starts in (after INIT-SIPI-SIPI) up through protected mode into 64-bit long mode,
|
|
# then jumps to the Zig AP entry (arch/x86_64/smp.zig:apEntry).
|
|
#
|
|
# A STARTUP IPI vectors a core to physical address `vector << 12` in real mode, so
|
|
# this blob is copied to a low (<1 MiB) page and started there; at entry CS = that
|
|
# page >> 4 and IP = 0. It is fully **position-independent**: it derives its own
|
|
# linear base (CS << 4) into EBX and addresses every internal datum as
|
|
# `(label - ap_trampoline_start)(%ebx)` — a difference of two symbols in the same
|
|
# section, which the assembler folds to a constant page offset no matter where the
|
|
# blob was linked or copied to. The BSP patches the parameter block (CR3, stack,
|
|
# entry, per-CPU pointer) before each wake; see arch/x86_64/smp.zig.
|
|
#
|
|
# It lives in .rodata (not .text): it is data to be copied out and executed
|
|
# elsewhere, never run at its link address, so it must not be a normal code segment.
|
|
|
|
.section .rodata
|
|
.balign 16
|
|
.code16
|
|
.global ap_trampoline_start
|
|
ap_trampoline_start:
|
|
cli
|
|
cld
|
|
|
|
# Linear base of this page (CS << 4) into EBX; all data is addressed off it.
|
|
xorl %eax, %eax
|
|
mov %cs, %ax
|
|
shll $4, %eax
|
|
movl %eax, %ebx
|
|
|
|
mov %cs, %ax # DS = CS, so we address our data as DS:(label - start):
|
|
mov %ax, %ds # the segment base (CS<<4) already supplies the page base,
|
|
# so data operands use the page *offset*, not EBX.
|
|
|
|
# Relocate the pointers whose absolute (linear) targets depend on where we were
|
|
# copied: the GDT base and the two far-jump targets = EBX + their page offsets.
|
|
# EBX supplies the base for the *value* (via leal); the store address is DS-rel.
|
|
leal (gdt32 - ap_trampoline_start)(%ebx), %eax
|
|
movl %eax, gdtr32_base - ap_trampoline_start
|
|
leal (prot_entry - ap_trampoline_start)(%ebx), %eax
|
|
movl %eax, jmp32_off - ap_trampoline_start
|
|
leal (long_entry - ap_trampoline_start)(%ebx), %eax
|
|
movl %eax, jmp64_off - ap_trampoline_start
|
|
|
|
lgdtl gdtr32 - ap_trampoline_start
|
|
|
|
movl %cr0, %eax # enter protected mode (CR0.PE)
|
|
orl $1, %eax
|
|
movl %eax, %cr0
|
|
|
|
ljmpl *(jmp32_ptr - ap_trampoline_start) # -> prot_entry, CS = 0x08
|
|
|
|
.code32
|
|
prot_entry:
|
|
movw $0x10, %ax # flat 32-bit data segments
|
|
movw %ax, %ds
|
|
movw %ax, %es
|
|
movw %ax, %ss
|
|
movw %ax, %fs
|
|
movw %ax, %gs
|
|
|
|
# CR4: PAE (required for long mode) + OSFXSR/OSXMMEXCPT. The kernel is built with
|
|
# SSE (part of the x86_64 baseline), and the compiler emits SSE for things as
|
|
# ordinary as a struct copy — without OSFXSR those instructions #UD. The BSP got
|
|
# these bits from UEFI; an AP starts fresh, so we must set them ourselves.
|
|
movl %cr4, %eax
|
|
orl $((1 << 5) | (1 << 9) | (1 << 10)), %eax
|
|
movl %eax, %cr4
|
|
|
|
# CR0: clear EM (no x87 emulation) and set MP, so SSE/x87 don't fault.
|
|
movl %cr0, %eax
|
|
andl $~(1 << 2), %eax # ~EM
|
|
orl $(1 << 1), %eax # MP
|
|
movl %eax, %cr0
|
|
|
|
movl (param_cr3 - ap_trampoline_start)(%ebx), %eax # kernel page tables
|
|
movl %eax, %cr3
|
|
|
|
movl $0xC0000080, %ecx # EFER: long mode enable (LME) + NX enable (NXE, since
|
|
rdmsr # the kernel's PTEs set the NX bit)
|
|
orl $((1 << 8) | (1 << 11)), %eax
|
|
wrmsr
|
|
|
|
movl %cr0, %eax # paging on (CR0.PG) — now in long mode (compat sub-mode)
|
|
orl $(1 << 31), %eax
|
|
movl %eax, %cr0
|
|
|
|
ljmpl *(jmp64_ptr - ap_trampoline_start)(%ebx) # -> long_entry, CS = 0x18 (L=1)
|
|
|
|
.code64
|
|
long_entry:
|
|
movw $0x10, %ax # sane flat data segments
|
|
movw %ax, %ds
|
|
movw %ax, %es
|
|
movw %ax, %ss
|
|
|
|
# RBX = EBX (zero-extended) = page base. Load our stack and per-CPU pointer, then
|
|
# call the Zig entry — which runs from the kernel image and never returns.
|
|
movq (param_stack - ap_trampoline_start)(%rbx), %rsp
|
|
movq (param_percpu - ap_trampoline_start)(%rbx), %rdi # SysV arg 0
|
|
movq (param_entry - ap_trampoline_start)(%rbx), %rax
|
|
callq *%rax
|
|
1: hlt # unreachable; guard against a stray return
|
|
jmp 1b
|
|
|
|
# --- data: GDT, far pointers, and the BSP-patched parameter block -----------
|
|
.balign 8
|
|
gdt32:
|
|
.quad 0x0000000000000000 # 0x00 null
|
|
.quad 0x00CF9A000000FFFF # 0x08 32-bit code (G, D, present, exec/read)
|
|
.quad 0x00CF92000000FFFF # 0x10 data (valid in 32- and 64-bit)
|
|
.quad 0x00AF9A000000FFFF # 0x18 64-bit code (L=1)
|
|
gdt32_end:
|
|
|
|
gdtr32:
|
|
.word gdt32_end - gdt32 - 1
|
|
gdtr32_base:
|
|
.long 0 # patched (16-bit code): linear base of gdt32
|
|
|
|
jmp32_ptr: # indirect far-jump operand: offset then selector
|
|
jmp32_off:
|
|
.long 0 # patched: linear address of prot_entry
|
|
.word 0x08 # 32-bit code selector
|
|
|
|
jmp64_ptr:
|
|
jmp64_off:
|
|
.long 0 # patched: linear address of long_entry
|
|
.word 0x18 # 64-bit code selector
|
|
|
|
# The parameter block, filled in by the BSP (smp.zig) before each STARTUP IPI. Global
|
|
# so the Zig side can locate each field as (symbol - ap_trampoline_start).
|
|
.global ap_tramp_cr3
|
|
.global ap_tramp_stack
|
|
.global ap_tramp_entry
|
|
.global ap_tramp_percpu
|
|
param_cr3:
|
|
ap_tramp_cr3:
|
|
.quad 0 # kernel PML4 physical address (CR3)
|
|
param_stack:
|
|
ap_tramp_stack:
|
|
.quad 0 # top of this AP's kernel stack
|
|
param_entry:
|
|
ap_tramp_entry:
|
|
.quad 0 # address of apEntry (the Zig AP entry)
|
|
param_percpu:
|
|
ap_tramp_percpu:
|
|
.quad 0 # this AP's per-CPU pointer (goes in GS base)
|
|
|
|
.global ap_trampoline_end
|
|
ap_trampoline_end:
|