1 use core::{
2 arch::asm,
3 hint::spin_loop,
4 sync::atomic::{compiler_fence, fence, AtomicBool, Ordering},
5 };
6
7 use kdepends::memoffset::offset_of;
8 use log::debug;
9 use system_error::SystemError;
10
11 use crate::{
12 arch::{mm::LowAddressRemapping, process::table::TSSManager, MMArch},
13 exception::InterruptArch,
14 libs::{cpumask::CpuMask, rwlock::RwLock},
15 mm::{percpu::PerCpu, MemoryManagementArch, PhysAddr, VirtAddr, IDLE_PROCESS_ADDRESS_SPACE},
16 process::ProcessManager,
17 smp::{
18 core::smp_get_processor_id,
19 cpu::{smp_cpu_manager, CpuHpCpuState, ProcessorId, SmpCpuManager},
20 init::smp_ap_start_stage2,
21 SMPArch,
22 },
23 };
24
25 use super::{
26 acpi::early_acpi_boot_init,
27 interrupt::ipi::{ipi_send_smp_init, ipi_send_smp_startup},
28 CurrentIrqArch,
29 };
30
31 extern "C" {
32 /// AP处理器启动时，会将CR3设置为这个值
33 pub static mut __APU_START_CR3: u64;
_apu_boot_start()34 fn _apu_boot_start();
_apu_boot_end()35 fn _apu_boot_end();
36 }
37
38 pub(super) static X86_64_SMP_MANAGER: X86_64SmpManager = X86_64SmpManager::new();
39
40 #[repr(C)]
41 struct ApStartStackInfo {
42 vaddr: usize,
43 }
44
45 /// AP处理器启动时执行
46 #[no_mangle]
smp_ap_start() -> !47 unsafe extern "C" fn smp_ap_start() -> ! {
48 CurrentIrqArch::interrupt_disable();
49
50 let vaddr = if let Some(t) = smp_cpu_manager()
51 .cpuhp_state(smp_get_processor_id())
52 .thread()
53 {
54 t.kernel_stack_force_ref().stack_max_address().data() - 16
55 } else {
56 // 没有设置ap核心的栈，那么就进入死循环。
57 loop {
58 spin_loop();
59 }
60 };
61 compiler_fence(core::sync::atomic::Ordering::SeqCst);
62 let v = ApStartStackInfo { vaddr };
63 smp_init_switch_stack(&v);
64 }
65
66 #[naked]
smp_init_switch_stack(st: &ApStartStackInfo) -> !67 unsafe extern "sysv64" fn smp_init_switch_stack(st: &ApStartStackInfo) -> ! {
68 asm!(concat!("
69 mov rsp, [rdi + {off_rsp}]
70 mov rbp, [rdi + {off_rsp}]
71 jmp {stage1}
72 "),
73 off_rsp = const(offset_of!(ApStartStackInfo, vaddr)),
74 stage1 = sym smp_ap_start_stage1,
75 options(noreturn));
76 }
77
smp_ap_start_stage1() -> !78 unsafe extern "C" fn smp_ap_start_stage1() -> ! {
79 let id = smp_get_processor_id();
80 debug!("smp_ap_start_stage1: id: {}\n", id.data());
81 let current_idle = ProcessManager::idle_pcb()[smp_get_processor_id().data() as usize].clone();
82
83 let tss = TSSManager::current_tss();
84
85 tss.set_rsp(
86 x86::Ring::Ring0,
87 current_idle.kernel_stack().stack_max_address().data() as u64,
88 );
89 TSSManager::load_tr();
90
91 CurrentIrqArch::arch_ap_early_irq_init().expect("arch_ap_early_irq_init failed");
92
93 smp_ap_start_stage2();
94 }
95
96 /// 多核的数据
97 #[derive(Debug)]
98 pub struct SmpBootData {
99 initialized: AtomicBool,
100 cpu_count: usize,
101 /// CPU的物理ID（指的是Local APIC ID）
102 ///
103 /// 这里必须保证第0项的是bsp的物理ID
104 phys_id: [usize; PerCpu::MAX_CPU_NUM as usize],
105 }
106
107 #[allow(dead_code)]
108 impl SmpBootData {
cpu_count(&self) -> usize109 pub fn cpu_count(&self) -> usize {
110 self.cpu_count
111 }
112
113 /// 获取CPU的物理ID
phys_id(&self, cpu_id: usize) -> usize114 pub fn phys_id(&self, cpu_id: usize) -> usize {
115 self.phys_id[cpu_id]
116 }
117
118 /// 获取BSP的物理ID
bsp_phys_id(&self) -> usize119 pub fn bsp_phys_id(&self) -> usize {
120 self.phys_id[0]
121 }
122
set_cpu_count(&self, cpu_count: u32)123 pub unsafe fn set_cpu_count(&self, cpu_count: u32) {
124 if !self.initialized.load(Ordering::SeqCst) {
125 let p = self as *const SmpBootData as *mut SmpBootData;
126 (*p).cpu_count = cpu_count.try_into().unwrap();
127 }
128 }
129
set_phys_id(&self, cpu_id: ProcessorId, phys_id: usize)130 pub unsafe fn set_phys_id(&self, cpu_id: ProcessorId, phys_id: usize) {
131 if !self.initialized.load(Ordering::SeqCst) {
132 let p = self as *const SmpBootData as *mut SmpBootData;
133 (*p).phys_id[cpu_id.data() as usize] = phys_id;
134 }
135 }
136
137 /// 标记boot data结构体已经初始化完成
mark_initialized(&self)138 pub fn mark_initialized(&self) {
139 self.initialized.store(true, Ordering::SeqCst);
140 }
141 }
142
143 pub(super) static SMP_BOOT_DATA: SmpBootData = SmpBootData {
144 initialized: AtomicBool::new(false),
145 cpu_count: 0,
146 phys_id: [0; PerCpu::MAX_CPU_NUM as usize],
147 };
148
149 #[allow(dead_code)]
150 #[derive(Debug)]
151 pub struct X86_64SmpManager {
152 ia64_cpu_to_sapicid: RwLock<[Option<usize>; PerCpu::MAX_CPU_NUM as usize]>,
153 }
154
155 impl X86_64SmpManager {
new() -> Self156 pub const fn new() -> Self {
157 return Self {
158 ia64_cpu_to_sapicid: RwLock::new([None; PerCpu::MAX_CPU_NUM as usize]),
159 };
160 }
161 /// initialize the logical cpu number to APIC ID mapping
build_cpu_map(&self) -> Result<(), SystemError>162 pub fn build_cpu_map(&self) -> Result<(), SystemError> {
163 // 参考：https://code.dragonos.org.cn/xref/linux-6.1.9/arch/ia64/kernel/smpboot.c?fi=smp_build_cpu_map#496
164 // todo!("build_cpu_map")
165 unsafe {
166 smp_cpu_manager().set_possible_cpu(ProcessorId::new(0), true);
167 smp_cpu_manager().set_present_cpu(ProcessorId::new(0), true);
168 smp_cpu_manager().set_online_cpu(ProcessorId::new(0));
169 }
170
171 for cpu in 1..SMP_BOOT_DATA.cpu_count() {
172 unsafe {
173 smp_cpu_manager().set_possible_cpu(ProcessorId::new(cpu as u32), true);
174 smp_cpu_manager().set_present_cpu(ProcessorId::new(cpu as u32), true);
175 }
176 }
177
178 print_cpus("possible", smp_cpu_manager().possible_cpus());
179 print_cpus("present", smp_cpu_manager().present_cpus());
180 return Ok(());
181 }
182 }
183
print_cpus(s: &str, mask: &CpuMask)184 fn print_cpus(s: &str, mask: &CpuMask) {
185 let mut v = vec![];
186 for cpu in mask.iter_cpu() {
187 v.push(cpu.data());
188 }
189
190 debug!("{s}: cpus: {v:?}\n");
191 }
192
193 pub struct X86_64SMPArch;
194
195 impl SMPArch for X86_64SMPArch {
196 #[inline(never)]
prepare_cpus() -> Result<(), SystemError>197 fn prepare_cpus() -> Result<(), SystemError> {
198 early_acpi_boot_init()?;
199 X86_64_SMP_MANAGER.build_cpu_map()?;
200 return Ok(());
201 }
202
post_init() -> Result<(), SystemError>203 fn post_init() -> Result<(), SystemError> {
204 // AP核心启动完毕，取消低地址映射
205 unsafe {
206 LowAddressRemapping::unmap_at_low_address(
207 &mut IDLE_PROCESS_ADDRESS_SPACE()
208 .write_irqsave()
209 .user_mapper
210 .utable,
211 true,
212 )
213 }
214 return Ok(());
215 }
216
start_cpu(cpu_id: ProcessorId, _cpu_hpstate: &CpuHpCpuState) -> Result<(), SystemError>217 fn start_cpu(cpu_id: ProcessorId, _cpu_hpstate: &CpuHpCpuState) -> Result<(), SystemError> {
218 Self::copy_smp_start_code();
219
220 fence(Ordering::SeqCst);
221 ipi_send_smp_init();
222 fence(Ordering::SeqCst);
223 ipi_send_smp_startup(cpu_id)?;
224
225 fence(Ordering::SeqCst);
226 ipi_send_smp_startup(cpu_id)?;
227
228 fence(Ordering::SeqCst);
229
230 return Ok(());
231 }
232 }
233
234 impl X86_64SMPArch {
235 const SMP_CODE_START: usize = 0x20000;
236 /// 复制SMP启动代码到0x20000处
copy_smp_start_code() -> (VirtAddr, usize)237 fn copy_smp_start_code() -> (VirtAddr, usize) {
238 let apu_boot_size = Self::start_code_size();
239
240 fence(Ordering::SeqCst);
241 unsafe {
242 core::ptr::copy(
243 _apu_boot_start as *const u8,
244 Self::SMP_CODE_START as *mut u8,
245 apu_boot_size,
246 )
247 };
248 fence(Ordering::SeqCst);
249
250 return (VirtAddr::new(Self::SMP_CODE_START), apu_boot_size);
251 }
252
start_code_size() -> usize253 fn start_code_size() -> usize {
254 let apu_boot_start = _apu_boot_start as usize;
255 let apu_boot_end = _apu_boot_end as usize;
256 let apu_boot_size = apu_boot_end - apu_boot_start;
257 return apu_boot_size;
258 }
259 }
260
261 impl SmpCpuManager {
262 #[allow(static_mut_refs)]
arch_init(_boot_cpu: ProcessorId)263 pub fn arch_init(_boot_cpu: ProcessorId) {
264 assert!(smp_get_processor_id().data() == 0);
265 // 写入APU_START_CR3，这个值会在AP处理器启动时设置到CR3寄存器
266 let addr = IDLE_PROCESS_ADDRESS_SPACE()
267 .read_irqsave()
268 .user_mapper
269 .utable
270 .table()
271 .phys();
272 let vaddr = unsafe {
273 MMArch::phys_2_virt(PhysAddr::new(&mut __APU_START_CR3 as *mut u64 as usize)).unwrap()
274 };
275 let ptr = vaddr.data() as *mut u64;
276 unsafe { *ptr = addr.data() as u64 };
277
278 // 添加低地址映射
279 unsafe {
280 LowAddressRemapping::remap_at_low_address(
281 &mut IDLE_PROCESS_ADDRESS_SPACE()
282 .write_irqsave()
283 .user_mapper
284 .utable,
285 )
286 };
287 }
288 }
289