1 use core::{
2 hint::spin_loop,
3 sync::atomic::{compiler_fence, fence, AtomicBool, Ordering},
4 };
5
6 use kdepends::memoffset::offset_of;
7 use log::debug;
8 use system_error::SystemError;
9
10 use crate::{
11 arch::{mm::LowAddressRemapping, process::table::TSSManager, MMArch},
12 exception::InterruptArch,
13 libs::{cpumask::CpuMask, rwlock::RwLock},
14 mm::{percpu::PerCpu, MemoryManagementArch, PhysAddr, VirtAddr, IDLE_PROCESS_ADDRESS_SPACE},
15 process::ProcessManager,
16 smp::{
17 core::smp_get_processor_id,
18 cpu::{smp_cpu_manager, CpuHpCpuState, ProcessorId, SmpCpuManager},
19 init::smp_ap_start_stage2,
20 SMPArch,
21 },
22 };
23
24 use super::{
25 acpi::early_acpi_boot_init,
26 interrupt::ipi::{ipi_send_smp_init, ipi_send_smp_startup},
27 CurrentIrqArch,
28 };
29
30 extern "C" {
31 /// AP处理器启动时，会将CR3设置为这个值
32 pub static mut __APU_START_CR3: u64;
_apu_boot_start()33 fn _apu_boot_start();
_apu_boot_end()34 fn _apu_boot_end();
35 }
36
37 pub(super) static X86_64_SMP_MANAGER: X86_64SmpManager = X86_64SmpManager::new();
38
39 #[repr(C)]
40 struct ApStartStackInfo {
41 vaddr: usize,
42 }
43
44 /// AP处理器启动时执行
45 #[no_mangle]
smp_ap_start() -> !46 unsafe extern "C" fn smp_ap_start() -> ! {
47 CurrentIrqArch::interrupt_disable();
48
49 let vaddr = if let Some(t) = smp_cpu_manager()
50 .cpuhp_state(smp_get_processor_id())
51 .thread()
52 {
53 t.kernel_stack_force_ref().stack_max_address().data() - 16
54 } else {
55 // 没有设置ap核心的栈，那么就进入死循环。
56 loop {
57 spin_loop();
58 }
59 };
60 compiler_fence(core::sync::atomic::Ordering::SeqCst);
61 let v = ApStartStackInfo { vaddr };
62 smp_init_switch_stack(&v);
63 }
64
65 #[naked]
smp_init_switch_stack(st: &ApStartStackInfo) -> !66 unsafe extern "sysv64" fn smp_init_switch_stack(st: &ApStartStackInfo) -> ! {
67 core::arch::naked_asm!(concat!("
68 mov rsp, [rdi + {off_rsp}]
69 mov rbp, [rdi + {off_rsp}]
70 jmp {stage1}
71 "),
72 off_rsp = const(offset_of!(ApStartStackInfo, vaddr)),
73 stage1 = sym smp_ap_start_stage1);
74 }
75
smp_ap_start_stage1() -> !76 unsafe extern "C" fn smp_ap_start_stage1() -> ! {
77 let id = smp_get_processor_id();
78 debug!("smp_ap_start_stage1: id: {}\n", id.data());
79 let current_idle = ProcessManager::idle_pcb()[smp_get_processor_id().data() as usize].clone();
80
81 let tss = TSSManager::current_tss();
82
83 tss.set_rsp(
84 x86::Ring::Ring0,
85 current_idle.kernel_stack().stack_max_address().data() as u64,
86 );
87 TSSManager::load_tr();
88
89 CurrentIrqArch::arch_ap_early_irq_init().expect("arch_ap_early_irq_init failed");
90
91 smp_ap_start_stage2();
92 }
93
94 /// 多核的数据
95 #[derive(Debug)]
96 pub struct SmpBootData {
97 initialized: AtomicBool,
98 cpu_count: usize,
99 /// CPU的物理ID（指的是Local APIC ID）
100 ///
101 /// 这里必须保证第0项的是bsp的物理ID
102 phys_id: [usize; PerCpu::MAX_CPU_NUM as usize],
103 }
104
105 #[allow(dead_code)]
106 impl SmpBootData {
cpu_count(&self) -> usize107 pub fn cpu_count(&self) -> usize {
108 self.cpu_count
109 }
110
111 /// 获取CPU的物理ID
phys_id(&self, cpu_id: usize) -> usize112 pub fn phys_id(&self, cpu_id: usize) -> usize {
113 self.phys_id[cpu_id]
114 }
115
116 /// 获取BSP的物理ID
bsp_phys_id(&self) -> usize117 pub fn bsp_phys_id(&self) -> usize {
118 self.phys_id[0]
119 }
120
set_cpu_count(&self, cpu_count: u32)121 pub unsafe fn set_cpu_count(&self, cpu_count: u32) {
122 if !self.initialized.load(Ordering::SeqCst) {
123 let p = self as *const SmpBootData as *mut SmpBootData;
124 (*p).cpu_count = cpu_count.try_into().unwrap();
125 }
126 }
127
set_phys_id(&self, cpu_id: ProcessorId, phys_id: usize)128 pub unsafe fn set_phys_id(&self, cpu_id: ProcessorId, phys_id: usize) {
129 if !self.initialized.load(Ordering::SeqCst) {
130 let p = self as *const SmpBootData as *mut SmpBootData;
131 (*p).phys_id[cpu_id.data() as usize] = phys_id;
132 }
133 }
134
135 /// 标记boot data结构体已经初始化完成
mark_initialized(&self)136 pub fn mark_initialized(&self) {
137 self.initialized.store(true, Ordering::SeqCst);
138 }
139 }
140
141 pub(super) static SMP_BOOT_DATA: SmpBootData = SmpBootData {
142 initialized: AtomicBool::new(false),
143 cpu_count: 0,
144 phys_id: [0; PerCpu::MAX_CPU_NUM as usize],
145 };
146
147 #[allow(dead_code)]
148 #[derive(Debug)]
149 pub struct X86_64SmpManager {
150 ia64_cpu_to_sapicid: RwLock<[Option<usize>; PerCpu::MAX_CPU_NUM as usize]>,
151 }
152
153 impl X86_64SmpManager {
new() -> Self154 pub const fn new() -> Self {
155 return Self {
156 ia64_cpu_to_sapicid: RwLock::new([None; PerCpu::MAX_CPU_NUM as usize]),
157 };
158 }
159 /// initialize the logical cpu number to APIC ID mapping
build_cpu_map(&self) -> Result<(), SystemError>160 pub fn build_cpu_map(&self) -> Result<(), SystemError> {
161 // 参考：https://code.dragonos.org.cn/xref/linux-6.1.9/arch/ia64/kernel/smpboot.c?fi=smp_build_cpu_map#496
162 // todo!("build_cpu_map")
163 unsafe {
164 smp_cpu_manager().set_possible_cpu(ProcessorId::new(0), true);
165 smp_cpu_manager().set_present_cpu(ProcessorId::new(0), true);
166 smp_cpu_manager().set_online_cpu(ProcessorId::new(0));
167 }
168
169 for cpu in 1..SMP_BOOT_DATA.cpu_count() {
170 unsafe {
171 smp_cpu_manager().set_possible_cpu(ProcessorId::new(cpu as u32), true);
172 smp_cpu_manager().set_present_cpu(ProcessorId::new(cpu as u32), true);
173 }
174 }
175
176 print_cpus("possible", smp_cpu_manager().possible_cpus());
177 print_cpus("present", smp_cpu_manager().present_cpus());
178 return Ok(());
179 }
180 }
181
print_cpus(s: &str, mask: &CpuMask)182 fn print_cpus(s: &str, mask: &CpuMask) {
183 let mut v = vec![];
184 for cpu in mask.iter_cpu() {
185 v.push(cpu.data());
186 }
187
188 debug!("{s}: cpus: {v:?}\n");
189 }
190
191 pub struct X86_64SMPArch;
192
193 impl SMPArch for X86_64SMPArch {
194 #[inline(never)]
prepare_cpus() -> Result<(), SystemError>195 fn prepare_cpus() -> Result<(), SystemError> {
196 early_acpi_boot_init()?;
197 X86_64_SMP_MANAGER.build_cpu_map()?;
198 return Ok(());
199 }
200
post_init() -> Result<(), SystemError>201 fn post_init() -> Result<(), SystemError> {
202 // AP核心启动完毕，取消低地址映射
203 unsafe {
204 LowAddressRemapping::unmap_at_low_address(
205 &mut IDLE_PROCESS_ADDRESS_SPACE()
206 .write_irqsave()
207 .user_mapper
208 .utable,
209 true,
210 )
211 }
212 return Ok(());
213 }
214
start_cpu(cpu_id: ProcessorId, _cpu_hpstate: &CpuHpCpuState) -> Result<(), SystemError>215 fn start_cpu(cpu_id: ProcessorId, _cpu_hpstate: &CpuHpCpuState) -> Result<(), SystemError> {
216 Self::copy_smp_start_code();
217
218 fence(Ordering::SeqCst);
219 ipi_send_smp_init();
220 fence(Ordering::SeqCst);
221 ipi_send_smp_startup(cpu_id)?;
222
223 fence(Ordering::SeqCst);
224 ipi_send_smp_startup(cpu_id)?;
225
226 fence(Ordering::SeqCst);
227
228 return Ok(());
229 }
230 }
231
232 impl X86_64SMPArch {
233 const SMP_CODE_START: usize = 0x20000;
234 /// 复制SMP启动代码到0x20000处
copy_smp_start_code() -> (VirtAddr, usize)235 fn copy_smp_start_code() -> (VirtAddr, usize) {
236 let apu_boot_size = Self::start_code_size();
237
238 fence(Ordering::SeqCst);
239 unsafe {
240 core::ptr::copy(
241 _apu_boot_start as *const u8,
242 Self::SMP_CODE_START as *mut u8,
243 apu_boot_size,
244 )
245 };
246 fence(Ordering::SeqCst);
247
248 return (VirtAddr::new(Self::SMP_CODE_START), apu_boot_size);
249 }
250
start_code_size() -> usize251 fn start_code_size() -> usize {
252 let apu_boot_start = _apu_boot_start as usize;
253 let apu_boot_end = _apu_boot_end as usize;
254 let apu_boot_size = apu_boot_end - apu_boot_start;
255 return apu_boot_size;
256 }
257 }
258
259 impl SmpCpuManager {
260 #[allow(static_mut_refs)]
arch_init(_boot_cpu: ProcessorId)261 pub fn arch_init(_boot_cpu: ProcessorId) {
262 assert!(smp_get_processor_id().data() == 0);
263 // 写入APU_START_CR3，这个值会在AP处理器启动时设置到CR3寄存器
264 let addr = IDLE_PROCESS_ADDRESS_SPACE()
265 .read_irqsave()
266 .user_mapper
267 .utable
268 .table()
269 .phys();
270 let vaddr = unsafe {
271 MMArch::phys_2_virt(PhysAddr::new(&mut __APU_START_CR3 as *mut u64 as usize)).unwrap()
272 };
273 let ptr = vaddr.data() as *mut u64;
274 unsafe { *ptr = addr.data() as u64 };
275
276 // 添加低地址映射
277 unsafe {
278 LowAddressRemapping::remap_at_low_address(
279 &mut IDLE_PROCESS_ADDRESS_SPACE()
280 .write_irqsave()
281 .user_mapper
282 .utable,
283 )
284 };
285 }
286 }
287