1 use core::{ 2 arch::asm, 3 hint::spin_loop, 4 sync::atomic::{compiler_fence, fence, AtomicBool, Ordering}, 5 }; 6 7 use kdepends::memoffset::offset_of; 8 use system_error::SystemError; 9 10 use crate::{ 11 arch::{mm::LowAddressRemapping, process::table::TSSManager, MMArch}, 12 exception::InterruptArch, 13 kdebug, 14 libs::{cpumask::CpuMask, rwlock::RwLock}, 15 mm::{percpu::PerCpu, MemoryManagementArch, PhysAddr, VirtAddr, IDLE_PROCESS_ADDRESS_SPACE}, 16 process::ProcessManager, 17 smp::{ 18 core::smp_get_processor_id, 19 cpu::{smp_cpu_manager, CpuHpCpuState, ProcessorId, SmpCpuManager}, 20 init::smp_ap_start_stage2, 21 SMPArch, 22 }, 23 }; 24 25 use super::{ 26 acpi::early_acpi_boot_init, 27 interrupt::ipi::{ipi_send_smp_init, ipi_send_smp_startup}, 28 CurrentIrqArch, 29 }; 30 31 extern "C" { 32 /// AP处理器启动时，会将CR3设置为这个值 33 pub static mut __APU_START_CR3: u64; 34 fn _apu_boot_start(); 35 fn _apu_boot_end(); 36 } 37 38 pub(super) static X86_64_SMP_MANAGER: X86_64SmpManager = X86_64SmpManager::new(); 39 40 #[repr(C)] 41 struct ApStartStackInfo { 42 vaddr: usize, 43 } 44 45 /// AP处理器启动时执行 46 #[no_mangle] 47 unsafe extern "C" fn smp_ap_start() -> ! { 48 CurrentIrqArch::interrupt_disable(); 49 let vaddr = if let Some(t) = smp_cpu_manager() 50 .cpuhp_state(smp_get_processor_id()) 51 .thread() 52 { 53 t.kernel_stack().stack_max_address().data() - 16 54 } else { 55 // 没有设置ap核心的栈，那么就进入死循环。 56 loop { 57 spin_loop(); 58 } 59 }; 60 compiler_fence(core::sync::atomic::Ordering::SeqCst); 61 let v = ApStartStackInfo { vaddr }; 62 smp_init_switch_stack(&v); 63 } 64 65 #[naked] 66 unsafe extern "sysv64" fn smp_init_switch_stack(st: &ApStartStackInfo) -> ! { 67 asm!(concat!(" 68 mov rsp, [rdi + {off_rsp}] 69 mov rbp, [rdi + {off_rsp}] 70 jmp {stage1} 71 "), 72 off_rsp = const(offset_of!(ApStartStackInfo, vaddr)), 73 stage1 = sym smp_ap_start_stage1, 74 options(noreturn)); 75 } 76 77 unsafe extern "C" fn smp_ap_start_stage1() -> ! { 78 let id = smp_get_processor_id(); 79 kdebug!("smp_ap_start_stage1: id: {}\n", id.data()); 80 let current_idle = ProcessManager::idle_pcb()[smp_get_processor_id().data() as usize].clone(); 81 82 let tss = TSSManager::current_tss(); 83 84 tss.set_rsp( 85 x86::Ring::Ring0, 86 current_idle.kernel_stack().stack_max_address().data() as u64, 87 ); 88 TSSManager::load_tr(); 89 90 CurrentIrqArch::arch_ap_early_irq_init().expect("arch_ap_early_irq_init failed"); 91 92 smp_ap_start_stage2(); 93 } 94 95 /// 多核的数据 96 #[derive(Debug)] 97 pub struct SmpBootData { 98 initialized: AtomicBool, 99 cpu_count: usize, 100 /// CPU的物理ID（指的是Local APIC ID） 101 /// 102 /// 这里必须保证第0项的是bsp的物理ID 103 phys_id: [usize; PerCpu::MAX_CPU_NUM as usize], 104 } 105 106 #[allow(dead_code)] 107 impl SmpBootData { 108 pub fn cpu_count(&self) -> usize { 109 self.cpu_count 110 } 111 112 /// 获取CPU的物理ID 113 pub fn phys_id(&self, cpu_id: usize) -> usize { 114 self.phys_id[cpu_id] 115 } 116 117 /// 获取BSP的物理ID 118 pub fn bsp_phys_id(&self) -> usize { 119 self.phys_id[0] 120 } 121 122 pub unsafe fn set_cpu_count(&self, cpu_count: u32) { 123 if !self.initialized.load(Ordering::SeqCst) { 124 let p = self as *const SmpBootData as *mut SmpBootData; 125 (*p).cpu_count = cpu_count.try_into().unwrap(); 126 } 127 } 128 129 pub unsafe fn set_phys_id(&self, cpu_id: ProcessorId, phys_id: usize) { 130 if !self.initialized.load(Ordering::SeqCst) { 131 let p = self as *const SmpBootData as *mut SmpBootData; 132 (*p).phys_id[cpu_id.data() as usize] = phys_id; 133 } 134 } 135 136 /// 标记boot data结构体已经初始化完成 137 pub fn mark_initialized(&self) { 138 self.initialized.store(true, Ordering::SeqCst); 139 } 140 } 141 142 pub(super) static SMP_BOOT_DATA: SmpBootData = SmpBootData { 143 initialized: AtomicBool::new(false), 144 cpu_count: 0, 145 phys_id: [0; PerCpu::MAX_CPU_NUM as usize], 146 }; 147 148 #[allow(dead_code)] 149 #[derive(Debug)] 150 pub struct X86_64SmpManager { 151 ia64_cpu_to_sapicid: RwLock<[Option<usize>; PerCpu::MAX_CPU_NUM as usize]>, 152 } 153 154 impl X86_64SmpManager { 155 pub const fn new() -> Self { 156 return Self { 157 ia64_cpu_to_sapicid: RwLock::new([None; PerCpu::MAX_CPU_NUM as usize]), 158 }; 159 } 160 /// initialize the logical cpu number to APIC ID mapping 161 pub fn build_cpu_map(&self) -> Result<(), SystemError> { 162 // 参考：https://code.dragonos.org.cn/xref/linux-6.1.9/arch/ia64/kernel/smpboot.c?fi=smp_build_cpu_map#496 163 // todo!("build_cpu_map") 164 unsafe { 165 smp_cpu_manager().set_possible_cpu(ProcessorId::new(0), true); 166 smp_cpu_manager().set_present_cpu(ProcessorId::new(0), true); 167 smp_cpu_manager().set_online_cpu(ProcessorId::new(0)); 168 } 169 170 for cpu in 1..SMP_BOOT_DATA.cpu_count() { 171 unsafe { 172 smp_cpu_manager().set_possible_cpu(ProcessorId::new(cpu as u32), true); 173 smp_cpu_manager().set_present_cpu(ProcessorId::new(cpu as u32), true); 174 } 175 } 176 177 print_cpus("possible", smp_cpu_manager().possible_cpus()); 178 print_cpus("present", smp_cpu_manager().present_cpus()); 179 return Ok(()); 180 } 181 } 182 183 fn print_cpus(s: &str, mask: &CpuMask) { 184 let mut v = vec![]; 185 for cpu in mask.iter_cpu() { 186 v.push(cpu.data()); 187 } 188 189 kdebug!("{s}: cpus: {v:?}\n"); 190 } 191 192 pub struct X86_64SMPArch; 193 194 impl SMPArch for X86_64SMPArch { 195 #[inline(never)] 196 fn prepare_cpus() -> Result<(), SystemError> { 197 early_acpi_boot_init()?; 198 X86_64_SMP_MANAGER.build_cpu_map()?; 199 return Ok(()); 200 } 201 202 fn post_init() -> Result<(), SystemError> { 203 // AP核心启动完毕，取消低地址映射 204 unsafe { 205 LowAddressRemapping::unmap_at_low_address( 206 &mut IDLE_PROCESS_ADDRESS_SPACE() 207 .write_irqsave() 208 .user_mapper 209 .utable, 210 true, 211 ) 212 } 213 return Ok(()); 214 } 215 216 fn start_cpu(cpu_id: ProcessorId, _cpu_hpstate: &CpuHpCpuState) -> Result<(), SystemError> { 217 kdebug!("start_cpu: cpu_id: {:#x}\n", cpu_id.data()); 218 219 Self::copy_smp_start_code(); 220 221 ipi_send_smp_init(); 222 fence(Ordering::SeqCst); 223 ipi_send_smp_startup(cpu_id)?; 224 fence(Ordering::SeqCst); 225 ipi_send_smp_startup(cpu_id)?; 226 fence(Ordering::SeqCst); 227 228 return Ok(()); 229 } 230 } 231 232 impl X86_64SMPArch { 233 const SMP_CODE_START: usize = 0x20000; 234 /// 复制SMP启动代码到0x20000处 235 fn copy_smp_start_code() -> (VirtAddr, usize) { 236 let apu_boot_size = Self::start_code_size(); 237 238 fence(Ordering::SeqCst); 239 unsafe { 240 core::ptr::copy( 241 _apu_boot_start as *const u8, 242 Self::SMP_CODE_START as *mut u8, 243 apu_boot_size, 244 ) 245 }; 246 fence(Ordering::SeqCst); 247 248 return (VirtAddr::new(Self::SMP_CODE_START), apu_boot_size); 249 } 250 251 fn start_code_size() -> usize { 252 let apu_boot_start = _apu_boot_start as usize; 253 let apu_boot_end = _apu_boot_end as usize; 254 let apu_boot_size = apu_boot_end - apu_boot_start; 255 return apu_boot_size; 256 } 257 } 258 259 impl SmpCpuManager { 260 pub fn arch_init(_boot_cpu: ProcessorId) { 261 assert!(smp_get_processor_id().data() == 0); 262 // 写入APU_START_CR3，这个值会在AP处理器启动时设置到CR3寄存器 263 let addr = IDLE_PROCESS_ADDRESS_SPACE() 264 .read_irqsave() 265 .user_mapper 266 .utable 267 .table() 268 .phys(); 269 let vaddr = unsafe { 270 MMArch::phys_2_virt(PhysAddr::new(&mut __APU_START_CR3 as *mut u64 as usize)).unwrap() 271 }; 272 let ptr = vaddr.data() as *mut u64; 273 unsafe { *ptr = addr.data() as u64 }; 274 275 // 添加低地址映射 276 unsafe { 277 LowAddressRemapping::remap_at_low_address( 278 &mut IDLE_PROCESS_ADDRESS_SPACE() 279 .write_irqsave() 280 .user_mapper 281 .utable, 282 ) 283 }; 284 } 285 } 286