1e2841179SLoGin use core::sync::atomic::AtomicU32;
2e2841179SLoGin
38cb2e9b3SLoGin use alloc::{sync::Arc, vec::Vec};
4*2eab6dd7S曾俊 use log::{debug, error, info};
58cb2e9b3SLoGin use system_error::SystemError;
6338f6903SLoGin
78cb2e9b3SLoGin use crate::{
88cb2e9b3SLoGin arch::CurrentSMPArch,
98cb2e9b3SLoGin libs::cpumask::CpuMask,
108cb2e9b3SLoGin mm::percpu::{PerCpu, PerCpuVar},
118cb2e9b3SLoGin process::{ProcessControlBlock, ProcessManager},
128cb2e9b3SLoGin sched::completion::Completion,
138cb2e9b3SLoGin };
148cb2e9b3SLoGin
158cb2e9b3SLoGin use super::{core::smp_get_processor_id, SMPArch};
16e2841179SLoGin
17e2841179SLoGin int_like!(ProcessorId, AtomicProcessorId, u32, AtomicU32);
18e2841179SLoGin
19e2841179SLoGin impl ProcessorId {
20e2841179SLoGin pub const INVALID: ProcessorId = ProcessorId::new(u32::MAX);
21e2841179SLoGin }
22338f6903SLoGin
23338f6903SLoGin static mut SMP_CPU_MANAGER: Option<SmpCpuManager> = None;
24338f6903SLoGin
25338f6903SLoGin #[inline]
smp_cpu_manager() -> &'static SmpCpuManager26338f6903SLoGin pub fn smp_cpu_manager() -> &'static SmpCpuManager {
27338f6903SLoGin unsafe { SMP_CPU_MANAGER.as_ref().unwrap() }
28338f6903SLoGin }
29338f6903SLoGin
308cb2e9b3SLoGin #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
318cb2e9b3SLoGin pub enum CpuHpState {
328cb2e9b3SLoGin /// 启动阈值
338cb2e9b3SLoGin ThresholdBringUp = 0,
348cb2e9b3SLoGin
358cb2e9b3SLoGin /// 该CPU是离线的
368cb2e9b3SLoGin Offline,
378cb2e9b3SLoGin
388cb2e9b3SLoGin /// 该CPU是在线的
398cb2e9b3SLoGin Online,
408cb2e9b3SLoGin }
418cb2e9b3SLoGin
428cb2e9b3SLoGin /// Per-Cpu Cpu的热插拔状态
438cb2e9b3SLoGin pub struct CpuHpCpuState {
448cb2e9b3SLoGin /// 当前状态
458cb2e9b3SLoGin state: CpuHpState,
468cb2e9b3SLoGin /// 目标状态
478cb2e9b3SLoGin target_state: CpuHpState,
488cb2e9b3SLoGin /// 指向热插拔的线程的PCB
498cb2e9b3SLoGin thread: Option<Arc<ProcessControlBlock>>,
508cb2e9b3SLoGin
518cb2e9b3SLoGin /// 当前是否为启动流程
528cb2e9b3SLoGin bringup: bool,
538cb2e9b3SLoGin /// 启动完成的信号
548cb2e9b3SLoGin comp_done_up: Completion,
558cb2e9b3SLoGin }
568cb2e9b3SLoGin
578cb2e9b3SLoGin impl CpuHpCpuState {
new() -> Self588cb2e9b3SLoGin const fn new() -> Self {
598cb2e9b3SLoGin Self {
608cb2e9b3SLoGin state: CpuHpState::Offline,
618cb2e9b3SLoGin target_state: CpuHpState::Offline,
628cb2e9b3SLoGin thread: None,
638cb2e9b3SLoGin bringup: false,
648cb2e9b3SLoGin comp_done_up: Completion::new(),
658cb2e9b3SLoGin }
668cb2e9b3SLoGin }
678cb2e9b3SLoGin
680102d69fSLoGin #[allow(dead_code)]
thread(&self) -> &Option<Arc<ProcessControlBlock>>698cb2e9b3SLoGin pub const fn thread(&self) -> &Option<Arc<ProcessControlBlock>> {
708cb2e9b3SLoGin &self.thread
718cb2e9b3SLoGin }
728cb2e9b3SLoGin }
738cb2e9b3SLoGin
74338f6903SLoGin pub struct SmpCpuManager {
758cb2e9b3SLoGin /// 可用的CPU
76338f6903SLoGin possible_cpus: CpuMask,
778cb2e9b3SLoGin /// 出现的CPU
788cb2e9b3SLoGin present_cpus: CpuMask,
798cb2e9b3SLoGin /// 出现在系统中的CPU的数量
808cb2e9b3SLoGin present_cnt: AtomicU32,
818cb2e9b3SLoGin /// 可用的CPU的数量
828cb2e9b3SLoGin possible_cnt: AtomicU32,
838cb2e9b3SLoGin /// CPU的状态
848cb2e9b3SLoGin cpuhp_state: PerCpuVar<CpuHpCpuState>,
85338f6903SLoGin }
86338f6903SLoGin
87338f6903SLoGin impl SmpCpuManager {
new() -> Self88338f6903SLoGin fn new() -> Self {
89338f6903SLoGin let possible_cpus = CpuMask::new();
908cb2e9b3SLoGin let present_cpus = CpuMask::new();
918cb2e9b3SLoGin let mut data = Vec::with_capacity(PerCpu::MAX_CPU_NUM as usize);
928cb2e9b3SLoGin for i in 0..PerCpu::MAX_CPU_NUM {
938cb2e9b3SLoGin let mut hpstate = CpuHpCpuState::new();
948cb2e9b3SLoGin hpstate.thread = Some(ProcessManager::idle_pcb()[i as usize].clone());
958cb2e9b3SLoGin data.push(hpstate);
968cb2e9b3SLoGin }
978cb2e9b3SLoGin let cpuhp_state = PerCpuVar::new(data).unwrap();
988cb2e9b3SLoGin
998cb2e9b3SLoGin Self {
1008cb2e9b3SLoGin possible_cpus,
1018cb2e9b3SLoGin present_cpus,
1028cb2e9b3SLoGin cpuhp_state,
1038cb2e9b3SLoGin present_cnt: AtomicU32::new(0),
1048cb2e9b3SLoGin possible_cnt: AtomicU32::new(0),
1058cb2e9b3SLoGin }
106338f6903SLoGin }
107338f6903SLoGin
108338f6903SLoGin /// 设置可用的CPU
109338f6903SLoGin ///
110338f6903SLoGin /// # Safety
111338f6903SLoGin ///
112338f6903SLoGin /// - 该函数不会检查CPU的有效性,调用者需要保证CPU的有效性。
113338f6903SLoGin /// - 由于possible_cpus是一个全局变量,且为了性能考虑,并不会加锁
114338f6903SLoGin /// 访问,因此该函数只能在初始化阶段调用。
set_possible_cpu(&self, cpu: ProcessorId, value: bool)115338f6903SLoGin pub unsafe fn set_possible_cpu(&self, cpu: ProcessorId, value: bool) {
116338f6903SLoGin // 强制获取mut引用,因为该函数只能在初始化阶段调用
117338f6903SLoGin let p = (self as *const Self as *mut Self).as_mut().unwrap();
118338f6903SLoGin
1198cb2e9b3SLoGin if let Some(prev) = p.possible_cpus.set(cpu, value) {
1208cb2e9b3SLoGin if prev != value {
1218cb2e9b3SLoGin if value {
1228cb2e9b3SLoGin p.possible_cnt
1238cb2e9b3SLoGin .fetch_add(1, core::sync::atomic::Ordering::SeqCst);
1248cb2e9b3SLoGin } else {
1258cb2e9b3SLoGin p.possible_cnt
1268cb2e9b3SLoGin .fetch_sub(1, core::sync::atomic::Ordering::SeqCst);
1278cb2e9b3SLoGin }
1288cb2e9b3SLoGin }
1298cb2e9b3SLoGin }
130338f6903SLoGin }
131338f6903SLoGin
132338f6903SLoGin /// 获取可用的CPU
possible_cpus(&self) -> &CpuMask133338f6903SLoGin pub fn possible_cpus(&self) -> &CpuMask {
134338f6903SLoGin &self.possible_cpus
135338f6903SLoGin }
1368cb2e9b3SLoGin
1378cb2e9b3SLoGin #[allow(dead_code)]
possible_cpus_count(&self) -> u321388cb2e9b3SLoGin pub fn possible_cpus_count(&self) -> u32 {
1398cb2e9b3SLoGin self.possible_cnt.load(core::sync::atomic::Ordering::SeqCst)
1408cb2e9b3SLoGin }
1418cb2e9b3SLoGin
present_cpus_count(&self) -> u321428cb2e9b3SLoGin pub fn present_cpus_count(&self) -> u32 {
1438cb2e9b3SLoGin self.present_cnt.load(core::sync::atomic::Ordering::SeqCst)
1448cb2e9b3SLoGin }
1458cb2e9b3SLoGin
set_present_cpu(&self, cpu: ProcessorId, value: bool)1468cb2e9b3SLoGin pub unsafe fn set_present_cpu(&self, cpu: ProcessorId, value: bool) {
1478cb2e9b3SLoGin // 强制获取mut引用,因为该函数只能在初始化阶段调用
1488cb2e9b3SLoGin let p = (self as *const Self as *mut Self).as_mut().unwrap();
1498cb2e9b3SLoGin
1508cb2e9b3SLoGin if let Some(prev) = p.present_cpus.set(cpu, value) {
1518cb2e9b3SLoGin if prev != value {
1528cb2e9b3SLoGin if value {
1538cb2e9b3SLoGin p.present_cnt
1548cb2e9b3SLoGin .fetch_add(1, core::sync::atomic::Ordering::SeqCst);
1558cb2e9b3SLoGin } else {
1568cb2e9b3SLoGin p.present_cnt
1578cb2e9b3SLoGin .fetch_sub(1, core::sync::atomic::Ordering::SeqCst);
1588cb2e9b3SLoGin }
1598cb2e9b3SLoGin }
1608cb2e9b3SLoGin }
1618cb2e9b3SLoGin }
1628cb2e9b3SLoGin
1638cb2e9b3SLoGin /// 获取CPU的状态
cpuhp_state(&self, cpu_id: ProcessorId) -> &CpuHpCpuState1648cb2e9b3SLoGin pub fn cpuhp_state(&self, cpu_id: ProcessorId) -> &CpuHpCpuState {
1658cb2e9b3SLoGin unsafe { self.cpuhp_state.force_get(cpu_id) }
1668cb2e9b3SLoGin }
1678cb2e9b3SLoGin
168b5b571e0SLoGin #[allow(clippy::mut_from_ref)]
cpuhp_state_mut(&self, cpu_id: ProcessorId) -> &mut CpuHpCpuState1698cb2e9b3SLoGin fn cpuhp_state_mut(&self, cpu_id: ProcessorId) -> &mut CpuHpCpuState {
1708cb2e9b3SLoGin unsafe { self.cpuhp_state.force_get_mut(cpu_id) }
1718cb2e9b3SLoGin }
1728cb2e9b3SLoGin
1738cb2e9b3SLoGin /// 设置CPU的状态, 返回旧的状态
set_cpuhp_state( &self, cpu_id: ProcessorId, target_state: CpuHpState, ) -> CpuHpState1748cb2e9b3SLoGin pub unsafe fn set_cpuhp_state(
1758cb2e9b3SLoGin &self,
1768cb2e9b3SLoGin cpu_id: ProcessorId,
1778cb2e9b3SLoGin target_state: CpuHpState,
1788cb2e9b3SLoGin ) -> CpuHpState {
1798cb2e9b3SLoGin let p = self.cpuhp_state.force_get_mut(cpu_id);
1808cb2e9b3SLoGin let old_state = p.state;
1818cb2e9b3SLoGin
1828cb2e9b3SLoGin let bringup = target_state > p.state;
1838cb2e9b3SLoGin p.target_state = target_state;
1848cb2e9b3SLoGin p.bringup = bringup;
1858cb2e9b3SLoGin
1868cb2e9b3SLoGin return old_state;
1878cb2e9b3SLoGin }
1888cb2e9b3SLoGin
set_online_cpu(&self, cpu_id: ProcessorId)1898cb2e9b3SLoGin pub fn set_online_cpu(&self, cpu_id: ProcessorId) {
1908cb2e9b3SLoGin unsafe { self.set_cpuhp_state(cpu_id, CpuHpState::Online) };
1918cb2e9b3SLoGin }
1928cb2e9b3SLoGin
1938cb2e9b3SLoGin /// 获取出现在系统中的CPU
1948cb2e9b3SLoGin #[allow(dead_code)]
present_cpus(&self) -> &CpuMask1958cb2e9b3SLoGin pub fn present_cpus(&self) -> &CpuMask {
1968cb2e9b3SLoGin &self.present_cpus
1978cb2e9b3SLoGin }
1988cb2e9b3SLoGin
1998cb2e9b3SLoGin /// 启动bsp以外的CPU
bringup_nonboot_cpus(&self)2008cb2e9b3SLoGin pub(super) fn bringup_nonboot_cpus(&self) {
2018cb2e9b3SLoGin for cpu_id in self.present_cpus().iter_cpu() {
2028cb2e9b3SLoGin if cpu_id == smp_get_processor_id() {
2038cb2e9b3SLoGin continue;
2048cb2e9b3SLoGin }
2058cb2e9b3SLoGin
206*2eab6dd7S曾俊 debug!("Bring up CPU {}", cpu_id.data());
2078cb2e9b3SLoGin
2088cb2e9b3SLoGin if let Err(e) = self.cpu_up(cpu_id, CpuHpState::Online) {
209*2eab6dd7S曾俊 error!("Failed to bring up CPU {}: {:?}", cpu_id.data(), e);
2108cb2e9b3SLoGin }
2118cb2e9b3SLoGin }
2128cb2e9b3SLoGin
213*2eab6dd7S曾俊 info!("All non-boot CPUs have been brought up");
2148cb2e9b3SLoGin }
2158cb2e9b3SLoGin
cpu_up(&self, cpu_id: ProcessorId, target_state: CpuHpState) -> Result<(), SystemError>2168cb2e9b3SLoGin fn cpu_up(&self, cpu_id: ProcessorId, target_state: CpuHpState) -> Result<(), SystemError> {
2178cb2e9b3SLoGin if !self.possible_cpus().get(cpu_id).unwrap_or(false) {
2188cb2e9b3SLoGin return Err(SystemError::EINVAL);
2198cb2e9b3SLoGin }
2208cb2e9b3SLoGin
2218cb2e9b3SLoGin let cpu_state = self.cpuhp_state(cpu_id).state;
222*2eab6dd7S曾俊 debug!(
2238cb2e9b3SLoGin "cpu_up: cpu_id: {}, cpu_state: {:?}, target_state: {:?}",
2248cb2e9b3SLoGin cpu_id.data(),
2258cb2e9b3SLoGin cpu_state,
2268cb2e9b3SLoGin target_state
2278cb2e9b3SLoGin );
2288cb2e9b3SLoGin // 如果CPU的状态已经达到或者超过目标状态,则直接返回
2298cb2e9b3SLoGin if cpu_state >= target_state {
2308cb2e9b3SLoGin return Ok(());
2318cb2e9b3SLoGin }
2328cb2e9b3SLoGin
2338cb2e9b3SLoGin unsafe { self.set_cpuhp_state(cpu_id, target_state) };
2348cb2e9b3SLoGin let cpu_state = self.cpuhp_state(cpu_id).state;
2358cb2e9b3SLoGin if cpu_state > CpuHpState::ThresholdBringUp {
2368cb2e9b3SLoGin self.cpuhp_kick_ap(cpu_id, target_state)?;
2378cb2e9b3SLoGin }
2388cb2e9b3SLoGin
2398cb2e9b3SLoGin return Ok(());
2408cb2e9b3SLoGin }
2418cb2e9b3SLoGin
cpuhp_kick_ap( &self, cpu_id: ProcessorId, target_state: CpuHpState, ) -> Result<(), SystemError>2428cb2e9b3SLoGin fn cpuhp_kick_ap(
2438cb2e9b3SLoGin &self,
2448cb2e9b3SLoGin cpu_id: ProcessorId,
2458cb2e9b3SLoGin target_state: CpuHpState,
2468cb2e9b3SLoGin ) -> Result<(), SystemError> {
2478cb2e9b3SLoGin let prev_state = unsafe { self.set_cpuhp_state(cpu_id, target_state) };
2488cb2e9b3SLoGin let hpstate = self.cpuhp_state_mut(cpu_id);
2498cb2e9b3SLoGin if let Err(e) = self.do_cpuhp_kick_ap(hpstate) {
2508cb2e9b3SLoGin self.cpuhp_reset_state(hpstate, prev_state);
2518cb2e9b3SLoGin self.do_cpuhp_kick_ap(hpstate).ok();
2528cb2e9b3SLoGin
2538cb2e9b3SLoGin return Err(e);
2548cb2e9b3SLoGin }
2558cb2e9b3SLoGin
2568cb2e9b3SLoGin return Ok(());
2578cb2e9b3SLoGin }
2588cb2e9b3SLoGin
do_cpuhp_kick_ap(&self, cpu_state: &mut CpuHpCpuState) -> Result<(), SystemError>2598cb2e9b3SLoGin fn do_cpuhp_kick_ap(&self, cpu_state: &mut CpuHpCpuState) -> Result<(), SystemError> {
2608cb2e9b3SLoGin let pcb = cpu_state.thread.as_ref().ok_or(SystemError::EINVAL)?;
2618cb2e9b3SLoGin let cpu_id = pcb.sched_info().on_cpu().ok_or(SystemError::EINVAL)?;
2628cb2e9b3SLoGin
2638cb2e9b3SLoGin // todo: 等待CPU启动完成
2648cb2e9b3SLoGin
2658cb2e9b3SLoGin ProcessManager::wakeup(cpu_state.thread.as_ref().unwrap())?;
2663959e94dS曾俊
2678cb2e9b3SLoGin CurrentSMPArch::start_cpu(cpu_id, cpu_state)?;
2688cb2e9b3SLoGin assert_eq!(ProcessManager::current_pcb().preempt_count(), 0);
2698cb2e9b3SLoGin self.wait_for_ap_thread(cpu_state, cpu_state.bringup);
2708cb2e9b3SLoGin
2718cb2e9b3SLoGin return Ok(());
2728cb2e9b3SLoGin }
2738cb2e9b3SLoGin
wait_for_ap_thread(&self, cpu_state: &mut CpuHpCpuState, bringup: bool)2748cb2e9b3SLoGin fn wait_for_ap_thread(&self, cpu_state: &mut CpuHpCpuState, bringup: bool) {
2758cb2e9b3SLoGin if bringup {
2763959e94dS曾俊 cpu_state
2773959e94dS曾俊 .comp_done_up
2783959e94dS曾俊 .wait_for_completion()
2793959e94dS曾俊 .expect("failed to wait ap thread");
2808cb2e9b3SLoGin } else {
2818cb2e9b3SLoGin todo!("wait_for_ap_thread")
2828cb2e9b3SLoGin }
2838cb2e9b3SLoGin }
2848cb2e9b3SLoGin
2858cb2e9b3SLoGin /// 完成AP的启动
complete_ap_thread(&self, bringup: bool)2868cb2e9b3SLoGin pub fn complete_ap_thread(&self, bringup: bool) {
2878cb2e9b3SLoGin let cpu_id = smp_get_processor_id();
2888cb2e9b3SLoGin let cpu_state = self.cpuhp_state_mut(cpu_id);
2898cb2e9b3SLoGin if bringup {
2908cb2e9b3SLoGin cpu_state.comp_done_up.complete();
2918cb2e9b3SLoGin } else {
2928cb2e9b3SLoGin todo!("complete_ap_thread")
2938cb2e9b3SLoGin }
2948cb2e9b3SLoGin }
2958cb2e9b3SLoGin
cpuhp_reset_state(&self, st: &mut CpuHpCpuState, prev_state: CpuHpState)2968cb2e9b3SLoGin fn cpuhp_reset_state(&self, st: &mut CpuHpCpuState, prev_state: CpuHpState) {
2978cb2e9b3SLoGin let bringup = !st.bringup;
2988cb2e9b3SLoGin st.target_state = prev_state;
2998cb2e9b3SLoGin
3008cb2e9b3SLoGin st.bringup = bringup;
3018cb2e9b3SLoGin }
302338f6903SLoGin }
303338f6903SLoGin
smp_cpu_manager_init(boot_cpu: ProcessorId)304338f6903SLoGin pub fn smp_cpu_manager_init(boot_cpu: ProcessorId) {
305338f6903SLoGin unsafe {
306338f6903SLoGin SMP_CPU_MANAGER = Some(SmpCpuManager::new());
307338f6903SLoGin }
308338f6903SLoGin
309338f6903SLoGin unsafe { smp_cpu_manager().set_possible_cpu(boot_cpu, true) };
3100102d69fSLoGin unsafe { smp_cpu_manager().set_present_cpu(boot_cpu, true) };
311338f6903SLoGin
312338f6903SLoGin SmpCpuManager::arch_init(boot_cpu);
313338f6903SLoGin }
314