1e2841179SLoGin use core::{intrinsics::unlikely, ops::BitAnd}; 23bc96fa4SLoGin 3e2841179SLoGin use alloc::sync::Arc; 4e2841179SLoGin use system_error::SystemError; 5e2841179SLoGin 6e2841179SLoGin use crate::{ 7e2841179SLoGin arch::{interrupt::TrapFrame, CurrentIrqArch}, 8*0102d69fSLoGin exception::{irqchip::IrqChipFlags, irqdesc::InnerIrqDesc}, 9338f6903SLoGin libs::{once::Once, spinlock::SpinLockGuard}, 10e2841179SLoGin process::{ProcessFlags, ProcessManager}, 11338f6903SLoGin smp::core::smp_get_processor_id, 12e2841179SLoGin }; 133bc96fa4SLoGin 143bc96fa4SLoGin use super::{ 15*0102d69fSLoGin irqchip::IrqChip, 16e2841179SLoGin irqdata::{IrqData, IrqHandlerData, IrqStatus}, 17e2841179SLoGin irqdesc::{ 18e2841179SLoGin InnerIrqAction, IrqDesc, IrqDescState, IrqFlowHandler, IrqReturn, ThreadedHandlerFlags, 19e2841179SLoGin }, 20e2841179SLoGin manage::{irq_manager, IrqManager}, 21e2841179SLoGin InterruptArch, IrqNumber, 223bc96fa4SLoGin }; 233bc96fa4SLoGin 243bc96fa4SLoGin /// 获取用于处理错误的中断的处理程序 253bc96fa4SLoGin #[inline(always)] 263bc96fa4SLoGin pub fn bad_irq_handler() -> &'static dyn IrqFlowHandler { 273bc96fa4SLoGin &HandleBadIrq 283bc96fa4SLoGin } 293bc96fa4SLoGin 30e2841179SLoGin /// 获取用于处理快速EOI的中断的处理程序 31e2841179SLoGin #[inline(always)] 32e2841179SLoGin pub fn fast_eoi_irq_handler() -> &'static dyn IrqFlowHandler { 33e2841179SLoGin &FastEOIIrqHandler 34e2841179SLoGin } 35e2841179SLoGin 36e2841179SLoGin /// 获取用于处理边沿触发中断的处理程序 37e2841179SLoGin #[inline(always)] 38*0102d69fSLoGin #[allow(dead_code)] 39e2841179SLoGin pub fn edge_irq_handler() -> &'static dyn IrqFlowHandler { 40e2841179SLoGin &EdgeIrqHandler 41e2841179SLoGin } 42e2841179SLoGin 433bc96fa4SLoGin /// handle spurious and unhandled irqs 443bc96fa4SLoGin #[derive(Debug)] 453bc96fa4SLoGin struct HandleBadIrq; 463bc96fa4SLoGin 473bc96fa4SLoGin impl IrqFlowHandler for HandleBadIrq { 483bc96fa4SLoGin /// 参考: https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/handle.c?fi=handle_bad_irq#33 49e2841179SLoGin fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) { 503bc96fa4SLoGin // todo: print_irq_desc 513bc96fa4SLoGin // todo: 增加kstat计数 523bc96fa4SLoGin CurrentIrqArch::ack_bad_irq(irq_desc.irq()); 533bc96fa4SLoGin } 543bc96fa4SLoGin } 55e2841179SLoGin 56e2841179SLoGin #[derive(Debug)] 57e2841179SLoGin struct FastEOIIrqHandler; 58e2841179SLoGin 59e2841179SLoGin impl IrqFlowHandler for FastEOIIrqHandler { 60*0102d69fSLoGin /// https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/chip.c?r=&mo=17578&fi=689#689 61*0102d69fSLoGin fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) { 62*0102d69fSLoGin let chip = irq_desc.irq_data().chip_info_read_irqsave().chip(); 63*0102d69fSLoGin 64*0102d69fSLoGin let mut desc_inner = irq_desc.inner(); 65*0102d69fSLoGin let out = |din: SpinLockGuard<InnerIrqDesc>| { 66*0102d69fSLoGin if !chip.flags().contains(IrqChipFlags::IRQCHIP_EOI_IF_HANDLED) { 67*0102d69fSLoGin chip.irq_eoi(din.irq_data()); 68*0102d69fSLoGin } 69*0102d69fSLoGin }; 70*0102d69fSLoGin if !irq_may_run(&desc_inner) { 71*0102d69fSLoGin out(desc_inner); 72*0102d69fSLoGin return; 73*0102d69fSLoGin } 74*0102d69fSLoGin 75*0102d69fSLoGin desc_inner 76*0102d69fSLoGin .internal_state_mut() 77*0102d69fSLoGin .remove(IrqDescState::IRQS_REPLAY | IrqDescState::IRQS_WAITING); 78*0102d69fSLoGin 79*0102d69fSLoGin if desc_inner.actions().is_empty() || desc_inner.common_data().disabled() { 80*0102d69fSLoGin desc_inner 81*0102d69fSLoGin .internal_state_mut() 82*0102d69fSLoGin .insert(IrqDescState::IRQS_PENDING); 83*0102d69fSLoGin mask_irq(desc_inner.irq_data()); 84*0102d69fSLoGin out(desc_inner); 85*0102d69fSLoGin return; 86*0102d69fSLoGin } 87*0102d69fSLoGin 88*0102d69fSLoGin desc_inner = handle_irq_event(irq_desc, desc_inner); 89*0102d69fSLoGin cond_unmask_eoi_irq(&desc_inner, &chip); 90*0102d69fSLoGin 91*0102d69fSLoGin return; 92e2841179SLoGin } 93e2841179SLoGin } 94e2841179SLoGin 95e2841179SLoGin #[derive(Debug)] 96e2841179SLoGin struct EdgeIrqHandler; 97e2841179SLoGin 98e2841179SLoGin impl IrqFlowHandler for EdgeIrqHandler { 99e2841179SLoGin // https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/chip.c?fi=handle_edge_irq#775 100e2841179SLoGin fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) { 101e2841179SLoGin let mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc> = irq_desc.inner(); 102e2841179SLoGin if !irq_may_run(&desc_inner_guard) { 103e2841179SLoGin // kdebug!("!irq_may_run"); 104e2841179SLoGin desc_inner_guard 105e2841179SLoGin .internal_state_mut() 106e2841179SLoGin .insert(IrqDescState::IRQS_PENDING); 107e2841179SLoGin mask_ack_irq(desc_inner_guard.irq_data()); 108e2841179SLoGin return; 109e2841179SLoGin } 110e2841179SLoGin 111e2841179SLoGin if desc_inner_guard.common_data().disabled() { 112e2841179SLoGin // kdebug!("desc_inner_guard.common_data().disabled()"); 113e2841179SLoGin desc_inner_guard 114e2841179SLoGin .internal_state_mut() 115e2841179SLoGin .insert(IrqDescState::IRQS_PENDING); 116e2841179SLoGin mask_ack_irq(desc_inner_guard.irq_data()); 117e2841179SLoGin return; 118e2841179SLoGin } 119e2841179SLoGin 120e2841179SLoGin let irq_data = desc_inner_guard.irq_data().clone(); 121e2841179SLoGin 122e2841179SLoGin irq_data.chip_info_read_irqsave().chip().irq_ack(&irq_data); 123e2841179SLoGin 124e2841179SLoGin loop { 125e2841179SLoGin if unlikely(desc_inner_guard.actions().is_empty()) { 126e2841179SLoGin kdebug!("no action for irq {}", irq_data.irq().data()); 127e2841179SLoGin irq_manager().mask_irq(&irq_data); 128e2841179SLoGin return; 129e2841179SLoGin } 130e2841179SLoGin 131e2841179SLoGin // 当我们在处理一个中断时,如果另一个中断到来,我们本可以屏蔽它. 132e2841179SLoGin // 如果在此期间没有被禁用,请重新启用它。 133e2841179SLoGin if desc_inner_guard 134e2841179SLoGin .internal_state() 135e2841179SLoGin .contains(IrqDescState::IRQS_PENDING) 136e2841179SLoGin { 137e2841179SLoGin let status = desc_inner_guard.common_data().status(); 138b5b571e0SLoGin if !status.disabled() && status.masked() { 139e2841179SLoGin // kdebug!("re-enable irq"); 140e2841179SLoGin irq_manager().unmask_irq(&desc_inner_guard); 141e2841179SLoGin } 142e2841179SLoGin } 143e2841179SLoGin 144e2841179SLoGin // kdebug!("handle_irq_event"); 145e2841179SLoGin 146*0102d69fSLoGin desc_inner_guard = handle_irq_event(irq_desc, desc_inner_guard); 147e2841179SLoGin 148b5b571e0SLoGin if !desc_inner_guard 149e2841179SLoGin .internal_state() 150e2841179SLoGin .contains(IrqDescState::IRQS_PENDING) 151b5b571e0SLoGin || desc_inner_guard.common_data().disabled() 152e2841179SLoGin { 153e2841179SLoGin break; 154e2841179SLoGin } 155e2841179SLoGin } 156e2841179SLoGin } 157e2841179SLoGin } 158e2841179SLoGin 159e2841179SLoGin /// 判断中断是否可以运行 160e2841179SLoGin fn irq_may_run(desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) -> bool { 161e2841179SLoGin let mask = IrqStatus::IRQD_IRQ_INPROGRESS | IrqStatus::IRQD_WAKEUP_ARMED; 162e2841179SLoGin let status = desc_inner_guard.common_data().status(); 163e2841179SLoGin 164e2841179SLoGin // 如果中断不在处理中并且没有被唤醒,则可以运行 165e2841179SLoGin if status.bitand(mask).is_empty() { 166e2841179SLoGin return true; 167e2841179SLoGin } 168e2841179SLoGin 169e2841179SLoGin // todo: 检查其他处理器是否在轮询当前中断 170e2841179SLoGin return false; 171e2841179SLoGin } 172e2841179SLoGin 173338f6903SLoGin pub(super) fn mask_ack_irq(irq_data: &Arc<IrqData>) { 174e2841179SLoGin let chip = irq_data.chip_info_read_irqsave().chip(); 175e2841179SLoGin if chip.can_mask_ack() { 176b5b571e0SLoGin chip.irq_mask_ack(irq_data); 177e2841179SLoGin irq_data.common_data().set_masked(); 178e2841179SLoGin } else { 179e2841179SLoGin irq_manager().mask_irq(irq_data); 180b5b571e0SLoGin chip.irq_ack(irq_data); 181e2841179SLoGin } 182e2841179SLoGin } 183e2841179SLoGin 184*0102d69fSLoGin pub(super) fn mask_irq(irq_data: &Arc<IrqData>) { 185*0102d69fSLoGin if irq_data.common_data().masked() { 186*0102d69fSLoGin return; 187*0102d69fSLoGin } 188*0102d69fSLoGin 189*0102d69fSLoGin let chip = irq_data.chip_info_read_irqsave().chip(); 190*0102d69fSLoGin if chip.irq_mask(irq_data).is_ok() { 191*0102d69fSLoGin irq_data.irqd_set(IrqStatus::IRQD_IRQ_MASKED); 192*0102d69fSLoGin } 193*0102d69fSLoGin } 194*0102d69fSLoGin 195*0102d69fSLoGin pub(super) fn unmask_irq(irq_data: &Arc<IrqData>) { 196*0102d69fSLoGin if !irq_data.common_data().masked() { 197*0102d69fSLoGin return; 198*0102d69fSLoGin } 199*0102d69fSLoGin 200*0102d69fSLoGin let chip = irq_data.chip_info_read_irqsave().chip(); 201*0102d69fSLoGin 202*0102d69fSLoGin if chip.irq_unmask(irq_data).is_ok() { 203*0102d69fSLoGin irq_data.irqd_clear(IrqStatus::IRQD_IRQ_MASKED); 204*0102d69fSLoGin } 205*0102d69fSLoGin } 206*0102d69fSLoGin 207e2841179SLoGin impl IrqManager { 208e2841179SLoGin pub(super) fn do_irq_wake_thread( 209e2841179SLoGin &self, 210e2841179SLoGin desc: &Arc<IrqDesc>, 211e2841179SLoGin action_inner: &mut SpinLockGuard<'_, InnerIrqAction>, 212e2841179SLoGin ) { 213e2841179SLoGin let thread = action_inner.thread(); 214e2841179SLoGin 215e2841179SLoGin if thread.is_none() { 216e2841179SLoGin return; 217e2841179SLoGin } 218e2841179SLoGin 219e2841179SLoGin let thread = thread.unwrap(); 220e2841179SLoGin if thread.flags().contains(ProcessFlags::EXITING) { 221e2841179SLoGin return; 222e2841179SLoGin } 223e2841179SLoGin 224e2841179SLoGin // 如果线程已经在运行,我们不需要唤醒它 225e2841179SLoGin if action_inner 226e2841179SLoGin .thread_flags_mut() 227e2841179SLoGin .test_and_set_bit(ThreadedHandlerFlags::IRQTF_RUNTHREAD) 228e2841179SLoGin { 229e2841179SLoGin return; 230e2841179SLoGin } 231e2841179SLoGin 232e2841179SLoGin desc.inc_threads_active(); 233e2841179SLoGin 234e2841179SLoGin ProcessManager::wakeup(&thread).ok(); 235e2841179SLoGin } 236e2841179SLoGin } 237e2841179SLoGin 238*0102d69fSLoGin fn handle_irq_event<'a>( 239*0102d69fSLoGin irq_desc: &'a Arc<IrqDesc>, 240*0102d69fSLoGin mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc>, 241*0102d69fSLoGin ) -> SpinLockGuard<'a, InnerIrqDesc> { 242*0102d69fSLoGin desc_inner_guard 243*0102d69fSLoGin .internal_state_mut() 244*0102d69fSLoGin .remove(IrqDescState::IRQS_PENDING); 245*0102d69fSLoGin desc_inner_guard.common_data().set_inprogress(); 246*0102d69fSLoGin 247*0102d69fSLoGin drop(desc_inner_guard); 248*0102d69fSLoGin 249*0102d69fSLoGin let _r = do_handle_irq_event(irq_desc); 250*0102d69fSLoGin 251*0102d69fSLoGin let desc_inner_guard = irq_desc.inner(); 252*0102d69fSLoGin desc_inner_guard.common_data().clear_inprogress(); 253*0102d69fSLoGin 254*0102d69fSLoGin return desc_inner_guard; 255*0102d69fSLoGin } 256e2841179SLoGin /// 处理中断事件 257e2841179SLoGin /// 258e2841179SLoGin /// https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/handle.c?fi=handle_irq_event#139 259e2841179SLoGin #[inline(never)] 260e2841179SLoGin fn do_handle_irq_event(desc: &Arc<IrqDesc>) -> Result<(), SystemError> { 261e2841179SLoGin let desc_inner_guard = desc.inner(); 262e2841179SLoGin let irq_data = desc_inner_guard.irq_data().clone(); 263e2841179SLoGin let actions = desc_inner_guard.actions().clone(); 264e2841179SLoGin drop(desc_inner_guard); 265e2841179SLoGin 266e2841179SLoGin let irq = irq_data.irq(); 267e2841179SLoGin let mut r = Ok(IrqReturn::NotHandled); 268e2841179SLoGin 269e2841179SLoGin for action in actions { 270e2841179SLoGin let mut action_inner: SpinLockGuard<'_, InnerIrqAction> = action.inner(); 271e2841179SLoGin // kdebug!("do_handle_irq_event: action: {:?}", action_inner.name()); 272e2841179SLoGin let dynamic_data = action_inner 273e2841179SLoGin .dev_id() 274e2841179SLoGin .clone() 275e2841179SLoGin .map(|d| d as Arc<dyn IrqHandlerData>); 276e2841179SLoGin r = action_inner 277e2841179SLoGin .handler() 278e2841179SLoGin .unwrap() 279e2841179SLoGin .handle(irq, None, dynamic_data); 280e2841179SLoGin 281e2841179SLoGin if let Ok(IrqReturn::WakeThread) = r { 282e2841179SLoGin if unlikely(action_inner.thread_fn().is_none()) { 283e2841179SLoGin warn_no_thread(irq, &mut action_inner); 284e2841179SLoGin } else { 285e2841179SLoGin irq_manager().do_irq_wake_thread(desc, &mut action_inner); 286e2841179SLoGin } 287e2841179SLoGin }; 288e2841179SLoGin } 289e2841179SLoGin 290e2841179SLoGin return r.map(|_| ()); 291e2841179SLoGin } 292e2841179SLoGin 293*0102d69fSLoGin /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/chip.c?r=&mo=17578&fi=659 294*0102d69fSLoGin fn cond_unmask_eoi_irq( 295*0102d69fSLoGin desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, 296*0102d69fSLoGin chip: &Arc<dyn IrqChip>, 297*0102d69fSLoGin ) { 298*0102d69fSLoGin if !desc_inner_guard 299*0102d69fSLoGin .internal_state() 300*0102d69fSLoGin .contains(IrqDescState::IRQS_ONESHOT) 301*0102d69fSLoGin { 302*0102d69fSLoGin chip.irq_eoi(desc_inner_guard.irq_data()); 303*0102d69fSLoGin return; 304*0102d69fSLoGin } 305*0102d69fSLoGin 306*0102d69fSLoGin /* 307*0102d69fSLoGin * We need to unmask in the following cases: 308*0102d69fSLoGin * - Oneshot irq which did not wake the thread (caused by a 309*0102d69fSLoGin * spurious interrupt or a primary handler handling it 310*0102d69fSLoGin * completely). 311*0102d69fSLoGin */ 312*0102d69fSLoGin 313*0102d69fSLoGin if !desc_inner_guard.common_data().disabled() 314*0102d69fSLoGin && desc_inner_guard.common_data().masked() 315*0102d69fSLoGin && desc_inner_guard.threads_oneshot() == 0 316*0102d69fSLoGin { 317*0102d69fSLoGin kdebug!( 318*0102d69fSLoGin "eoi unmask irq {}", 319*0102d69fSLoGin desc_inner_guard.irq_data().irq().data() 320*0102d69fSLoGin ); 321*0102d69fSLoGin chip.irq_eoi(desc_inner_guard.irq_data()); 322*0102d69fSLoGin unmask_irq(desc_inner_guard.irq_data()); 323*0102d69fSLoGin } else if !chip.flags().contains(IrqChipFlags::IRQCHIP_EOI_THREADED) { 324*0102d69fSLoGin kdebug!("eoi irq {}", desc_inner_guard.irq_data().irq().data()); 325*0102d69fSLoGin chip.irq_eoi(desc_inner_guard.irq_data()); 326*0102d69fSLoGin } else { 327*0102d69fSLoGin kwarn!( 328*0102d69fSLoGin "irq {} eoi failed", 329*0102d69fSLoGin desc_inner_guard.irq_data().irq().data() 330*0102d69fSLoGin ); 331*0102d69fSLoGin } 332*0102d69fSLoGin } 333*0102d69fSLoGin 334e2841179SLoGin fn warn_no_thread(irq: IrqNumber, action_inner: &mut SpinLockGuard<'_, InnerIrqAction>) { 335e2841179SLoGin // warn on once 336e2841179SLoGin if action_inner 337e2841179SLoGin .thread_flags_mut() 338e2841179SLoGin .test_and_set_bit(ThreadedHandlerFlags::IRQTF_WARNED) 339e2841179SLoGin { 340e2841179SLoGin return; 341e2841179SLoGin } 342e2841179SLoGin 343e2841179SLoGin kwarn!( 344e2841179SLoGin "irq {}, device {} returned IRQ_WAKE_THREAD, but no threaded handler", 345e2841179SLoGin irq.data(), 346e2841179SLoGin action_inner.name() 347e2841179SLoGin ); 348e2841179SLoGin } 349338f6903SLoGin 350338f6903SLoGin /// `handle_percpu_devid_irq` - 带有per-CPU设备id的perCPU本地中断处理程序 351338f6903SLoGin /// 352338f6903SLoGin /// 353338f6903SLoGin /// * `desc`: 此中断的中断描述结构 354338f6903SLoGin /// 355338f6903SLoGin /// 在没有锁定要求的SMP机器上的每个CPU中断。与linux的`handle_percpu_irq()`相同,但有以下额外内容: 356338f6903SLoGin /// 357338f6903SLoGin /// `action->percpu_dev_id`是一个指向per-cpu变量的指针,这些变量 358338f6903SLoGin /// 包含调用此处理程序的cpu的真实设备id 359338f6903SLoGin #[derive(Debug)] 360338f6903SLoGin pub struct PerCpuDevIdIrqHandler; 361338f6903SLoGin 362338f6903SLoGin impl IrqFlowHandler for PerCpuDevIdIrqHandler { 363338f6903SLoGin fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) { 364338f6903SLoGin let desc_inner_guard = irq_desc.inner(); 365338f6903SLoGin let irq_data = desc_inner_guard.irq_data().clone(); 366338f6903SLoGin let chip = irq_data.chip_info_read().chip(); 367338f6903SLoGin 368338f6903SLoGin chip.irq_ack(&irq_data); 369338f6903SLoGin 370338f6903SLoGin let irq = irq_data.irq(); 371338f6903SLoGin 372338f6903SLoGin let action = desc_inner_guard.actions().first().cloned(); 373338f6903SLoGin 374338f6903SLoGin drop(desc_inner_guard); 375338f6903SLoGin 376338f6903SLoGin if let Some(action) = action { 377338f6903SLoGin let action_inner = action.inner(); 378338f6903SLoGin let per_cpu_devid = action_inner.per_cpu_dev_id().cloned(); 379338f6903SLoGin 380338f6903SLoGin let handler = action_inner.handler().unwrap(); 381338f6903SLoGin drop(action_inner); 382338f6903SLoGin 383338f6903SLoGin let _r = handler.handle( 384338f6903SLoGin irq, 385338f6903SLoGin None, 386338f6903SLoGin per_cpu_devid.map(|d| d as Arc<dyn IrqHandlerData>), 387338f6903SLoGin ); 388338f6903SLoGin } else { 389338f6903SLoGin let cpu = smp_get_processor_id(); 390338f6903SLoGin 391338f6903SLoGin let enabled = irq_desc 392338f6903SLoGin .inner() 393338f6903SLoGin .percpu_enabled() 394338f6903SLoGin .as_ref() 395338f6903SLoGin .unwrap() 396338f6903SLoGin .get(cpu) 397338f6903SLoGin .unwrap_or(false); 398338f6903SLoGin 399338f6903SLoGin if enabled { 400338f6903SLoGin irq_manager().irq_percpu_disable(irq_desc, &irq_data, &chip, cpu); 401338f6903SLoGin } 402338f6903SLoGin static ONCE: Once = Once::new(); 403338f6903SLoGin 404338f6903SLoGin ONCE.call_once(|| { 405338f6903SLoGin kerror!( 406338f6903SLoGin "Spurious percpu irq {} on cpu {:?}, enabled: {}", 407338f6903SLoGin irq.data(), 408338f6903SLoGin cpu, 409338f6903SLoGin enabled 410338f6903SLoGin ); 411338f6903SLoGin }); 412338f6903SLoGin } 413338f6903SLoGin 414338f6903SLoGin chip.irq_eoi(&irq_data); 415338f6903SLoGin } 416338f6903SLoGin } 417