1 use core::ops::{BitXor, Deref, DerefMut}; 2 3 use alloc::{string::String, sync::Arc}; 4 5 use log::{debug, error, warn}; 6 use system_error::SystemError; 7 8 use crate::{ 9 driver::base::device::DeviceId, 10 exception::{ 11 irqchip::IrqChipSetMaskResult, 12 irqdesc::{irq_desc_manager, InnerIrqDesc, IrqAction}, 13 }, 14 libs::{cpumask::CpuMask, spinlock::SpinLockGuard}, 15 process::{kthread::KernelThreadMechanism, ProcessManager}, 16 smp::cpu::ProcessorId, 17 }; 18 19 use super::{ 20 dummychip::no_irq_chip, 21 irqchip::IrqChipFlags, 22 irqdata::{IrqData, IrqHandlerData, IrqLineStatus, IrqStatus}, 23 irqdesc::{InnerIrqAction, IrqDesc, IrqDescState, IrqHandleFlags, IrqHandler, IrqReturn}, 24 irqdomain::irq_domain_manager, 25 IrqNumber, 26 }; 27 28 lazy_static! { 29 /// 默认的中断亲和性 30 static ref IRQ_DEFAULT_AFFINITY: CpuMask = { 31 let mut mask = CpuMask::new(); 32 // 默认情况下，中断处理程序将在第一个处理器上运行 33 mask.set(ProcessorId::new(0), true); 34 mask 35 }; 36 } 37 38 pub fn irq_manager() -> &'static IrqManager { 39 &IrqManager 40 } 41 42 /// 中断管理器 43 pub struct IrqManager; 44 45 impl IrqManager { 46 pub const IRQ_RESEND: bool = true; 47 #[allow(dead_code)] 48 pub const IRQ_NORESEND: bool = false; 49 #[allow(dead_code)] 50 pub const IRQ_START_FORCE: bool = true; 51 pub const IRQ_START_COND: bool = false; 52 53 /// 在中断线上添加一个处理函数 54 /// 55 /// ## 参数 56 /// 57 /// - irq: 虚拟中断号(中断线号) 58 /// - name: 生成该中断的设备名称 59 /// - handler: 中断处理函数 60 /// - flags: 中断处理标志 61 /// - dev_id: 一个用于标识设备的cookie 62 pub fn request_irq( 63 &self, 64 irq: IrqNumber, 65 name: String, 66 handler: &'static dyn IrqHandler, 67 flags: IrqHandleFlags, 68 dev_id: Option<Arc<DeviceId>>, 69 ) -> Result<(), SystemError> { 70 return self.request_threaded_irq(irq, Some(handler), None, flags, name, dev_id); 71 } 72 73 /// 在中断线上添加一个处理函数（可以是线程化的中断） 74 /// 75 /// ## 参数 76 /// 77 /// - irq: 虚拟中断号 78 /// - handler: 当中断发生时将被调用的函数，是 79 /// 线程化中断的初级处理程序。如果handler为`None`并且thread_fn不为`None`， 80 /// 将安装默认的初级处理程序 81 /// - thread_fn: 在中断处理程序线程中调用的函数. 如果为`None`，则不会创建irq线程 82 /// - flags: 中断处理标志 83 /// - IRQF_SHARED: 中断是共享的 84 /// - IRQF_TRIGGER*: 指定中断触发方式 85 /// - IRQF_ONESHOT: 在thread_fn中运行时，中断线被遮蔽 86 /// - dev_name: 生成该中断的设备名称 87 /// - dev_id: 一个用于标识设备的cookie 88 /// 89 /// ## 说明 90 /// 91 /// 此调用分配中断资源并启用中断线和IRQ处理。 92 /// 从这一点开始，您的处理程序函数可能会被调用。 93 /// 因此，您必须确保首先初始化您的硬件， 94 /// 并确保以正确的顺序设置中断处理程序。 95 /// 96 /// 如果您想为您的设备设置线程化中断处理程序 97 /// 则需要提供@handler和@thread_fn。@handler仍然 98 /// 在硬中断上下文中调用，并且必须检查 99 /// 中断是否来自设备。如果是，它需要禁用设备上的中断 100 /// 并返回IRQ_WAKE_THREAD，这将唤醒处理程序线程并运行 101 /// @thread_fn。这种拆分处理程序设计是为了支持 102 /// 共享中断。 103 /// 104 /// dev_id必须是全局唯一的。通常使用设备数据结构的地址或者uuid 105 /// 作为cookie。由于处理程序接收这个值，因此使用它是有意义的。 106 /// 107 /// 如果您的中断是共享的，您必须传递一个非NULL的dev_id 108 /// 因为当释放中断时需要它。 109 pub fn request_threaded_irq( 110 &self, 111 irq: IrqNumber, 112 mut handler: Option<&'static dyn IrqHandler>, 113 thread_fn: Option<&'static dyn IrqHandler>, 114 flags: IrqHandleFlags, 115 dev_name: String, 116 dev_id: Option<Arc<DeviceId>>, 117 ) -> Result<(), SystemError> { 118 if irq == IrqNumber::IRQ_NOTCONNECTED { 119 return Err(SystemError::ENOTCONN); 120 } 121 122 // 逻辑检查：共享中断必须传入一个真正的设备ID， 123 // 否则后来我们将难以确定哪个中断是哪个（会搞乱中断释放逻辑等）。 124 // 此外，共享中断与禁用自动使能不相符。 共享中断可能在仍然禁用时请求它，然后永远等待中断。 125 // 另外，IRQF_COND_SUSPEND 仅适用于共享中断，并且它不能与 IRQF_NO_SUSPEND 同时设置。 126 127 if ((flags.contains(IrqHandleFlags::IRQF_SHARED)) && dev_id.is_none()) 128 || ((flags.contains(IrqHandleFlags::IRQF_SHARED)) 129 && (flags.contains(IrqHandleFlags::IRQF_NO_AUTOEN))) 130 || (!(flags.contains(IrqHandleFlags::IRQF_SHARED)) 131 && (flags.contains(IrqHandleFlags::IRQF_COND_SUSPEND))) 132 || ((flags.contains(IrqHandleFlags::IRQF_NO_SUSPEND)) 133 && (flags.contains(IrqHandleFlags::IRQF_COND_SUSPEND))) 134 { 135 return Err(SystemError::EINVAL); 136 } 137 let desc = irq_desc_manager().lookup(irq).ok_or(SystemError::EINVAL)?; 138 if !desc.can_request() { 139 warn!("desc {} can not request", desc.irq().data()); 140 return Err(SystemError::EINVAL); 141 } 142 143 if handler.is_none() { 144 if thread_fn.is_none() { 145 // 不允许中断处理函数和线程处理函数都为空 146 return Err(SystemError::EINVAL); 147 } 148 149 // 如果中断处理函数为空，线程处理函数不为空，则使用默认的中断处理函数 150 handler = Some(&DefaultPrimaryIrqHandler); 151 } 152 153 let irqaction = IrqAction::new(irq, dev_name, handler, thread_fn); 154 155 let mut action_guard = irqaction.inner(); 156 *action_guard.flags_mut() = flags; 157 *action_guard.dev_id_mut() = dev_id; 158 drop(action_guard); 159 debug!("to inner_setup_irq"); 160 return self.inner_setup_irq(irq, irqaction, desc); 161 } 162 163 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c?r=&mo=59252&fi=2138#1497 164 #[inline(never)] 165 fn inner_setup_irq( 166 &self, 167 irq: IrqNumber, 168 action: Arc<IrqAction>, 169 desc: Arc<IrqDesc>, 170 ) -> Result<(), SystemError> { 171 // ==== 定义错误处理函数 ==== 172 let err_out_thread = 173 |e: SystemError, mut action_guard: SpinLockGuard<'_, InnerIrqAction>| -> SystemError { 174 if let Some(thread_pcb) = action_guard.thread() { 175 action_guard.set_thread(None); 176 KernelThreadMechanism::stop(&thread_pcb).ok(); 177 } 178 179 if let Some(secondary) = action_guard.secondary() { 180 let mut secondary_guard = secondary.inner(); 181 if let Some(thread_pcb) = secondary_guard.thread() { 182 secondary_guard.set_thread(None); 183 KernelThreadMechanism::stop(&thread_pcb).ok(); 184 } 185 } 186 return e; 187 }; 188 189 let err_out_bus_unlock = |e: SystemError, 190 desc: Arc<IrqDesc>, 191 req_mutex_guard: crate::libs::mutex::MutexGuard<'_, ()>, 192 action_guard: SpinLockGuard<'_, InnerIrqAction>| 193 -> SystemError { 194 desc.chip_bus_sync_unlock(); 195 drop(req_mutex_guard); 196 return err_out_thread(e, action_guard); 197 }; 198 199 let err_out_unlock = |e: SystemError, 200 desc_guard: SpinLockGuard<'_, InnerIrqDesc>, 201 desc: Arc<IrqDesc>, 202 req_mutex_guard: crate::libs::mutex::MutexGuard<'_, ()>, 203 action_guard: SpinLockGuard<'_, InnerIrqAction>| 204 -> SystemError { 205 drop(desc_guard); 206 return err_out_bus_unlock(e, desc, req_mutex_guard, action_guard); 207 }; 208 209 let err_out_mismatch = |old_action_guard: SpinLockGuard<'_, InnerIrqAction>, 210 desc_guard: SpinLockGuard<'_, InnerIrqDesc>, 211 action_guard: SpinLockGuard<'_, InnerIrqAction>, 212 desc: Arc<IrqDesc>, 213 req_mutex_guard: crate::libs::mutex::MutexGuard<'_, ()>| 214 -> SystemError { 215 if !action_guard 216 .flags() 217 .contains(IrqHandleFlags::IRQF_PROBE_SHARED) 218 { 219 error!("Flags mismatch for irq {} (name: {}, flags: {:?}). old action name: {}, old flags: {:?}", irq.data(), action_guard.name(), action_guard.flags(), old_action_guard.name(), old_action_guard.flags()); 220 } 221 return err_out_unlock( 222 SystemError::EBUSY, 223 desc_guard, 224 desc, 225 req_mutex_guard, 226 action_guard, 227 ); 228 }; 229 230 // ===== 代码开始 ===== 231 232 if Arc::ptr_eq( 233 &desc.irq_data().chip_info_read_irqsave().chip(), 234 &no_irq_chip(), 235 ) { 236 return Err(SystemError::ENOSYS); 237 } 238 239 let mut action_guard = action.inner(); 240 if !action_guard.flags().trigger_type_specified() { 241 // 如果没有指定触发类型，则使用默认的触发类型 242 action_guard 243 .flags_mut() 244 .insert_trigger_type(desc.irq_data().common_data().trigger_type()) 245 } 246 247 let nested = desc.nested_thread(); 248 249 if nested { 250 if action_guard.thread_fn().is_none() { 251 return Err(SystemError::EINVAL); 252 } 253 254 action_guard.set_handler(Some(&IrqNestedPrimaryHandler)); 255 } else if desc.can_thread() { 256 self.setup_forced_threading(action_guard.deref_mut())?; 257 } 258 259 // 如果具有中断线程处理程序，并且中断不是嵌套的，则设置中断线程 260 if action_guard.thread_fn().is_some() && !nested { 261 self.setup_irq_thread(irq, action_guard.deref(), false)?; 262 263 if let Some(secondary) = action_guard.secondary() { 264 let secondary_guard = secondary.inner(); 265 if let Err(e) = self.setup_irq_thread(irq, secondary_guard.deref(), true) { 266 return Err(err_out_thread(e, action_guard)); 267 } 268 } 269 } 270 271 // Drivers are often written to work w/o knowledge about the 272 // underlying irq chip implementation, so a request for a 273 // threaded irq without a primary hard irq context handler 274 // requires the ONESHOT flag to be set. Some irq chips like 275 // MSI based interrupts are per se one shot safe. Check the 276 // chip flags, so we can avoid the unmask dance at the end of 277 // the threaded handler for those. 278 279 if desc 280 .irq_data() 281 .chip_info_read_irqsave() 282 .chip() 283 .flags() 284 .contains(IrqChipFlags::IRQCHIP_ONESHOT_SAFE) 285 { 286 *action_guard.flags_mut() &= !IrqHandleFlags::IRQF_ONESHOT; 287 } 288 289 // Protects against a concurrent __free_irq() call which might wait 290 // for synchronize_hardirq() to complete without holding the optional 291 // chip bus lock and desc->lock. Also protects against handing out 292 // a recycled oneshot thread_mask bit while it's still in use by 293 // its previous owner. 294 let req_mutex_guard = desc.request_mutex_lock(); 295 296 // Acquire bus lock as the irq_request_resources() callback below 297 // might rely on the serialization or the magic power management 298 // functions which are abusing the irq_bus_lock() callback, 299 desc.chip_bus_lock(); 300 301 // 如果当前中断线上还没有irqaction, 则先为中断线申请资源 302 if desc.actions().is_empty() { 303 if let Err(e) = self.irq_request_resources(desc.clone()) { 304 error!( 305 "Failed to request resources for {} (irq {}) on irqchip {}, error {:?}", 306 action_guard.name(), 307 irq.data(), 308 desc.irq_data().chip_info_read_irqsave().chip().name(), 309 e 310 ); 311 return Err(err_out_bus_unlock( 312 e, 313 desc.clone(), 314 req_mutex_guard, 315 action_guard, 316 )); 317 } 318 } 319 320 let mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc> = desc.inner(); 321 322 // 标记当前irq是否是共享的 323 let mut irq_shared = false; 324 if !desc_inner_guard.actions().is_empty() { 325 // 除非双方都同意并且是相同类型（级别、边沿、极性），否则不能共享中断。 326 // 因此，两个标志字段都必须设置IRQF_SHARED，并且设置触发类型的位必须匹配。 327 // 另外，所有各方都必须就ONESHOT达成一致。 328 // NMI用途的中断线不能共享。 329 if desc_inner_guard 330 .internal_state() 331 .contains(IrqDescState::IRQS_NMI) 332 { 333 error!( 334 "Invalid attempt to share NMI for {} (irq {}) on irqchip {}", 335 action_guard.name(), 336 irq.data(), 337 desc_inner_guard 338 .irq_data() 339 .chip_info_read_irqsave() 340 .chip() 341 .name() 342 ); 343 return Err(err_out_unlock( 344 SystemError::EINVAL, 345 desc_inner_guard, 346 desc.clone(), 347 req_mutex_guard, 348 action_guard, 349 )); 350 } 351 352 let irq_data = desc_inner_guard.irq_data(); 353 354 let old_trigger_type: super::irqdata::IrqLineStatus; 355 let status = irq_data.common_data().status(); 356 if status.trigger_type_was_set() { 357 old_trigger_type = status.trigger_type(); 358 } else { 359 old_trigger_type = action_guard.flags().trigger_type(); 360 irq_data.common_data().set_trigger_type(old_trigger_type); 361 } 362 363 let old = &desc_inner_guard.actions()[0].clone(); 364 let old_guard = old.inner(); 365 366 if (!(old_guard 367 .flags() 368 .intersection(*action_guard.flags()) 369 .contains(IrqHandleFlags::IRQF_SHARED))) 370 || (old_trigger_type != (action_guard.flags().trigger_type())) 371 || ((old_guard.flags().bitxor(*action_guard.flags())) 372 .contains(IrqHandleFlags::IRQF_ONESHOT)) 373 { 374 debug!( 375 "Flags mismatch for irq {} (name: {}, flags: {:?}). old action name: {}, old flags: {:?}", 376 irq.data(), 377 action_guard.name(), 378 action_guard.flags(), 379 old_guard.name(), 380 old_guard.flags() 381 ); 382 383 return Err(err_out_mismatch( 384 old_guard, 385 desc_inner_guard, 386 action_guard, 387 desc.clone(), 388 req_mutex_guard, 389 )); 390 } 391 392 // all handlers must agree on per-cpuness 393 if *old_guard.flags() & IrqHandleFlags::IRQF_PERCPU 394 != *action_guard.flags() & IrqHandleFlags::IRQF_PERCPU 395 { 396 debug!( 397 "Per-cpu mismatch for irq {} (name: {}, flags: {:?})", 398 irq.data(), 399 action_guard.name(), 400 action_guard.flags() 401 ); 402 return Err(err_out_mismatch( 403 old_guard, 404 desc_inner_guard, 405 action_guard, 406 desc.clone(), 407 req_mutex_guard, 408 )); 409 } 410 411 irq_shared = true; 412 } 413 414 if action_guard.flags().contains(IrqHandleFlags::IRQF_ONESHOT) { 415 // todo: oneshot 416 } else if action_guard.handler().is_some_and(|h| { 417 h.type_id() == (&DefaultPrimaryIrqHandler as &dyn IrqHandler).type_id() 418 }) && !desc_inner_guard 419 .irq_data() 420 .chip_info_read_irqsave() 421 .chip() 422 .flags() 423 .contains(IrqChipFlags::IRQCHIP_ONESHOT_SAFE) 424 { 425 // 请求中断时 hander = NULL，因此我们为其使用默认的主处理程序。 426 // 但它没有设置ONESHOT标志。与电平触发中断结合时， 427 // 这是致命的，因为默认的主处理程序只是唤醒线程，然后重新启用 irq 线路， 428 // 但设备仍然保持电平中断生效。周而复始.... 429 // 虽然这对于边缘类型中断来说可行，但我们为了安全起见，不加条件地拒绝， 430 // 因为我们不能确定这个中断实际上具有什么类型。 431 // 由于底层芯片实现可能会覆盖它们，所以类型标志并不可靠. 432 433 error!( 434 "Requesting irq {} without a handler, and ONESHOT flags not set for irqaction: {}", 435 irq.data(), 436 action_guard.name() 437 ); 438 return Err(err_out_unlock( 439 SystemError::EINVAL, 440 desc_inner_guard, 441 desc.clone(), 442 req_mutex_guard, 443 action_guard, 444 )); 445 } 446 447 // 第一次在当前irqdesc上注册中断处理函数 448 if !irq_shared { 449 // 设置中断触发方式 450 if action_guard.flags().trigger_type_specified() { 451 let trigger_type = action_guard.flags().trigger_type(); 452 if let Err(e) = 453 self.do_set_irq_trigger(desc.clone(), &mut desc_inner_guard, trigger_type) 454 { 455 debug!( 456 "Failed to set trigger type for irq {} (name: {}, flags: {:?}), error {:?}", 457 irq.data(), 458 action_guard.name(), 459 action_guard.flags(), 460 e 461 ); 462 return Err(err_out_unlock( 463 e, 464 desc_inner_guard, 465 desc.clone(), 466 req_mutex_guard, 467 action_guard, 468 )); 469 } 470 } 471 debug!("to irq_activate"); 472 // 激活中断。这种激活必须独立于IRQ_NOAUTOEN进行*desc_inner_guard.internal_state_mut() |= IrqDescState::IRQS_NOREQUEST;uest. 473 if let Err(e) = self.irq_activate(&desc, &mut desc_inner_guard) { 474 debug!( 475 "Failed to activate irq {} (name: {}, flags: {:?}), error {:?}", 476 irq.data(), 477 action_guard.name(), 478 action_guard.flags(), 479 e 480 ); 481 return Err(err_out_unlock( 482 e, 483 desc_inner_guard, 484 desc.clone(), 485 req_mutex_guard, 486 action_guard, 487 )); 488 } 489 490 *desc_inner_guard.internal_state_mut() &= !(IrqDescState::IRQS_AUTODETECT 491 | IrqDescState::IRQS_SPURIOUS_DISABLED 492 | IrqDescState::IRQS_ONESHOT 493 | IrqDescState::IRQS_WAITING); 494 desc_inner_guard 495 .common_data() 496 .clear_status(IrqStatus::IRQD_IRQ_INPROGRESS); 497 498 if action_guard.flags().contains(IrqHandleFlags::IRQF_PERCPU) { 499 desc_inner_guard 500 .common_data() 501 .insert_status(IrqStatus::IRQD_PER_CPU); 502 desc_inner_guard.line_status_set_per_cpu(); 503 504 if action_guard.flags().contains(IrqHandleFlags::IRQF_NO_DEBUG) { 505 desc_inner_guard.line_status_set_no_debug(); 506 } 507 } 508 509 if action_guard.flags().contains(IrqHandleFlags::IRQF_ONESHOT) { 510 *desc_inner_guard.internal_state_mut() |= IrqDescState::IRQS_ONESHOT; 511 } 512 513 // 如果有要求的话，则忽略IRQ的均衡。 514 if action_guard 515 .flags() 516 .contains(IrqHandleFlags::IRQF_NOBALANCING) 517 { 518 todo!("IRQF_NO_BALANCING"); 519 } 520 521 if !action_guard 522 .flags() 523 .contains(IrqHandleFlags::IRQF_NO_AUTOEN) 524 && desc_inner_guard.can_autoenable() 525 { 526 // 如果没有设置IRQF_NOAUTOEN，则自动使能中断 527 self.irq_startup( 528 &desc, 529 &mut desc_inner_guard, 530 Self::IRQ_RESEND, 531 Self::IRQ_START_COND, 532 ) 533 .ok(); 534 } else { 535 // 共享中断与禁用自动使能不太兼容。 536 // 共享中断可能在它仍然被禁用时请求它，然后永远等待中断。 537 538 static mut WARNED: bool = false; 539 if action_guard.flags().contains(IrqHandleFlags::IRQF_SHARED) && unsafe { !WARNED } 540 { 541 warn!( 542 "Shared interrupt {} for {} requested but not auto enabled", 543 irq.data(), 544 action_guard.name() 545 ); 546 unsafe { WARNED = true }; 547 } 548 549 desc_inner_guard.set_depth(1); 550 } 551 } else if action_guard.flags().trigger_type_specified() { 552 let new_trigger_type = action_guard.flags().trigger_type(); 553 let old_trigger_type = desc_inner_guard.common_data().trigger_type(); 554 if new_trigger_type != old_trigger_type { 555 warn!("Irq {} uses trigger type: {old_trigger_type:?}, but requested trigger type: {new_trigger_type:?}.", irq.data()); 556 } 557 } 558 559 // 在队列末尾添加新的irqaction 560 desc_inner_guard.add_action(action.clone()); 561 562 // 检查我们是否曾经通过虚构的中断处理程序禁用过irq。重新启用它并再给它一次机会。 563 if irq_shared 564 && desc_inner_guard 565 .internal_state() 566 .contains(IrqDescState::IRQS_SPURIOUS_DISABLED) 567 { 568 desc_inner_guard 569 .internal_state_mut() 570 .remove(IrqDescState::IRQS_SPURIOUS_DISABLED); 571 self.do_enable_irq(desc.clone(), &mut desc_inner_guard).ok(); 572 } 573 574 drop(desc_inner_guard); 575 desc.chip_bus_sync_unlock(); 576 drop(req_mutex_guard); 577 578 drop(action_guard); 579 self.wake_up_and_wait_for_irq_thread_ready(&desc, Some(action.clone())); 580 self.wake_up_and_wait_for_irq_thread_ready(&desc, action.inner().secondary()); 581 return Ok(()); 582 } 583 584 /// 唤醒中断线程并等待中断线程准备好 585 /// 586 /// ## 参数 587 /// 588 /// - desc: 中断描述符 589 /// - action: 要唤醒的中断处理函数 590 /// 591 /// ## 锁 592 /// 593 /// 进入当前函数时，`action`的锁需要被释放 594 fn wake_up_and_wait_for_irq_thread_ready( 595 &self, 596 desc: &Arc<IrqDesc>, 597 action: Option<Arc<IrqAction>>, 598 ) { 599 if action.is_none() { 600 return; 601 } 602 603 let action = action.unwrap(); 604 605 let action_guard = action.inner(); 606 if action_guard.thread().is_none() { 607 return; 608 } 609 610 ProcessManager::wakeup(&action_guard.thread().unwrap()).ok(); 611 drop(action_guard); 612 action 613 .thread_completion() 614 .wait_for_completion() 615 .map_err(|e| { 616 warn!( 617 "Failed to wait for irq thread ready for {} (irq {:?}), error {:?}", 618 action.inner().name(), 619 desc.irq_data().irq(), 620 e 621 ); 622 }) 623 .ok(); 624 } 625 626 pub(super) fn irq_activate_and_startup( 627 &self, 628 desc: &Arc<IrqDesc>, 629 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, 630 resend: bool, 631 ) -> Result<(), SystemError> { 632 debug!( 633 "irq_activate_and_startup: irq: {}, name: {:?}", 634 desc.irq().data(), 635 desc_inner_guard.name() 636 ); 637 self.irq_activate(desc, desc_inner_guard)?; 638 self.irq_startup(desc, desc_inner_guard, resend, Self::IRQ_START_FORCE) 639 } 640 641 pub(super) fn irq_activate( 642 &self, 643 _desc: &Arc<IrqDesc>, 644 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, 645 ) -> Result<(), SystemError> { 646 let irq_data = desc_inner_guard.irq_data(); 647 648 if !desc_inner_guard.common_data().status().affinity_managed() { 649 return irq_domain_manager().activate_irq(irq_data, false); 650 } 651 652 return Ok(()); 653 } 654 655 /// 设置CPU亲和性并开启中断 656 pub(super) fn irq_startup( 657 &self, 658 desc: &Arc<IrqDesc>, 659 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, 660 resend: bool, 661 force: bool, 662 ) -> Result<(), SystemError> { 663 debug!( 664 "irq_startup: irq: {}, name: {:?}", 665 desc_inner_guard.irq_data().irq().data(), 666 desc_inner_guard.name() 667 ); 668 let mut ret = Ok(()); 669 let irq_data = desc_inner_guard.irq_data().clone(); 670 let affinity = desc_inner_guard.common_data().affinity(); 671 desc_inner_guard.set_depth(0); 672 673 if desc_inner_guard.common_data().status().started() { 674 self.irq_enable(desc_inner_guard); 675 } else { 676 match self.__irq_startup_managed(desc_inner_guard, &affinity, force) { 677 IrqStartupResult::Normal => { 678 if irq_data 679 .chip_info_read_irqsave() 680 .chip() 681 .flags() 682 .contains(IrqChipFlags::IRQCHIP_AFFINITY_PRE_STARTUP) 683 { 684 self.irq_setup_affinity(desc, desc_inner_guard).ok(); 685 } 686 687 ret = self.__irq_startup(desc_inner_guard); 688 689 if !irq_data 690 .chip_info_read_irqsave() 691 .chip() 692 .flags() 693 .contains(IrqChipFlags::IRQCHIP_AFFINITY_PRE_STARTUP) 694 { 695 self.irq_setup_affinity(desc, desc_inner_guard).ok(); 696 } 697 } 698 IrqStartupResult::Managed => { 699 self.irq_do_set_affinity(&irq_data, desc_inner_guard, &affinity, false) 700 .ok(); 701 ret = self.__irq_startup(desc_inner_guard); 702 } 703 IrqStartupResult::Abort => { 704 desc_inner_guard 705 .common_data() 706 .insert_status(IrqStatus::IRQD_MANAGED_SHUTDOWN); 707 return Ok(()); 708 } 709 } 710 } 711 712 if resend { 713 if let Err(e) = self.irq_check_and_resend(desc_inner_guard, false) { 714 error!( 715 "Failed to check and resend irq {}, error {:?}", 716 irq_data.irq().data(), 717 e 718 ); 719 } 720 } 721 722 return ret; 723 } 724 725 pub fn irq_enable(&self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) { 726 let common_data = desc_inner_guard.common_data(); 727 if !common_data.status().disabled() { 728 self.unmask_irq(desc_inner_guard); 729 } else { 730 common_data.clear_disabled(); 731 732 let chip = desc_inner_guard.irq_data().chip_info_read_irqsave().chip(); 733 734 if let Err(e) = chip.irq_enable(desc_inner_guard.irq_data()) { 735 if e == SystemError::ENOSYS { 736 self.unmask_irq(desc_inner_guard); 737 } 738 error!( 739 "Failed to enable irq {} (name: {:?}), error {:?}", 740 desc_inner_guard.irq_data().irq().data(), 741 desc_inner_guard.name(), 742 e 743 ); 744 } else { 745 common_data.clear_masked(); 746 } 747 } 748 } 749 750 /// 自动设置中断的CPU亲和性 751 /// 752 /// 753 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c#589 754 pub fn irq_setup_affinity( 755 &self, 756 _desc: &Arc<IrqDesc>, 757 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, 758 ) -> Result<(), SystemError> { 759 let common_data = desc_inner_guard.common_data(); 760 if !desc_inner_guard.can_set_affinity() { 761 return Ok(()); 762 } 763 764 let mut to_set = IRQ_DEFAULT_AFFINITY.clone(); 765 if common_data.status().affinity_managed() 766 || common_data.status().contains(IrqStatus::IRQD_AFFINITY_SET) 767 { 768 // FIXME: 要判断affinity跟已上线的CPU是否有交集 769 770 let irq_aff = common_data.affinity(); 771 if irq_aff.is_empty() { 772 common_data.clear_status(IrqStatus::IRQD_AFFINITY_SET); 773 } else { 774 to_set = irq_aff; 775 } 776 } 777 778 // FIXME: 求to_set和在线CPU的交集 779 780 return self.irq_do_set_affinity( 781 desc_inner_guard.irq_data(), 782 desc_inner_guard, 783 &to_set, 784 false, 785 ); 786 } 787 788 pub fn irq_set_affinity( 789 &self, 790 irq_data: &Arc<IrqData>, 791 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, 792 cpumask: &CpuMask, 793 ) -> Result<(), SystemError> { 794 return self.irq_do_set_affinity(irq_data, desc_inner_guard, cpumask, false); 795 } 796 fn irq_do_set_affinity( 797 &self, 798 irq_data: &Arc<IrqData>, 799 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, 800 cpumask: &CpuMask, 801 force: bool, 802 ) -> Result<(), SystemError> { 803 let chip = irq_data.chip_info_read_irqsave().chip(); 804 if !chip.can_set_affinity() { 805 return Err(SystemError::EINVAL); 806 } 807 808 // todo: 处理CPU中断隔离相关的逻辑 809 810 let common_data = desc_inner_guard.common_data(); 811 let r = if force || !cpumask.is_empty() { 812 chip.irq_set_affinity(irq_data, cpumask, force) 813 } else { 814 return Err(SystemError::EINVAL); 815 }; 816 817 let mut ret = Ok(()); 818 if let Ok(rs) = r { 819 match rs { 820 IrqChipSetMaskResult::Success | IrqChipSetMaskResult::Done => { 821 common_data.set_affinity(cpumask.clone()); 822 } 823 IrqChipSetMaskResult::NoChange => { 824 825 // irq_validate_effective_affinity(data); 826 // irq_set_thread_affinity(desc); 827 } 828 } 829 } else { 830 ret = Err(r.unwrap_err()); 831 } 832 833 return ret; 834 } 835 836 fn __irq_startup( 837 &self, 838 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, 839 ) -> Result<(), SystemError> { 840 let common_data = desc_inner_guard.common_data(); 841 842 if let Err(e) = desc_inner_guard 843 .irq_data() 844 .chip_info_read_irqsave() 845 .chip() 846 .irq_startup(desc_inner_guard.irq_data()) 847 { 848 if e == SystemError::ENOSYS { 849 self.irq_enable(desc_inner_guard); 850 } else { 851 return Err(e); 852 } 853 } else { 854 common_data.clear_disabled(); 855 common_data.clear_masked(); 856 } 857 858 common_data.set_started(); 859 860 return Ok(()); 861 } 862 863 fn __irq_startup_managed( 864 &self, 865 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, 866 _affinity: &CpuMask, 867 _force: bool, 868 ) -> IrqStartupResult { 869 let irq_data = desc_inner_guard.irq_data(); 870 let common_data = desc_inner_guard.common_data(); 871 872 if !common_data.status().affinity_managed() { 873 return IrqStartupResult::Normal; 874 } 875 876 common_data.clear_managed_shutdown(); 877 878 /* 879 - 检查Affinity掩码是否包括所有的在线CPU。如果是，这意味着有代码试图在管理的中断上使用enable_irq()， 880 这可能是非法的。在这种情况下，如果force不是真值，函数会返回IRQ_STARTUP_ABORT，表示中断处理应该被放弃。 881 - 如果Affinity掩码中没有任何在线的CPU，那么中断请求是不可用的，因为没有任何CPU可以处理它。 882 在这种情况下，如果force不是真值，函数同样会返回IRQ_STARTUP_ABORT。 883 - 如果以上条件都不满足，尝试激活中断，并将其设置为管理模式。这是通过调用 `irq_domain_manager().activate_irq()` 函数来实现的。 884 如果这个调用失败，表示有保留的资源无法访问，函数会返回IRQ_STARTUP_ABORT。 885 - 如果一切顺利，函数会返回IRQ_STARTUP_MANAGED，表示中断已经被成功管理并激活。 886 */ 887 888 // if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) { 889 // /* 890 // * Catch code which fiddles with enable_irq() on a managed 891 // * and potentially shutdown IRQ. Chained interrupt 892 // * installment or irq auto probing should not happen on 893 // * managed irqs either. 894 // */ 895 // if (WARN_ON_ONCE(force)) 896 // return IRQ_STARTUP_ABORT; 897 // /* 898 // * The interrupt was requested, but there is no online CPU 899 // * in it's affinity mask. Put it into managed shutdown 900 // * state and let the cpu hotplug mechanism start it up once 901 // * a CPU in the mask becomes available. 902 // */ 903 // return IRQ_STARTUP_ABORT; 904 // } 905 906 let r = irq_domain_manager().activate_irq(irq_data, false); 907 if r.is_err() { 908 return IrqStartupResult::Abort; 909 } 910 911 return IrqStartupResult::Managed; 912 } 913 914 pub fn do_enable_irq( 915 &self, 916 _desc: Arc<IrqDesc>, 917 _desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, 918 ) -> Result<(), SystemError> { 919 // https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c?r=&mo=59252&fi=2138#776 920 todo!("do_enable_irq") 921 } 922 923 #[inline(never)] 924 pub fn do_set_irq_trigger( 925 &self, 926 _desc: Arc<IrqDesc>, 927 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, 928 mut trigger_type: IrqLineStatus, 929 ) -> Result<(), SystemError> { 930 let chip = desc_inner_guard.irq_data().chip_info_read_irqsave().chip(); 931 let mut to_unmask = false; 932 933 if !chip.can_set_flow_type() { 934 // debug!( 935 // "No set_irq_type function for irq {}, chip {}", 936 // desc_inner_guard.irq_data().irq().data(), 937 // chip.name() 938 // ); 939 return Ok(()); 940 } 941 942 if chip.flags().contains(IrqChipFlags::IRQCHIP_SET_TYPE_MASKED) { 943 if !desc_inner_guard.common_data().status().masked() { 944 self.mask_irq(desc_inner_guard.irq_data()); 945 } 946 if !desc_inner_guard.common_data().status().disabled() { 947 to_unmask = true; 948 } 949 } 950 951 trigger_type &= IrqLineStatus::IRQ_TYPE_SENSE_MASK; 952 953 let r = chip.irq_set_type(desc_inner_guard.irq_data(), trigger_type); 954 let ret; 955 if let Ok(rs) = r { 956 match rs { 957 IrqChipSetMaskResult::Success | IrqChipSetMaskResult::Done => { 958 let common_data = desc_inner_guard.common_data(); 959 common_data.clear_status(IrqStatus::IRQD_TRIGGER_MASK); 960 let mut irqstatus = IrqStatus::empty(); 961 irqstatus.set_trigger_type(trigger_type); 962 common_data.insert_status(irqstatus); 963 } 964 IrqChipSetMaskResult::NoChange => { 965 let flags = desc_inner_guard.common_data().trigger_type(); 966 desc_inner_guard.set_trigger_type(flags); 967 desc_inner_guard 968 .common_data() 969 .clear_status(IrqStatus::IRQD_LEVEL); 970 desc_inner_guard.clear_level(); 971 972 if !(flags & IrqLineStatus::IRQ_TYPE_LEVEL_MASK).is_empty() { 973 desc_inner_guard.set_level(); 974 desc_inner_guard 975 .common_data() 976 .insert_status(IrqStatus::IRQD_LEVEL); 977 } 978 } 979 } 980 981 ret = Ok(()); 982 } else { 983 error!( 984 "Failed to set irq {} trigger type to {:?} on irqchip {}, error {:?}", 985 desc_inner_guard.irq_data().irq().data(), 986 trigger_type, 987 chip.name(), 988 r 989 ); 990 991 ret = Err(r.unwrap_err()); 992 } 993 994 if to_unmask { 995 self.unmask_irq(desc_inner_guard); 996 } 997 return ret; 998 } 999 1000 fn irq_request_resources(&self, desc: Arc<IrqDesc>) -> Result<(), SystemError> { 1001 let irq_data = desc.irq_data(); 1002 let irq_chip = irq_data.chip_info_read_irqsave().chip(); 1003 irq_chip.irq_request_resources(&irq_data) 1004 } 1005 1006 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c?r=&mo=59252&fi=2138#1448 1007 fn setup_irq_thread( 1008 &self, 1009 _irq: IrqNumber, 1010 _action: &InnerIrqAction, 1011 _secondary: bool, 1012 ) -> Result<(), SystemError> { 1013 // if secondary { 1014 // KernelThreadMechanism::create(func, name) 1015 // } 1016 1017 todo!("setup_irq_thread") 1018 } 1019 1020 fn setup_forced_threading(&self, _action: &mut InnerIrqAction) -> Result<(), SystemError> { 1021 // todo: 处理强制线程化的逻辑，参考linux的`irq_setup_forced_threading()` 1022 return Ok(()); 1023 } 1024 1025 pub fn irq_clear_status_flags( 1026 &self, 1027 irq: IrqNumber, 1028 status: IrqLineStatus, 1029 ) -> Result<(), SystemError> { 1030 let desc = irq_desc_manager().lookup(irq).ok_or(SystemError::EINVAL)?; 1031 desc.modify_status(status, IrqLineStatus::empty()); 1032 return Ok(()); 1033 } 1034 1035 /// 屏蔽中断 1036 pub(super) fn mask_irq(&self, irq_data: &Arc<IrqData>) { 1037 if irq_data.common_data().status().masked() { 1038 return; 1039 } 1040 1041 let chip = irq_data.chip_info_read_irqsave().chip(); 1042 let r = chip.irq_mask(irq_data); 1043 1044 if r.is_ok() { 1045 irq_data.common_data().set_masked(); 1046 } 1047 } 1048 1049 /// 解除屏蔽中断 1050 pub(super) fn unmask_irq(&self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) { 1051 if !desc_inner_guard.common_data().status().masked() { 1052 return; 1053 } 1054 1055 let r = desc_inner_guard 1056 .irq_data() 1057 .chip_info_read_irqsave() 1058 .chip() 1059 .irq_unmask(desc_inner_guard.irq_data()); 1060 1061 if let Err(e) = r { 1062 if e != SystemError::ENOSYS { 1063 error!( 1064 "Failed to unmask irq {} on irqchip {}, error {:?}", 1065 desc_inner_guard.irq_data().irq().data(), 1066 desc_inner_guard 1067 .irq_data() 1068 .chip_info_read_irqsave() 1069 .chip() 1070 .name(), 1071 e 1072 ); 1073 } 1074 } else { 1075 desc_inner_guard 1076 .common_data() 1077 .clear_status(IrqStatus::IRQD_IRQ_MASKED); 1078 } 1079 } 1080 1081 /// 释放使用request_irq分配的中断 1082 /// 1083 /// ## 参数 1084 /// 1085 /// - irq: 要释放的中断线 1086 /// - dev_id: 要释放的设备身份 1087 /// 1088 /// ## 返回 1089 /// 1090 /// 返回传递给request_irq的devname参数 1091 /// 1092 /// ## 说明 1093 /// 1094 /// 移除一个中断处理程序。处理程序被移除，如果该中断线不再被任何驱动程序使用，则会被禁用。 1095 /// 1096 /// 在共享IRQ的情况下，调用者必须确保在调用此功能之前，它在所驱动的卡上禁用了中断。 1097 /// 1098 /// ## 注意 1099 /// 1100 /// 此函数不可以在中断上下文中调用。 1101 /// 1102 /// 参考 https://code.dragonos.org.cn/xref/linux-6.6.21/kernel/irq/manage.c#2026 1103 pub fn free_irq(&self, _irq: IrqNumber, _dev_id: Option<Arc<DeviceId>>) { 1104 warn!("Unimplemented free_irq"); 1105 } 1106 } 1107 1108 enum IrqStartupResult { 1109 Normal, 1110 Managed, 1111 Abort, 1112 } 1113 /// 默认的初级中断处理函数 1114 /// 1115 /// 该处理函数仅仅返回`WakeThread`，即唤醒中断线程 1116 #[derive(Debug)] 1117 struct DefaultPrimaryIrqHandler; 1118 1119 impl IrqHandler for DefaultPrimaryIrqHandler { 1120 fn handle( 1121 &self, 1122 _irq: IrqNumber, 1123 _static_data: Option<&dyn IrqHandlerData>, 1124 _dynamic_data: Option<Arc<dyn IrqHandlerData>>, 1125 ) -> Result<IrqReturn, SystemError> { 1126 return Ok(IrqReturn::WakeThread); 1127 } 1128 } 1129 1130 /// Primary handler for nested threaded interrupts. 1131 /// Should never be called. 1132 #[derive(Debug)] 1133 struct IrqNestedPrimaryHandler; 1134 1135 impl IrqHandler for IrqNestedPrimaryHandler { 1136 fn handle( 1137 &self, 1138 irq: IrqNumber, 1139 _static_data: Option<&dyn IrqHandlerData>, 1140 _dynamic_data: Option<Arc<dyn IrqHandlerData>>, 1141 ) -> Result<IrqReturn, SystemError> { 1142 warn!("Primary handler called for nested irq {}", irq.data()); 1143 return Ok(IrqReturn::NotHandled); 1144 } 1145 } 1146