1 use core::ops::{BitXor, Deref, DerefMut};
2
3 use alloc::{string::String, sync::Arc};
4
5 use log::{debug, error, warn};
6 use system_error::SystemError;
7
8 use crate::{
9 driver::base::device::DeviceId,
10 exception::{
11 irqchip::IrqChipSetMaskResult,
12 irqdesc::{irq_desc_manager, InnerIrqDesc, IrqAction},
13 },
14 libs::{cpumask::CpuMask, spinlock::SpinLockGuard},
15 process::{kthread::KernelThreadMechanism, ProcessManager},
16 smp::cpu::ProcessorId,
17 };
18
19 use super::{
20 dummychip::no_irq_chip,
21 irqchip::IrqChipFlags,
22 irqdata::{IrqData, IrqHandlerData, IrqLineStatus, IrqStatus},
23 irqdesc::{InnerIrqAction, IrqDesc, IrqDescState, IrqHandleFlags, IrqHandler, IrqReturn},
24 irqdomain::irq_domain_manager,
25 IrqNumber,
26 };
27
28 lazy_static! {
29 /// 默认的中断亲和性
30 static ref IRQ_DEFAULT_AFFINITY: CpuMask = {
31 let mut mask = CpuMask::new();
32 // 默认情况下，中断处理程序将在第一个处理器上运行
33 mask.set(ProcessorId::new(0), true);
34 mask
35 };
36 }
37
irq_manager() -> &'static IrqManager38 pub fn irq_manager() -> &'static IrqManager {
39 &IrqManager
40 }
41
42 /// 中断管理器
43 pub struct IrqManager;
44
45 impl IrqManager {
46 pub const IRQ_RESEND: bool = true;
47 #[allow(dead_code)]
48 pub const IRQ_NORESEND: bool = false;
49 #[allow(dead_code)]
50 pub const IRQ_START_FORCE: bool = true;
51 pub const IRQ_START_COND: bool = false;
52
53 /// 在中断线上添加一个处理函数
54 ///
55 /// ## 参数
56 ///
57 /// - irq: 虚拟中断号(中断线号)
58 /// - name: 生成该中断的设备名称
59 /// - handler: 中断处理函数
60 /// - flags: 中断处理标志
61 /// - dev_id: 一个用于标识设备的cookie
request_irq( &self, irq: IrqNumber, name: String, handler: &'static dyn IrqHandler, flags: IrqHandleFlags, dev_id: Option<Arc<DeviceId>>, ) -> Result<(), SystemError>62 pub fn request_irq(
63 &self,
64 irq: IrqNumber,
65 name: String,
66 handler: &'static dyn IrqHandler,
67 flags: IrqHandleFlags,
68 dev_id: Option<Arc<DeviceId>>,
69 ) -> Result<(), SystemError> {
70 return self.request_threaded_irq(irq, Some(handler), None, flags, name, dev_id);
71 }
72
73 /// 在中断线上添加一个处理函数（可以是线程化的中断）
74 ///
75 /// ## 参数
76 ///
77 /// - irq: 虚拟中断号
78 /// - handler: 当中断发生时将被调用的函数，是
79 /// 线程化中断的初级处理程序。如果handler为`None`并且thread_fn不为`None`，
80 /// 将安装默认的初级处理程序
81 /// - thread_fn: 在中断处理程序线程中调用的函数. 如果为`None`，则不会创建irq线程
82 /// - flags: 中断处理标志
83 /// - IRQF_SHARED: 中断是共享的
84 /// - IRQF_TRIGGER*: 指定中断触发方式
85 /// - IRQF_ONESHOT: 在thread_fn中运行时，中断线被遮蔽
86 /// - dev_name: 生成该中断的设备名称
87 /// - dev_id: 一个用于标识设备的cookie
88 ///
89 /// ## 说明
90 ///
91 /// 此调用分配中断资源并启用中断线和IRQ处理。
92 /// 从这一点开始，您的处理程序函数可能会被调用。
93 /// 因此，您必须确保首先初始化您的硬件，
94 /// 并确保以正确的顺序设置中断处理程序。
95 ///
96 /// 如果您想为您的设备设置线程化中断处理程序
97 /// 则需要提供@handler和@thread_fn。@handler仍然
98 /// 在硬中断上下文中调用，并且必须检查
99 /// 中断是否来自设备。如果是，它需要禁用设备上的中断
100 /// 并返回IRQ_WAKE_THREAD，这将唤醒处理程序线程并运行
101 /// @thread_fn。这种拆分处理程序设计是为了支持
102 /// 共享中断。
103 ///
104 /// dev_id必须是全局唯一的。通常使用设备数据结构的地址或者uuid
105 /// 作为cookie。由于处理程序接收这个值，因此使用它是有意义的。
106 ///
107 /// 如果您的中断是共享的，您必须传递一个非NULL的dev_id
108 /// 因为当释放中断时需要它。
request_threaded_irq( &self, irq: IrqNumber, mut handler: Option<&'static dyn IrqHandler>, thread_fn: Option<&'static dyn IrqHandler>, flags: IrqHandleFlags, dev_name: String, dev_id: Option<Arc<DeviceId>>, ) -> Result<(), SystemError>109 pub fn request_threaded_irq(
110 &self,
111 irq: IrqNumber,
112 mut handler: Option<&'static dyn IrqHandler>,
113 thread_fn: Option<&'static dyn IrqHandler>,
114 flags: IrqHandleFlags,
115 dev_name: String,
116 dev_id: Option<Arc<DeviceId>>,
117 ) -> Result<(), SystemError> {
118 if irq == IrqNumber::IRQ_NOTCONNECTED {
119 return Err(SystemError::ENOTCONN);
120 }
121
122 // 逻辑检查：共享中断必须传入一个真正的设备ID，
123 // 否则后来我们将难以确定哪个中断是哪个（会搞乱中断释放逻辑等）。
124 // 此外，共享中断与禁用自动使能不相符。 共享中断可能在仍然禁用时请求它，然后永远等待中断。
125 // 另外，IRQF_COND_SUSPEND 仅适用于共享中断，并且它不能与 IRQF_NO_SUSPEND 同时设置。
126
127 if ((flags.contains(IrqHandleFlags::IRQF_SHARED)) && dev_id.is_none())
128 || ((flags.contains(IrqHandleFlags::IRQF_SHARED))
129 && (flags.contains(IrqHandleFlags::IRQF_NO_AUTOEN)))
130 || (!(flags.contains(IrqHandleFlags::IRQF_SHARED))
131 && (flags.contains(IrqHandleFlags::IRQF_COND_SUSPEND)))
132 || ((flags.contains(IrqHandleFlags::IRQF_NO_SUSPEND))
133 && (flags.contains(IrqHandleFlags::IRQF_COND_SUSPEND)))
134 {
135 return Err(SystemError::EINVAL);
136 }
137 let desc = irq_desc_manager().lookup(irq).ok_or(SystemError::EINVAL)?;
138 if !desc.can_request() {
139 warn!("desc {} can not request", desc.irq().data());
140 return Err(SystemError::EINVAL);
141 }
142
143 if handler.is_none() {
144 if thread_fn.is_none() {
145 // 不允许中断处理函数和线程处理函数都为空
146 return Err(SystemError::EINVAL);
147 }
148
149 // 如果中断处理函数为空，线程处理函数不为空，则使用默认的中断处理函数
150 handler = Some(&DefaultPrimaryIrqHandler);
151 }
152
153 let irqaction = IrqAction::new(irq, dev_name, handler, thread_fn);
154
155 let mut action_guard = irqaction.inner();
156 *action_guard.flags_mut() = flags;
157 *action_guard.dev_id_mut() = dev_id;
158 drop(action_guard);
159 debug!("to inner_setup_irq: {irq:?}");
160 return self.inner_setup_irq(irq, irqaction, desc);
161 }
162
163 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c?r=&mo=59252&fi=2138#1497
164 #[inline(never)]
inner_setup_irq( &self, irq: IrqNumber, action: Arc<IrqAction>, desc: Arc<IrqDesc>, ) -> Result<(), SystemError>165 fn inner_setup_irq(
166 &self,
167 irq: IrqNumber,
168 action: Arc<IrqAction>,
169 desc: Arc<IrqDesc>,
170 ) -> Result<(), SystemError> {
171 // ==== 定义错误处理函数 ====
172 let err_out_thread =
173 |e: SystemError, mut action_guard: SpinLockGuard<'_, InnerIrqAction>| -> SystemError {
174 if let Some(thread_pcb) = action_guard.thread() {
175 action_guard.set_thread(None);
176 KernelThreadMechanism::stop(&thread_pcb).ok();
177 }
178
179 if let Some(secondary) = action_guard.secondary() {
180 let mut secondary_guard = secondary.inner();
181 if let Some(thread_pcb) = secondary_guard.thread() {
182 secondary_guard.set_thread(None);
183 KernelThreadMechanism::stop(&thread_pcb).ok();
184 }
185 }
186 return e;
187 };
188
189 let err_out_bus_unlock = |e: SystemError,
190 desc: Arc<IrqDesc>,
191 req_mutex_guard: crate::libs::mutex::MutexGuard<'_, ()>,
192 action_guard: SpinLockGuard<'_, InnerIrqAction>|
193 -> SystemError {
194 desc.chip_bus_sync_unlock();
195 drop(req_mutex_guard);
196 return err_out_thread(e, action_guard);
197 };
198
199 let err_out_unlock = |e: SystemError,
200 desc_guard: SpinLockGuard<'_, InnerIrqDesc>,
201 desc: Arc<IrqDesc>,
202 req_mutex_guard: crate::libs::mutex::MutexGuard<'_, ()>,
203 action_guard: SpinLockGuard<'_, InnerIrqAction>|
204 -> SystemError {
205 drop(desc_guard);
206 return err_out_bus_unlock(e, desc, req_mutex_guard, action_guard);
207 };
208
209 let err_out_mismatch = |old_action_guard: SpinLockGuard<'_, InnerIrqAction>,
210 desc_guard: SpinLockGuard<'_, InnerIrqDesc>,
211 action_guard: SpinLockGuard<'_, InnerIrqAction>,
212 desc: Arc<IrqDesc>,
213 req_mutex_guard: crate::libs::mutex::MutexGuard<'_, ()>|
214 -> SystemError {
215 if !action_guard
216 .flags()
217 .contains(IrqHandleFlags::IRQF_PROBE_SHARED)
218 {
219 error!("Flags mismatch for irq {} (name: {}, flags: {:?}). old action name: {}, old flags: {:?}", irq.data(), action_guard.name(), action_guard.flags(), old_action_guard.name(), old_action_guard.flags());
220 }
221 return err_out_unlock(
222 SystemError::EBUSY,
223 desc_guard,
224 desc,
225 req_mutex_guard,
226 action_guard,
227 );
228 };
229
230 // ===== 代码开始 =====
231
232 if Arc::ptr_eq(
233 &desc.irq_data().chip_info_read_irqsave().chip(),
234 &no_irq_chip(),
235 ) {
236 return Err(SystemError::ENOSYS);
237 }
238
239 let mut action_guard = action.inner();
240 if !action_guard.flags().trigger_type_specified() {
241 // 如果没有指定触发类型，则使用默认的触发类型
242 action_guard
243 .flags_mut()
244 .insert_trigger_type(desc.irq_data().common_data().trigger_type())
245 }
246
247 let nested = desc.nested_thread();
248
249 if nested {
250 if action_guard.thread_fn().is_none() {
251 return Err(SystemError::EINVAL);
252 }
253
254 action_guard.set_handler(Some(&IrqNestedPrimaryHandler));
255 } else if desc.can_thread() {
256 self.setup_forced_threading(action_guard.deref_mut())?;
257 }
258
259 // 如果具有中断线程处理程序，并且中断不是嵌套的，则设置中断线程
260 if action_guard.thread_fn().is_some() && !nested {
261 self.setup_irq_thread(irq, action_guard.deref(), false)?;
262
263 if let Some(secondary) = action_guard.secondary() {
264 let secondary_guard = secondary.inner();
265 if let Err(e) = self.setup_irq_thread(irq, secondary_guard.deref(), true) {
266 return Err(err_out_thread(e, action_guard));
267 }
268 }
269 }
270
271 // Drivers are often written to work w/o knowledge about the
272 // underlying irq chip implementation, so a request for a
273 // threaded irq without a primary hard irq context handler
274 // requires the ONESHOT flag to be set. Some irq chips like
275 // MSI based interrupts are per se one shot safe. Check the
276 // chip flags, so we can avoid the unmask dance at the end of
277 // the threaded handler for those.
278
279 if desc
280 .irq_data()
281 .chip_info_read_irqsave()
282 .chip()
283 .flags()
284 .contains(IrqChipFlags::IRQCHIP_ONESHOT_SAFE)
285 {
286 *action_guard.flags_mut() &= !IrqHandleFlags::IRQF_ONESHOT;
287 }
288
289 // Protects against a concurrent __free_irq() call which might wait
290 // for synchronize_hardirq() to complete without holding the optional
291 // chip bus lock and desc->lock. Also protects against handing out
292 // a recycled oneshot thread_mask bit while it's still in use by
293 // its previous owner.
294 let req_mutex_guard = desc.request_mutex_lock();
295
296 // Acquire bus lock as the irq_request_resources() callback below
297 // might rely on the serialization or the magic power management
298 // functions which are abusing the irq_bus_lock() callback,
299 desc.chip_bus_lock();
300
301 // 如果当前中断线上还没有irqaction, 则先为中断线申请资源
302 if desc.actions().is_empty() {
303 if let Err(e) = self.irq_request_resources(desc.clone()) {
304 error!(
305 "Failed to request resources for {} (irq {}) on irqchip {}, error {:?}",
306 action_guard.name(),
307 irq.data(),
308 desc.irq_data().chip_info_read_irqsave().chip().name(),
309 e
310 );
311 return Err(err_out_bus_unlock(
312 e,
313 desc.clone(),
314 req_mutex_guard,
315 action_guard,
316 ));
317 }
318 }
319
320 let mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc> = desc.inner();
321
322 // 标记当前irq是否是共享的
323 let mut irq_shared = false;
324 if !desc_inner_guard.actions().is_empty() {
325 // 除非双方都同意并且是相同类型（级别、边沿、极性），否则不能共享中断。
326 // 因此，两个标志字段都必须设置IRQF_SHARED，并且设置触发类型的位必须匹配。
327 // 另外，所有各方都必须就ONESHOT达成一致。
328 // NMI用途的中断线不能共享。
329 if desc_inner_guard
330 .internal_state()
331 .contains(IrqDescState::IRQS_NMI)
332 {
333 error!(
334 "Invalid attempt to share NMI for {} (irq {}) on irqchip {}",
335 action_guard.name(),
336 irq.data(),
337 desc_inner_guard
338 .irq_data()
339 .chip_info_read_irqsave()
340 .chip()
341 .name()
342 );
343 return Err(err_out_unlock(
344 SystemError::EINVAL,
345 desc_inner_guard,
346 desc.clone(),
347 req_mutex_guard,
348 action_guard,
349 ));
350 }
351
352 let irq_data = desc_inner_guard.irq_data();
353
354 let old_trigger_type: super::irqdata::IrqLineStatus;
355 let status = irq_data.common_data().status();
356 if status.trigger_type_was_set() {
357 old_trigger_type = status.trigger_type();
358 } else {
359 old_trigger_type = action_guard.flags().trigger_type();
360 irq_data.common_data().set_trigger_type(old_trigger_type);
361 }
362
363 let old = &desc_inner_guard.actions()[0].clone();
364 let old_guard = old.inner();
365
366 if (!(old_guard
367 .flags()
368 .intersection(*action_guard.flags())
369 .contains(IrqHandleFlags::IRQF_SHARED)))
370 || (old_trigger_type != (action_guard.flags().trigger_type()))
371 || ((old_guard.flags().bitxor(*action_guard.flags()))
372 .contains(IrqHandleFlags::IRQF_ONESHOT))
373 {
374 debug!(
375 "Flags mismatch for irq {} (name: {}, flags: {:?}). old action name: {}, old flags: {:?}",
376 irq.data(),
377 action_guard.name(),
378 action_guard.flags(),
379 old_guard.name(),
380 old_guard.flags()
381 );
382
383 return Err(err_out_mismatch(
384 old_guard,
385 desc_inner_guard,
386 action_guard,
387 desc.clone(),
388 req_mutex_guard,
389 ));
390 }
391
392 // all handlers must agree on per-cpuness
393 if *old_guard.flags() & IrqHandleFlags::IRQF_PERCPU
394 != *action_guard.flags() & IrqHandleFlags::IRQF_PERCPU
395 {
396 debug!(
397 "Per-cpu mismatch for irq {} (name: {}, flags: {:?})",
398 irq.data(),
399 action_guard.name(),
400 action_guard.flags()
401 );
402 return Err(err_out_mismatch(
403 old_guard,
404 desc_inner_guard,
405 action_guard,
406 desc.clone(),
407 req_mutex_guard,
408 ));
409 }
410
411 irq_shared = true;
412 }
413
414 if action_guard.flags().contains(IrqHandleFlags::IRQF_ONESHOT) {
415 // todo: oneshot
416 } else if action_guard.handler().is_some_and(|h| {
417 h.type_id() == (&DefaultPrimaryIrqHandler as &dyn IrqHandler).type_id()
418 }) && !desc_inner_guard
419 .irq_data()
420 .chip_info_read_irqsave()
421 .chip()
422 .flags()
423 .contains(IrqChipFlags::IRQCHIP_ONESHOT_SAFE)
424 {
425 // 请求中断时 hander = NULL，因此我们为其使用默认的主处理程序。
426 // 但它没有设置ONESHOT标志。与电平触发中断结合时，
427 // 这是致命的，因为默认的主处理程序只是唤醒线程，然后重新启用 irq 线路，
428 // 但设备仍然保持电平中断生效。周而复始....
429 // 虽然这对于边缘类型中断来说可行，但我们为了安全起见，不加条件地拒绝，
430 // 因为我们不能确定这个中断实际上具有什么类型。
431 // 由于底层芯片实现可能会覆盖它们，所以类型标志并不可靠.
432
433 error!(
434 "Requesting irq {} without a handler, and ONESHOT flags not set for irqaction: {}",
435 irq.data(),
436 action_guard.name()
437 );
438 return Err(err_out_unlock(
439 SystemError::EINVAL,
440 desc_inner_guard,
441 desc.clone(),
442 req_mutex_guard,
443 action_guard,
444 ));
445 }
446
447 // 第一次在当前irqdesc上注册中断处理函数
448 if !irq_shared {
449 // 设置中断触发方式
450 if action_guard.flags().trigger_type_specified() {
451 let trigger_type = action_guard.flags().trigger_type();
452 if let Err(e) =
453 self.do_set_irq_trigger(desc.clone(), &mut desc_inner_guard, trigger_type)
454 {
455 debug!(
456 "Failed to set trigger type for irq {} (name: {}, flags: {:?}), error {:?}",
457 irq.data(),
458 action_guard.name(),
459 action_guard.flags(),
460 e
461 );
462 return Err(err_out_unlock(
463 e,
464 desc_inner_guard,
465 desc.clone(),
466 req_mutex_guard,
467 action_guard,
468 ));
469 }
470 }
471 debug!("to irq_activate");
472 // 激活中断。这种激活必须独立于IRQ_NOAUTOEN进行*desc_inner_guard.internal_state_mut() |= IrqDescState::IRQS_NOREQUEST;uest.
473 if let Err(e) = self.irq_activate(&desc, &mut desc_inner_guard) {
474 debug!(
475 "Failed to activate irq {} (name: {}, flags: {:?}), error {:?}",
476 irq.data(),
477 action_guard.name(),
478 action_guard.flags(),
479 e
480 );
481 return Err(err_out_unlock(
482 e,
483 desc_inner_guard,
484 desc.clone(),
485 req_mutex_guard,
486 action_guard,
487 ));
488 }
489
490 *desc_inner_guard.internal_state_mut() &= !(IrqDescState::IRQS_AUTODETECT
491 | IrqDescState::IRQS_SPURIOUS_DISABLED
492 | IrqDescState::IRQS_ONESHOT
493 | IrqDescState::IRQS_WAITING);
494 desc_inner_guard
495 .common_data()
496 .clear_status(IrqStatus::IRQD_IRQ_INPROGRESS);
497
498 if action_guard.flags().contains(IrqHandleFlags::IRQF_PERCPU) {
499 desc_inner_guard
500 .common_data()
501 .insert_status(IrqStatus::IRQD_PER_CPU);
502 desc_inner_guard.line_status_set_per_cpu();
503
504 if action_guard.flags().contains(IrqHandleFlags::IRQF_NO_DEBUG) {
505 desc_inner_guard.line_status_set_no_debug();
506 }
507 }
508
509 if action_guard.flags().contains(IrqHandleFlags::IRQF_ONESHOT) {
510 *desc_inner_guard.internal_state_mut() |= IrqDescState::IRQS_ONESHOT;
511 }
512
513 // 如果有要求的话，则忽略IRQ的均衡。
514 if action_guard
515 .flags()
516 .contains(IrqHandleFlags::IRQF_NOBALANCING)
517 {
518 todo!("IRQF_NO_BALANCING");
519 }
520
521 if !action_guard
522 .flags()
523 .contains(IrqHandleFlags::IRQF_NO_AUTOEN)
524 && desc_inner_guard.can_autoenable()
525 {
526 // 如果没有设置IRQF_NOAUTOEN，则自动使能中断
527 self.irq_startup(
528 &desc,
529 &mut desc_inner_guard,
530 Self::IRQ_RESEND,
531 Self::IRQ_START_COND,
532 )
533 .ok();
534 } else {
535 // 共享中断与禁用自动使能不太兼容。
536 // 共享中断可能在它仍然被禁用时请求它，然后永远等待中断。
537
538 static mut WARNED: bool = false;
539 if action_guard.flags().contains(IrqHandleFlags::IRQF_SHARED) && unsafe { !WARNED }
540 {
541 warn!(
542 "Shared interrupt {} for {} requested but not auto enabled",
543 irq.data(),
544 action_guard.name()
545 );
546 unsafe { WARNED = true };
547 }
548
549 desc_inner_guard.set_depth(1);
550 }
551 } else if action_guard.flags().trigger_type_specified() {
552 let new_trigger_type = action_guard.flags().trigger_type();
553 let old_trigger_type = desc_inner_guard.common_data().trigger_type();
554 if new_trigger_type != old_trigger_type {
555 warn!("Irq {} uses trigger type: {old_trigger_type:?}, but requested trigger type: {new_trigger_type:?}.", irq.data());
556 }
557 }
558
559 // 在队列末尾添加新的irqaction
560 desc_inner_guard.add_action(action.clone());
561
562 // 检查我们是否曾经通过虚构的中断处理程序禁用过irq。重新启用它并再给它一次机会。
563 if irq_shared
564 && desc_inner_guard
565 .internal_state()
566 .contains(IrqDescState::IRQS_SPURIOUS_DISABLED)
567 {
568 desc_inner_guard
569 .internal_state_mut()
570 .remove(IrqDescState::IRQS_SPURIOUS_DISABLED);
571 self.do_enable_irq(desc.clone(), &mut desc_inner_guard).ok();
572 }
573
574 drop(desc_inner_guard);
575 desc.chip_bus_sync_unlock();
576 drop(req_mutex_guard);
577
578 drop(action_guard);
579 self.wake_up_and_wait_for_irq_thread_ready(&desc, Some(action.clone()));
580 self.wake_up_and_wait_for_irq_thread_ready(&desc, action.inner().secondary());
581 return Ok(());
582 }
583
584 /// 唤醒中断线程并等待中断线程准备好
585 ///
586 /// ## 参数
587 ///
588 /// - desc: 中断描述符
589 /// - action: 要唤醒的中断处理函数
590 ///
591 /// ## 锁
592 ///
593 /// 进入当前函数时，`action`的锁需要被释放
wake_up_and_wait_for_irq_thread_ready( &self, desc: &Arc<IrqDesc>, action: Option<Arc<IrqAction>>, )594 fn wake_up_and_wait_for_irq_thread_ready(
595 &self,
596 desc: &Arc<IrqDesc>,
597 action: Option<Arc<IrqAction>>,
598 ) {
599 if action.is_none() {
600 return;
601 }
602
603 let action = action.unwrap();
604
605 let action_guard = action.inner();
606 if action_guard.thread().is_none() {
607 return;
608 }
609
610 ProcessManager::wakeup(&action_guard.thread().unwrap()).ok();
611 drop(action_guard);
612 action
613 .thread_completion()
614 .wait_for_completion()
615 .map_err(|e| {
616 warn!(
617 "Failed to wait for irq thread ready for {} (irq {:?}), error {:?}",
618 action.inner().name(),
619 desc.irq_data().irq(),
620 e
621 );
622 })
623 .ok();
624 }
625
irq_activate_and_startup( &self, desc: &Arc<IrqDesc>, desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, resend: bool, ) -> Result<(), SystemError>626 pub(super) fn irq_activate_and_startup(
627 &self,
628 desc: &Arc<IrqDesc>,
629 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>,
630 resend: bool,
631 ) -> Result<(), SystemError> {
632 debug!(
633 "irq_activate_and_startup: irq: {}, name: {:?}",
634 desc.irq().data(),
635 desc_inner_guard.name()
636 );
637 self.irq_activate(desc, desc_inner_guard)?;
638 self.irq_startup(desc, desc_inner_guard, resend, Self::IRQ_START_FORCE)
639 }
640
irq_activate( &self, _desc: &Arc<IrqDesc>, desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, ) -> Result<(), SystemError>641 pub(super) fn irq_activate(
642 &self,
643 _desc: &Arc<IrqDesc>,
644 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>,
645 ) -> Result<(), SystemError> {
646 let irq_data = desc_inner_guard.irq_data();
647
648 if !desc_inner_guard.common_data().status().affinity_managed() {
649 return irq_domain_manager().activate_irq(irq_data, false);
650 }
651
652 return Ok(());
653 }
654
655 /// 设置CPU亲和性并开启中断
irq_startup( &self, desc: &Arc<IrqDesc>, desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, resend: bool, force: bool, ) -> Result<(), SystemError>656 pub(super) fn irq_startup(
657 &self,
658 desc: &Arc<IrqDesc>,
659 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>,
660 resend: bool,
661 force: bool,
662 ) -> Result<(), SystemError> {
663 debug!(
664 "irq_startup: irq: {}, name: {:?}",
665 desc_inner_guard.irq_data().irq().data(),
666 desc_inner_guard.name()
667 );
668 let mut ret = Ok(());
669 let irq_data = desc_inner_guard.irq_data().clone();
670 let affinity = desc_inner_guard.common_data().affinity();
671 desc_inner_guard.set_depth(0);
672
673 if desc_inner_guard.common_data().status().started() {
674 self.irq_enable(desc_inner_guard);
675 } else {
676 match self.__irq_startup_managed(desc_inner_guard, &affinity, force) {
677 IrqStartupResult::Normal => {
678 if irq_data
679 .chip_info_read_irqsave()
680 .chip()
681 .flags()
682 .contains(IrqChipFlags::IRQCHIP_AFFINITY_PRE_STARTUP)
683 {
684 self.irq_setup_affinity(desc, desc_inner_guard).ok();
685 }
686
687 ret = self.__irq_startup(desc_inner_guard);
688
689 if !irq_data
690 .chip_info_read_irqsave()
691 .chip()
692 .flags()
693 .contains(IrqChipFlags::IRQCHIP_AFFINITY_PRE_STARTUP)
694 {
695 self.irq_setup_affinity(desc, desc_inner_guard).ok();
696 }
697 }
698 IrqStartupResult::Managed => {
699 self.irq_do_set_affinity(&irq_data, desc_inner_guard, &affinity, false)
700 .ok();
701 ret = self.__irq_startup(desc_inner_guard);
702 }
703 IrqStartupResult::Abort => {
704 desc_inner_guard
705 .common_data()
706 .insert_status(IrqStatus::IRQD_MANAGED_SHUTDOWN);
707 return Ok(());
708 }
709 }
710 }
711
712 if resend {
713 if let Err(e) = self.irq_check_and_resend(desc_inner_guard, false) {
714 error!(
715 "Failed to check and resend irq {}, error {:?}",
716 irq_data.irq().data(),
717 e
718 );
719 }
720 }
721
722 return ret;
723 }
724
irq_enable(&self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>)725 pub fn irq_enable(&self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) {
726 let common_data = desc_inner_guard.common_data();
727 if !common_data.status().disabled() {
728 self.unmask_irq(desc_inner_guard);
729 } else {
730 common_data.clear_disabled();
731
732 let chip = desc_inner_guard.irq_data().chip_info_read_irqsave().chip();
733
734 if let Err(e) = chip.irq_enable(desc_inner_guard.irq_data()) {
735 if e == SystemError::ENOSYS {
736 self.unmask_irq(desc_inner_guard);
737 }
738 error!(
739 "Failed to enable irq {} (name: {:?}), error {:?}",
740 desc_inner_guard.irq_data().irq().data(),
741 desc_inner_guard.name(),
742 e
743 );
744 } else {
745 common_data.clear_masked();
746 }
747 }
748 }
749
750 /// 自动设置中断的CPU亲和性
751 ///
752 ///
753 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c#589
irq_setup_affinity( &self, _desc: &Arc<IrqDesc>, desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, ) -> Result<(), SystemError>754 pub fn irq_setup_affinity(
755 &self,
756 _desc: &Arc<IrqDesc>,
757 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>,
758 ) -> Result<(), SystemError> {
759 let common_data = desc_inner_guard.common_data();
760 if !desc_inner_guard.can_set_affinity() {
761 return Ok(());
762 }
763
764 let mut to_set = IRQ_DEFAULT_AFFINITY.clone();
765 if common_data.status().affinity_managed()
766 || common_data.status().contains(IrqStatus::IRQD_AFFINITY_SET)
767 {
768 // FIXME: 要判断affinity跟已上线的CPU是否有交集
769
770 let irq_aff = common_data.affinity();
771 if irq_aff.is_empty() {
772 common_data.clear_status(IrqStatus::IRQD_AFFINITY_SET);
773 } else {
774 to_set = irq_aff;
775 }
776 }
777
778 // FIXME: 求to_set和在线CPU的交集
779
780 return self.irq_do_set_affinity(
781 desc_inner_guard.irq_data(),
782 desc_inner_guard,
783 &to_set,
784 false,
785 );
786 }
787
788 #[allow(dead_code)]
irq_set_affinity( &self, irq_data: &Arc<IrqData>, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, cpumask: &CpuMask, ) -> Result<(), SystemError>789 pub fn irq_set_affinity(
790 &self,
791 irq_data: &Arc<IrqData>,
792 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>,
793 cpumask: &CpuMask,
794 ) -> Result<(), SystemError> {
795 return self.irq_do_set_affinity(irq_data, desc_inner_guard, cpumask, false);
796 }
irq_do_set_affinity( &self, irq_data: &Arc<IrqData>, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, cpumask: &CpuMask, force: bool, ) -> Result<(), SystemError>797 fn irq_do_set_affinity(
798 &self,
799 irq_data: &Arc<IrqData>,
800 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>,
801 cpumask: &CpuMask,
802 force: bool,
803 ) -> Result<(), SystemError> {
804 let chip = irq_data.chip_info_read_irqsave().chip();
805 if !chip.can_set_affinity() {
806 return Err(SystemError::EINVAL);
807 }
808
809 // todo: 处理CPU中断隔离相关的逻辑
810
811 let common_data = desc_inner_guard.common_data();
812 let r = if force || !cpumask.is_empty() {
813 chip.irq_set_affinity(irq_data, cpumask, force)
814 } else {
815 return Err(SystemError::EINVAL);
816 };
817
818 let mut ret = Ok(());
819 if let Ok(rs) = r {
820 match rs {
821 IrqChipSetMaskResult::Success | IrqChipSetMaskResult::Done => {
822 common_data.set_affinity(cpumask.clone());
823 }
824 IrqChipSetMaskResult::NoChange => {
825
826 // irq_validate_effective_affinity(data);
827 // irq_set_thread_affinity(desc);
828 }
829 }
830 } else {
831 ret = Err(r.unwrap_err());
832 }
833
834 return ret;
835 }
836
__irq_startup( &self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, ) -> Result<(), SystemError>837 fn __irq_startup(
838 &self,
839 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>,
840 ) -> Result<(), SystemError> {
841 let common_data = desc_inner_guard.common_data();
842
843 if let Err(e) = desc_inner_guard
844 .irq_data()
845 .chip_info_read_irqsave()
846 .chip()
847 .irq_startup(desc_inner_guard.irq_data())
848 {
849 if e == SystemError::ENOSYS {
850 self.irq_enable(desc_inner_guard);
851 } else {
852 return Err(e);
853 }
854 } else {
855 common_data.clear_disabled();
856 common_data.clear_masked();
857 }
858
859 common_data.set_started();
860
861 return Ok(());
862 }
863
__irq_startup_managed( &self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, _affinity: &CpuMask, _force: bool, ) -> IrqStartupResult864 fn __irq_startup_managed(
865 &self,
866 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>,
867 _affinity: &CpuMask,
868 _force: bool,
869 ) -> IrqStartupResult {
870 let irq_data = desc_inner_guard.irq_data();
871 let common_data = desc_inner_guard.common_data();
872
873 if !common_data.status().affinity_managed() {
874 return IrqStartupResult::Normal;
875 }
876
877 common_data.clear_managed_shutdown();
878
879 /*
880 - 检查Affinity掩码是否包括所有的在线CPU。如果是，这意味着有代码试图在管理的中断上使用enable_irq()，
881 这可能是非法的。在这种情况下，如果force不是真值，函数会返回IRQ_STARTUP_ABORT，表示中断处理应该被放弃。
882 - 如果Affinity掩码中没有任何在线的CPU，那么中断请求是不可用的，因为没有任何CPU可以处理它。
883 在这种情况下，如果force不是真值，函数同样会返回IRQ_STARTUP_ABORT。
884 - 如果以上条件都不满足，尝试激活中断，并将其设置为管理模式。这是通过调用 `irq_domain_manager().activate_irq()` 函数来实现的。
885 如果这个调用失败，表示有保留的资源无法访问，函数会返回IRQ_STARTUP_ABORT。
886 - 如果一切顺利，函数会返回IRQ_STARTUP_MANAGED，表示中断已经被成功管理并激活。
887 */
888
889 // if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
890 // /*
891 // * Catch code which fiddles with enable_irq() on a managed
892 // * and potentially shutdown IRQ. Chained interrupt
893 // * installment or irq auto probing should not happen on
894 // * managed irqs either.
895 // */
896 // if (WARN_ON_ONCE(force))
897 // return IRQ_STARTUP_ABORT;
898 // /*
899 // * The interrupt was requested, but there is no online CPU
900 // * in it's affinity mask. Put it into managed shutdown
901 // * state and let the cpu hotplug mechanism start it up once
902 // * a CPU in the mask becomes available.
903 // */
904 // return IRQ_STARTUP_ABORT;
905 // }
906
907 let r = irq_domain_manager().activate_irq(irq_data, false);
908 if r.is_err() {
909 return IrqStartupResult::Abort;
910 }
911
912 return IrqStartupResult::Managed;
913 }
914
do_enable_irq( &self, _desc: Arc<IrqDesc>, _desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, ) -> Result<(), SystemError>915 pub fn do_enable_irq(
916 &self,
917 _desc: Arc<IrqDesc>,
918 _desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>,
919 ) -> Result<(), SystemError> {
920 // https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c?r=&mo=59252&fi=2138#776
921 todo!("do_enable_irq")
922 }
923
924 #[inline(never)]
do_set_irq_trigger( &self, _desc: Arc<IrqDesc>, desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>, mut trigger_type: IrqLineStatus, ) -> Result<(), SystemError>925 pub fn do_set_irq_trigger(
926 &self,
927 _desc: Arc<IrqDesc>,
928 desc_inner_guard: &mut SpinLockGuard<'_, InnerIrqDesc>,
929 mut trigger_type: IrqLineStatus,
930 ) -> Result<(), SystemError> {
931 let chip = desc_inner_guard.irq_data().chip_info_read_irqsave().chip();
932 let mut to_unmask = false;
933
934 if !chip.can_set_flow_type() {
935 // debug!(
936 // "No set_irq_type function for irq {}, chip {}",
937 // desc_inner_guard.irq_data().irq().data(),
938 // chip.name()
939 // );
940 return Ok(());
941 }
942
943 if chip.flags().contains(IrqChipFlags::IRQCHIP_SET_TYPE_MASKED) {
944 if !desc_inner_guard.common_data().status().masked() {
945 self.mask_irq(desc_inner_guard.irq_data());
946 }
947 if !desc_inner_guard.common_data().status().disabled() {
948 to_unmask = true;
949 }
950 }
951
952 trigger_type &= IrqLineStatus::IRQ_TYPE_SENSE_MASK;
953
954 let r = chip.irq_set_type(desc_inner_guard.irq_data(), trigger_type);
955 let ret;
956 if let Ok(rs) = r {
957 match rs {
958 IrqChipSetMaskResult::Success | IrqChipSetMaskResult::Done => {
959 let common_data = desc_inner_guard.common_data();
960 common_data.clear_status(IrqStatus::IRQD_TRIGGER_MASK);
961 let mut irqstatus = IrqStatus::empty();
962 irqstatus.set_trigger_type(trigger_type);
963 common_data.insert_status(irqstatus);
964 }
965 IrqChipSetMaskResult::NoChange => {
966 let flags = desc_inner_guard.common_data().trigger_type();
967 desc_inner_guard.set_trigger_type(flags);
968 desc_inner_guard
969 .common_data()
970 .clear_status(IrqStatus::IRQD_LEVEL);
971 desc_inner_guard.clear_level();
972
973 if !(flags & IrqLineStatus::IRQ_TYPE_LEVEL_MASK).is_empty() {
974 desc_inner_guard.set_level();
975 desc_inner_guard
976 .common_data()
977 .insert_status(IrqStatus::IRQD_LEVEL);
978 }
979 }
980 }
981
982 ret = Ok(());
983 } else {
984 error!(
985 "Failed to set irq {} trigger type to {:?} on irqchip {}, error {:?}",
986 desc_inner_guard.irq_data().irq().data(),
987 trigger_type,
988 chip.name(),
989 r
990 );
991
992 ret = Err(r.unwrap_err());
993 }
994
995 if to_unmask {
996 self.unmask_irq(desc_inner_guard);
997 }
998 return ret;
999 }
1000
irq_request_resources(&self, desc: Arc<IrqDesc>) -> Result<(), SystemError>1001 fn irq_request_resources(&self, desc: Arc<IrqDesc>) -> Result<(), SystemError> {
1002 let irq_data = desc.irq_data();
1003 let irq_chip = irq_data.chip_info_read_irqsave().chip();
1004 irq_chip.irq_request_resources(&irq_data)
1005 }
1006
1007 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/manage.c?r=&mo=59252&fi=2138#1448
setup_irq_thread( &self, _irq: IrqNumber, _action: &InnerIrqAction, _secondary: bool, ) -> Result<(), SystemError>1008 fn setup_irq_thread(
1009 &self,
1010 _irq: IrqNumber,
1011 _action: &InnerIrqAction,
1012 _secondary: bool,
1013 ) -> Result<(), SystemError> {
1014 // if secondary {
1015 // KernelThreadMechanism::create(func, name)
1016 // }
1017
1018 todo!("setup_irq_thread")
1019 }
1020
setup_forced_threading(&self, _action: &mut InnerIrqAction) -> Result<(), SystemError>1021 fn setup_forced_threading(&self, _action: &mut InnerIrqAction) -> Result<(), SystemError> {
1022 // todo: 处理强制线程化的逻辑，参考linux的`irq_setup_forced_threading()`
1023 return Ok(());
1024 }
1025
irq_clear_status_flags( &self, irq: IrqNumber, status: IrqLineStatus, ) -> Result<(), SystemError>1026 pub fn irq_clear_status_flags(
1027 &self,
1028 irq: IrqNumber,
1029 status: IrqLineStatus,
1030 ) -> Result<(), SystemError> {
1031 let desc = irq_desc_manager().lookup(irq).ok_or(SystemError::EINVAL)?;
1032 desc.modify_status(status, IrqLineStatus::empty());
1033 return Ok(());
1034 }
1035
1036 /// 屏蔽中断
mask_irq(&self, irq_data: &Arc<IrqData>)1037 pub(super) fn mask_irq(&self, irq_data: &Arc<IrqData>) {
1038 if irq_data.common_data().status().masked() {
1039 return;
1040 }
1041
1042 let chip = irq_data.chip_info_read_irqsave().chip();
1043 let r = chip.irq_mask(irq_data);
1044
1045 if r.is_ok() {
1046 irq_data.common_data().set_masked();
1047 }
1048 }
1049
1050 /// 解除屏蔽中断
unmask_irq(&self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>)1051 pub(super) fn unmask_irq(&self, desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) {
1052 if !desc_inner_guard.common_data().status().masked() {
1053 return;
1054 }
1055
1056 let r = desc_inner_guard
1057 .irq_data()
1058 .chip_info_read_irqsave()
1059 .chip()
1060 .irq_unmask(desc_inner_guard.irq_data());
1061
1062 if let Err(e) = r {
1063 if e != SystemError::ENOSYS {
1064 error!(
1065 "Failed to unmask irq {} on irqchip {}, error {:?}",
1066 desc_inner_guard.irq_data().irq().data(),
1067 desc_inner_guard
1068 .irq_data()
1069 .chip_info_read_irqsave()
1070 .chip()
1071 .name(),
1072 e
1073 );
1074 }
1075 } else {
1076 desc_inner_guard
1077 .common_data()
1078 .clear_status(IrqStatus::IRQD_IRQ_MASKED);
1079 }
1080 }
1081
1082 /// 释放使用request_irq分配的中断
1083 ///
1084 /// ## 参数
1085 ///
1086 /// - irq: 要释放的中断线
1087 /// - dev_id: 要释放的设备身份
1088 ///
1089 /// ## 返回
1090 ///
1091 /// 返回传递给request_irq的devname参数
1092 ///
1093 /// ## 说明
1094 ///
1095 /// 移除一个中断处理程序。处理程序被移除，如果该中断线不再被任何驱动程序使用，则会被禁用。
1096 ///
1097 /// 在共享IRQ的情况下，调用者必须确保在调用此功能之前，它在所驱动的卡上禁用了中断。
1098 ///
1099 /// ## 注意
1100 ///
1101 /// 此函数不可以在中断上下文中调用。
1102 ///
1103 /// 参考 https://code.dragonos.org.cn/xref/linux-6.6.21/kernel/irq/manage.c#2026
free_irq(&self, _irq: IrqNumber, _dev_id: Option<Arc<DeviceId>>)1104 pub fn free_irq(&self, _irq: IrqNumber, _dev_id: Option<Arc<DeviceId>>) {
1105 warn!("Unimplemented free_irq");
1106 }
1107 }
1108
1109 enum IrqStartupResult {
1110 Normal,
1111 Managed,
1112 Abort,
1113 }
1114 /// 默认的初级中断处理函数
1115 ///
1116 /// 该处理函数仅仅返回`WakeThread`，即唤醒中断线程
1117 #[derive(Debug)]
1118 struct DefaultPrimaryIrqHandler;
1119
1120 impl IrqHandler for DefaultPrimaryIrqHandler {
handle( &self, _irq: IrqNumber, _static_data: Option<&dyn IrqHandlerData>, _dynamic_data: Option<Arc<dyn IrqHandlerData>>, ) -> Result<IrqReturn, SystemError>1121 fn handle(
1122 &self,
1123 _irq: IrqNumber,
1124 _static_data: Option<&dyn IrqHandlerData>,
1125 _dynamic_data: Option<Arc<dyn IrqHandlerData>>,
1126 ) -> Result<IrqReturn, SystemError> {
1127 return Ok(IrqReturn::WakeThread);
1128 }
1129 }
1130
1131 /// Primary handler for nested threaded interrupts.
1132 /// Should never be called.
1133 #[derive(Debug)]
1134 struct IrqNestedPrimaryHandler;
1135
1136 impl IrqHandler for IrqNestedPrimaryHandler {
handle( &self, irq: IrqNumber, _static_data: Option<&dyn IrqHandlerData>, _dynamic_data: Option<Arc<dyn IrqHandlerData>>, ) -> Result<IrqReturn, SystemError>1137 fn handle(
1138 &self,
1139 irq: IrqNumber,
1140 _static_data: Option<&dyn IrqHandlerData>,
1141 _dynamic_data: Option<Arc<dyn IrqHandlerData>>,
1142 ) -> Result<IrqReturn, SystemError> {
1143 warn!("Primary handler called for nested irq {}", irq.data());
1144 return Ok(IrqReturn::NotHandled);
1145 }
1146 }
1147