1 use core::{
2 any::Any,
3 fmt::Debug,
4 sync::atomic::{AtomicI64, Ordering},
5 };
6
7 use alloc::{
8 collections::{btree_map, BTreeMap},
9 string::{String, ToString},
10 sync::{Arc, Weak},
11 vec::Vec,
12 };
13 use system_error::SystemError;
14
15 use crate::{
16 arch::{interrupt::TrapFrame, CurrentIrqArch},
17 driver::base::{
18 device::DeviceId,
19 kobject::{KObjType, KObject, KObjectState, LockedKObjectState},
20 kset::KSet,
21 },
22 filesystem::kernfs::KernFSInode,
23 libs::{
24 cpumask::CpuMask,
25 mutex::{Mutex, MutexGuard},
26 rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard},
27 spinlock::{SpinLock, SpinLockGuard},
28 },
29 mm::percpu::PerCpuVar,
30 process::ProcessControlBlock,
31 sched::completion::Completion,
32 smp::cpu::smp_cpu_manager,
33 };
34
35 use super::{
36 dummychip::no_irq_chip,
37 handle::bad_irq_handler,
38 irqchip::IrqChip,
39 irqdata::{IrqCommonData, IrqData, IrqHandlerData, IrqLineStatus, IrqStatus},
40 irqdomain::{irq_domain_manager, IrqDomain},
41 sysfs::{irq_sysfs_del, IrqKObjType},
42 HardwareIrqNumber, InterruptArch, IrqNumber,
43 };
44
45 /// 中断流处理程序
46 pub trait IrqFlowHandler: Debug + Send + Sync + Any {
handle(&self, irq_desc: &Arc<IrqDesc>, trap_frame: &mut TrapFrame)47 fn handle(&self, irq_desc: &Arc<IrqDesc>, trap_frame: &mut TrapFrame);
48 }
49
50 /// 中断处理程序
51 pub trait IrqHandler: Debug + Send + Sync + Any {
handle( &self, irq: IrqNumber, static_data: Option<&dyn IrqHandlerData>, dynamic_data: Option<Arc<dyn IrqHandlerData>>, ) -> Result<IrqReturn, SystemError>52 fn handle(
53 &self,
54 irq: IrqNumber,
55 static_data: Option<&dyn IrqHandlerData>,
56 dynamic_data: Option<Arc<dyn IrqHandlerData>>,
57 ) -> Result<IrqReturn, SystemError>;
58 }
59
60 /// 中断处理函数返回值
61 ///
62 /// 用于指示中断处理函数是否处理了中断
63 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
64 pub enum IrqReturn {
65 /// 中断未被处理
66 NotHandled,
67 /// 中断已被处理
68 Handled,
69 /// 中断已被处理，并且需要唤醒中断线程
70 WakeThread,
71 }
72
73 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/include/linux/irqdesc.h#55
74 #[derive(Debug)]
75 pub struct IrqDesc {
76 inner: SpinLock<InnerIrqDesc>,
77
78 handler: RwLock<Option<&'static dyn IrqFlowHandler>>,
79 /// 一个用于串行化 request_irq()和free_irq() 的互斥锁
80 request_mutex: Mutex<()>,
81 kobj_state: LockedKObjectState,
82 /// 当前描述符内正在运行的中断线程数
83 threads_active: AtomicI64,
84 }
85
86 impl IrqDesc {
87 #[inline(never)]
new(irq: IrqNumber, name: Option<String>, irqd_flags: IrqStatus) -> Arc<Self>88 pub fn new(irq: IrqNumber, name: Option<String>, irqd_flags: IrqStatus) -> Arc<Self> {
89 // 初始化的过程参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/irqdesc.c#392
90 let common_data = Arc::new(IrqCommonData::new());
91 let irq_data = Arc::new(IrqData::new(
92 irq,
93 HardwareIrqNumber::new(irq.data()),
94 common_data.clone(),
95 no_irq_chip(),
96 ));
97
98 irq_data.irqd_set(IrqStatus::IRQD_IRQ_DISABLED);
99 common_data.insert_status(IrqStatus::IRQD_IRQ_MASKED);
100
101 let irq_desc = IrqDesc {
102 inner: SpinLock::new(InnerIrqDesc {
103 percpu_affinity: None,
104 percpu_enabled: None,
105 common_data,
106 irq_data,
107 desc_internal_state: IrqDescState::empty(),
108 line_status: IrqLineStatus::empty(),
109 actions: Vec::new(),
110 name,
111 parent_irq: None,
112 depth: 1,
113 wake_depth: 0,
114 kern_inode: None,
115 kset: None,
116 parent_kobj: None,
117 threads_oneshot: 0,
118 }),
119 request_mutex: Mutex::new(()),
120 handler: RwLock::new(None),
121 kobj_state: LockedKObjectState::new(Some(KObjectState::INITIALIZED)),
122 threads_active: AtomicI64::new(0),
123 };
124 let irq_desc = Arc::new(irq_desc);
125 irq_desc.irq_data().set_irq_desc(Arc::downgrade(&irq_desc));
126 irq_desc.set_handler(bad_irq_handler());
127 irq_desc.inner().irq_data.irqd_set(irqd_flags);
128
129 return irq_desc;
130 }
131
132 /// 返回当前活跃的中断线程数量
133 #[allow(dead_code)]
threads_active(&self) -> i64134 pub fn threads_active(&self) -> i64 {
135 self.threads_active.load(Ordering::SeqCst)
136 }
137
138 /// 增加当前活跃的中断线程数量, 返回增加前的值
inc_threads_active(&self) -> i64139 pub fn inc_threads_active(&self) -> i64 {
140 self.threads_active.fetch_add(1, Ordering::SeqCst)
141 }
142
143 /// 减少当前活跃的中断线程数量, 返回减少前的值
144 #[allow(dead_code)]
dec_threads_active(&self) -> i64145 pub fn dec_threads_active(&self) -> i64 {
146 self.threads_active.fetch_sub(1, Ordering::SeqCst)
147 }
148
set_handler(&self, handler: &'static dyn IrqFlowHandler)149 pub fn set_handler(&self, handler: &'static dyn IrqFlowHandler) {
150 self.chip_bus_lock();
151 let mut guard = self.handler.write_irqsave();
152 *guard = Some(handler);
153 self.chip_bus_sync_unlock();
154 }
155
156 /// 设置中断处理程序（不对desc->inner）
157 ///
158 ///
159 /// ## Safety
160 ///
161 /// 需要保证irq_data和chip是当前irqdesc的
set_handler_no_lock_inner( &self, handler: &'static dyn IrqFlowHandler, irq_data: &Arc<IrqData>, chip: &Arc<dyn IrqChip>, )162 pub fn set_handler_no_lock_inner(
163 &self,
164 handler: &'static dyn IrqFlowHandler,
165 irq_data: &Arc<IrqData>,
166 chip: &Arc<dyn IrqChip>,
167 ) {
168 chip.irq_bus_lock(irq_data).ok();
169 let mut guard = self.handler.write_irqsave();
170 *guard = Some(handler);
171 chip.irq_bus_sync_unlock(irq_data).ok();
172 }
173
handler(&self) -> Option<&'static dyn IrqFlowHandler>174 pub fn handler(&self) -> Option<&'static dyn IrqFlowHandler> {
175 let guard = self.handler.read_irqsave();
176 *guard
177 }
178
inner(&self) -> SpinLockGuard<InnerIrqDesc>179 pub fn inner(&self) -> SpinLockGuard<InnerIrqDesc> {
180 self.inner.lock_irqsave()
181 }
182
actions(&self) -> Vec<Arc<IrqAction>>183 pub fn actions(&self) -> Vec<Arc<IrqAction>> {
184 self.inner().actions.clone()
185 }
186
187 /// 对中断请求过程加锁
request_mutex_lock(&self) -> MutexGuard<()>188 pub fn request_mutex_lock(&self) -> MutexGuard<()> {
189 self.request_mutex.lock()
190 }
191
irq(&self) -> IrqNumber192 pub fn irq(&self) -> IrqNumber {
193 self.inner().irq_data.irq()
194 }
195
hardware_irq(&self) -> HardwareIrqNumber196 pub fn hardware_irq(&self) -> HardwareIrqNumber {
197 self.inner().irq_data.hardware_irq()
198 }
199
irq_data(&self) -> Arc<IrqData>200 pub fn irq_data(&self) -> Arc<IrqData> {
201 self.inner().irq_data.clone()
202 }
203
204 /// 标记当前irq描述符已经被添加到sysfs
mark_in_sysfs(&self)205 pub fn mark_in_sysfs(&self) {
206 self.inner()
207 .desc_internal_state
208 .insert(IrqDescState::IRQS_SYSFS);
209 }
210
mark_not_in_sysfs(&self)211 pub fn mark_not_in_sysfs(&self) {
212 self.inner()
213 .desc_internal_state
214 .remove(IrqDescState::IRQS_SYSFS);
215 }
216
217 /// 判断当前描述符是否已经添加到了sysfs
in_sysfs(&self) -> bool218 pub fn in_sysfs(&self) -> bool {
219 self.inner()
220 .desc_internal_state
221 .contains(IrqDescState::IRQS_SYSFS)
222 }
223
name(&self) -> Option<String>224 pub fn name(&self) -> Option<String> {
225 self.inner().name.clone()
226 }
227
can_request(&self) -> bool228 pub fn can_request(&self) -> bool {
229 self.inner().can_request()
230 }
231
232 #[allow(dead_code)]
set_norequest(&self)233 pub fn set_norequest(&self) {
234 self.inner().set_norequest();
235 }
236
237 #[allow(dead_code)]
clear_norequest(&self)238 pub fn clear_norequest(&self) {
239 self.inner().clear_norequest();
240 }
241
nested_thread(&self) -> bool242 pub fn nested_thread(&self) -> bool {
243 self.inner().nested_thread()
244 }
245
246 /// 中断是否可以线程化
can_thread(&self) -> bool247 pub fn can_thread(&self) -> bool {
248 !self
249 .inner()
250 .line_status
251 .contains(IrqLineStatus::IRQ_NOTHREAD)
252 }
253
chip_bus_lock(&self)254 pub fn chip_bus_lock(&self) {
255 let irq_data = self.inner().irq_data.clone();
256 irq_data
257 .chip_info_read_irqsave()
258 .chip()
259 .irq_bus_lock(&irq_data)
260 .ok();
261 }
262
263 /// 同步释放低速总线锁
264 ///
265 /// ## 锁
266 ///
267 /// 进入此函数时，必须持有低速总线锁，并且desc的inner锁和irqdata的inner锁
268 /// 必须已经释放。否则将死锁。
chip_bus_sync_unlock(&self)269 pub fn chip_bus_sync_unlock(&self) {
270 let irq_data = self.inner().irq_data.clone();
271 irq_data
272 .chip_info_write_irqsave()
273 .chip()
274 .irq_bus_sync_unlock(&irq_data)
275 .ok();
276 }
277
set_percpu_devid_flags(&self)278 pub fn set_percpu_devid_flags(&self) {
279 self.modify_status(
280 IrqLineStatus::empty(),
281 IrqLineStatus::IRQ_NOAUTOEN
282 | IrqLineStatus::IRQ_PER_CPU
283 | IrqLineStatus::IRQ_NOTHREAD
284 | IrqLineStatus::IRQ_NOPROBE
285 | IrqLineStatus::IRQ_PER_CPU_DEVID,
286 );
287 }
288
289 #[allow(dead_code)]
set_probe(&self)290 pub fn set_probe(&self) {
291 self.modify_status(IrqLineStatus::IRQ_NOPROBE, IrqLineStatus::empty());
292 }
293
294 #[allow(dead_code)]
set_noprobe(&self)295 pub fn set_noprobe(&self) {
296 self.modify_status(IrqLineStatus::empty(), IrqLineStatus::IRQ_NOPROBE);
297 }
298
modify_status(&self, clear: IrqLineStatus, set: IrqLineStatus)299 pub fn modify_status(&self, clear: IrqLineStatus, set: IrqLineStatus) {
300 let mut desc_guard = self.inner();
301 desc_guard.line_status.remove(clear);
302 desc_guard.line_status.insert(set);
303
304 let mut trigger = desc_guard.common_data().trigger_type();
305
306 desc_guard.common_data().clear_status(
307 IrqStatus::IRQD_NO_BALANCING
308 | IrqStatus::IRQD_PER_CPU
309 | IrqStatus::IRQD_TRIGGER_MASK
310 | IrqStatus::IRQD_LEVEL
311 | IrqStatus::IRQD_MOVE_PCNTXT,
312 );
313
314 if desc_guard
315 .line_status
316 .contains(IrqLineStatus::IRQ_NO_BALANCING)
317 {
318 desc_guard
319 .common_data()
320 .insert_status(IrqStatus::IRQD_NO_BALANCING);
321 }
322
323 if desc_guard.line_status.contains(IrqLineStatus::IRQ_PER_CPU) {
324 desc_guard
325 .common_data()
326 .insert_status(IrqStatus::IRQD_PER_CPU);
327 }
328
329 if desc_guard
330 .line_status
331 .contains(IrqLineStatus::IRQ_MOVE_PCNTXT)
332 {
333 desc_guard
334 .common_data()
335 .insert_status(IrqStatus::IRQD_MOVE_PCNTXT);
336 }
337
338 if desc_guard.line_status.is_level_type() {
339 desc_guard
340 .common_data()
341 .insert_status(IrqStatus::IRQD_LEVEL);
342 }
343
344 let tmp = desc_guard.line_status.trigger_type();
345
346 if tmp != IrqLineStatus::IRQ_TYPE_NONE {
347 trigger = tmp;
348 }
349
350 desc_guard.common_data().set_trigger_type(trigger);
351 }
352 }
353
354 #[allow(dead_code)]
355 #[derive(Debug)]
356 pub struct InnerIrqDesc {
357 /// per irq and chip data passed down to chip functions
358 common_data: Arc<IrqCommonData>,
359 irq_data: Arc<IrqData>,
360 actions: Vec<Arc<IrqAction>>,
361 name: Option<String>,
362 parent_irq: Option<IrqNumber>,
363 /// nested irq disables
364 depth: u32,
365 /// nested wake enables
366 wake_depth: u32,
367 desc_internal_state: IrqDescState,
368 /// 中断线的状态
369 line_status: IrqLineStatus,
370
371 kern_inode: Option<Arc<KernFSInode>>,
372 kset: Option<Arc<KSet>>,
373 parent_kobj: Option<Weak<dyn KObject>>,
374 /// per-cpu enabled mask
375 percpu_enabled: Option<CpuMask>,
376 /// per-cpu affinity
377 percpu_affinity: Option<CpuMask>,
378 // wait_for_threads: EventWaitQueue
379 /// bitfield to handle shared oneshot threads
380 threads_oneshot: u64,
381 }
382
383 impl InnerIrqDesc {
name(&self) -> Option<&String>384 pub fn name(&self) -> Option<&String> {
385 self.name.as_ref()
386 }
387
388 #[allow(dead_code)]
set_name(&mut self, name: Option<String>)389 pub fn set_name(&mut self, name: Option<String>) {
390 self.name = name;
391 }
392
can_request(&self) -> bool393 pub fn can_request(&self) -> bool {
394 !self.line_status.contains(IrqLineStatus::IRQ_NOREQUEST)
395 }
396
397 #[allow(dead_code)]
set_norequest(&mut self)398 pub fn set_norequest(&mut self) {
399 self.line_status.insert(IrqLineStatus::IRQ_NOREQUEST);
400 }
401
402 #[allow(dead_code)]
clear_norequest(&mut self)403 pub fn clear_norequest(&mut self) {
404 self.line_status.remove(IrqLineStatus::IRQ_NOREQUEST);
405 }
406
407 #[allow(dead_code)]
set_noprobe(&mut self)408 pub fn set_noprobe(&mut self) {
409 self.line_status.insert(IrqLineStatus::IRQ_NOPROBE);
410 }
411
412 #[allow(dead_code)]
clear_noprobe(&mut self)413 pub fn clear_noprobe(&mut self) {
414 self.line_status.remove(IrqLineStatus::IRQ_NOPROBE);
415 }
416
set_nothread(&mut self)417 pub fn set_nothread(&mut self) {
418 self.line_status.insert(IrqLineStatus::IRQ_NOTHREAD);
419 }
420
421 #[allow(dead_code)]
clear_nothread(&mut self)422 pub fn clear_nothread(&mut self) {
423 self.line_status.remove(IrqLineStatus::IRQ_NOTHREAD);
424 }
425
nested_thread(&self) -> bool426 pub fn nested_thread(&self) -> bool {
427 self.line_status.contains(IrqLineStatus::IRQ_NESTED_THREAD)
428 }
429
line_status_set_per_cpu(&mut self)430 pub fn line_status_set_per_cpu(&mut self) {
431 self.line_status.insert(IrqLineStatus::IRQ_PER_CPU);
432 }
433
434 #[allow(dead_code)]
line_status_clear_per_cpu(&mut self)435 pub fn line_status_clear_per_cpu(&mut self) {
436 self.line_status.remove(IrqLineStatus::IRQ_PER_CPU);
437 }
438
439 #[allow(dead_code)]
line_status(&self) -> &IrqLineStatus440 pub fn line_status(&self) -> &IrqLineStatus {
441 &self.line_status
442 }
443
line_status_set_no_debug(&mut self)444 pub fn line_status_set_no_debug(&mut self) {
445 self.line_status.insert(IrqLineStatus::IRQ_NO_BALANCING);
446 }
447
448 #[allow(dead_code)]
line_status_clear_no_debug(&mut self)449 pub fn line_status_clear_no_debug(&mut self) {
450 self.line_status.remove(IrqLineStatus::IRQ_NO_BALANCING);
451 }
452
can_autoenable(&self) -> bool453 pub fn can_autoenable(&self) -> bool {
454 !self.line_status.contains(IrqLineStatus::IRQ_NOAUTOEN)
455 }
456
457 #[allow(dead_code)]
can_thread(&self) -> bool458 pub fn can_thread(&self) -> bool {
459 !self.line_status.contains(IrqLineStatus::IRQ_NOTHREAD)
460 }
461
threads_oneshot(&self) -> u64462 pub fn threads_oneshot(&self) -> u64 {
463 self.threads_oneshot
464 }
465
466 /// 中断是否可以设置CPU亲和性
can_set_affinity(&self) -> bool467 pub fn can_set_affinity(&self) -> bool {
468 if !self.common_data.status().can_balance()
469 || !self
470 .irq_data()
471 .chip_info_read_irqsave()
472 .chip()
473 .can_set_affinity()
474 {
475 return false;
476 }
477
478 return true;
479 }
480
actions(&self) -> &Vec<Arc<IrqAction>>481 pub fn actions(&self) -> &Vec<Arc<IrqAction>> {
482 &self.actions
483 }
484
add_action(&mut self, action: Arc<IrqAction>)485 pub fn add_action(&mut self, action: Arc<IrqAction>) {
486 self.actions.push(action);
487 }
488
clear_actions(&mut self)489 pub fn clear_actions(&mut self) {
490 self.actions.clear();
491 }
492
493 #[allow(dead_code)]
remove_action(&mut self, action: &Arc<IrqAction>)494 pub fn remove_action(&mut self, action: &Arc<IrqAction>) {
495 self.actions.retain(|a| !Arc::ptr_eq(a, action));
496 }
497
internal_state(&self) -> &IrqDescState498 pub fn internal_state(&self) -> &IrqDescState {
499 &self.desc_internal_state
500 }
501
internal_state_mut(&mut self) -> &mut IrqDescState502 pub(super) fn internal_state_mut(&mut self) -> &mut IrqDescState {
503 &mut self.desc_internal_state
504 }
505
irq_data(&self) -> &Arc<IrqData>506 pub fn irq_data(&self) -> &Arc<IrqData> {
507 &self.irq_data
508 }
509
common_data(&self) -> &Arc<IrqCommonData>510 pub fn common_data(&self) -> &Arc<IrqCommonData> {
511 &self.common_data
512 }
513
514 #[allow(dead_code)]
depth(&self) -> u32515 pub fn depth(&self) -> u32 {
516 self.depth
517 }
518
519 #[allow(dead_code)]
wake_depth(&self) -> u32520 pub fn wake_depth(&self) -> u32 {
521 self.wake_depth
522 }
523
524 #[allow(dead_code)]
set_depth(&mut self, depth: u32)525 pub fn set_depth(&mut self, depth: u32) {
526 self.depth = depth;
527 }
528
set_trigger_type(&mut self, trigger: IrqLineStatus)529 pub fn set_trigger_type(&mut self, trigger: IrqLineStatus) {
530 self.line_status.remove(IrqLineStatus::IRQ_TYPE_SENSE_MASK);
531 self.line_status
532 .insert(trigger & IrqLineStatus::IRQ_TYPE_SENSE_MASK);
533 }
534
clear_level(&mut self)535 pub fn clear_level(&mut self) {
536 self.line_status.remove(IrqLineStatus::IRQ_LEVEL);
537 }
538
set_level(&mut self)539 pub fn set_level(&mut self) {
540 self.line_status.insert(IrqLineStatus::IRQ_LEVEL);
541 }
542
percpu_enabled(&self) -> &Option<CpuMask>543 pub fn percpu_enabled(&self) -> &Option<CpuMask> {
544 &self.percpu_enabled
545 }
546
percpu_enabled_mut(&mut self) -> &mut Option<CpuMask>547 pub fn percpu_enabled_mut(&mut self) -> &mut Option<CpuMask> {
548 &mut self.percpu_enabled
549 }
550
551 #[allow(dead_code)]
percpu_affinity(&self) -> &Option<CpuMask>552 pub fn percpu_affinity(&self) -> &Option<CpuMask> {
553 &self.percpu_affinity
554 }
555
percpu_affinity_mut(&mut self) -> &mut Option<CpuMask>556 pub fn percpu_affinity_mut(&mut self) -> &mut Option<CpuMask> {
557 &mut self.percpu_affinity
558 }
559 }
560
561 impl KObject for IrqDesc {
as_any_ref(&self) -> &dyn Any562 fn as_any_ref(&self) -> &dyn Any {
563 self
564 }
565
set_inode(&self, inode: Option<Arc<KernFSInode>>)566 fn set_inode(&self, inode: Option<Arc<KernFSInode>>) {
567 self.inner().kern_inode = inode;
568 }
569
inode(&self) -> Option<Arc<KernFSInode>>570 fn inode(&self) -> Option<Arc<KernFSInode>> {
571 self.inner().kern_inode.clone()
572 }
573
parent(&self) -> Option<Weak<dyn KObject>>574 fn parent(&self) -> Option<Weak<dyn KObject>> {
575 self.inner().parent_kobj.clone()
576 }
577
set_parent(&self, parent: Option<Weak<dyn KObject>>)578 fn set_parent(&self, parent: Option<Weak<dyn KObject>>) {
579 self.inner().parent_kobj = parent;
580 }
581
kset(&self) -> Option<Arc<KSet>>582 fn kset(&self) -> Option<Arc<KSet>> {
583 self.inner().kset.clone()
584 }
585
set_kset(&self, kset: Option<Arc<KSet>>)586 fn set_kset(&self, kset: Option<Arc<KSet>>) {
587 self.inner().kset = kset;
588 }
589
kobj_type(&self) -> Option<&'static dyn KObjType>590 fn kobj_type(&self) -> Option<&'static dyn KObjType> {
591 Some(&IrqKObjType)
592 }
593
set_kobj_type(&self, _ktype: Option<&'static dyn KObjType>)594 fn set_kobj_type(&self, _ktype: Option<&'static dyn KObjType>) {}
595
name(&self) -> String596 fn name(&self) -> String {
597 self.inner().irq_data.irq().data().to_string()
598 }
599
set_name(&self, _name: String)600 fn set_name(&self, _name: String) {}
601
kobj_state(&self) -> RwLockReadGuard<KObjectState>602 fn kobj_state(&self) -> RwLockReadGuard<KObjectState> {
603 self.kobj_state.read()
604 }
605
kobj_state_mut(&self) -> RwLockWriteGuard<KObjectState>606 fn kobj_state_mut(&self) -> RwLockWriteGuard<KObjectState> {
607 self.kobj_state.write()
608 }
609
set_kobj_state(&self, state: KObjectState)610 fn set_kobj_state(&self, state: KObjectState) {
611 *self.kobj_state_mut() = state;
612 }
613 }
614
615 bitflags! {
616 /// Bit masks for desc->desc_internal_state
617 pub struct IrqDescState: u32 {
618 /// autodetection in progress
619 const IRQS_AUTODETECT = 0x00000001;
620 /// was disabled due to spurious interrupt detection
621 const IRQS_SPURIOUS_DISABLED = 0x00000002;
622 /// polling in progress
623 const IRQS_POLL_INPROGRESS = 0x00000008;
624 /// irq is not unmasked in primary handler
625 const IRQS_ONESHOT = 0x00000020;
626 /// irq is replayed
627 const IRQS_REPLAY = 0x00000040;
628 /// irq is waiting
629 const IRQS_WAITING = 0x00000080;
630 /// irq is pending and replayed later
631 const IRQS_PENDING = 0x00000200;
632 /// irq is suspended
633 const IRQS_SUSPENDED = 0x00000800;
634 /// irq line is used to deliver NMIs
635 const IRQS_NMI = 0x00002000;
636 /// descriptor has been added to sysfs
637 const IRQS_SYSFS = 0x00004000;
638 }
639 }
640
641 /// 每个中断的响应动作的描述符
642 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/include/linux/interrupt.h#118
643 #[allow(dead_code)]
644 #[derive(Debug)]
645 pub struct IrqAction {
646 inner: SpinLock<InnerIrqAction>,
647 /// 用于等待线程被创建的完成量
648 thread_completion: Completion,
649 }
650
651 impl IrqAction {
652 #[allow(dead_code)]
new( irq: IrqNumber, name: String, handler: Option<&'static dyn IrqHandler>, thread_fn: Option<&'static dyn IrqHandler>, ) -> Arc<Self>653 pub fn new(
654 irq: IrqNumber,
655 name: String,
656 handler: Option<&'static dyn IrqHandler>,
657 thread_fn: Option<&'static dyn IrqHandler>,
658 ) -> Arc<Self> {
659 let action: IrqAction = IrqAction {
660 inner: SpinLock::new(InnerIrqAction {
661 dev_id: None,
662 per_cpu_dev_id: None,
663 handler,
664 thread_fn,
665 thread: None,
666 secondary: None,
667 irq,
668 flags: IrqHandleFlags::empty(),
669 name,
670 thread_flags: ThreadedHandlerFlags::empty(),
671 }),
672 thread_completion: Completion::new(),
673 };
674
675 return Arc::new(action);
676 }
677
inner(&self) -> SpinLockGuard<InnerIrqAction>678 pub fn inner(&self) -> SpinLockGuard<InnerIrqAction> {
679 self.inner.lock_irqsave()
680 }
681
thread_completion(&self) -> &Completion682 pub fn thread_completion(&self) -> &Completion {
683 &self.thread_completion
684 }
685 }
686
687 #[allow(dead_code)]
688 #[derive(Debug)]
689 pub struct InnerIrqAction {
690 /// cookie to identify the device
691 dev_id: Option<Arc<DeviceId>>,
692 /// cookie to identify the device (per cpu)
693 per_cpu_dev_id: Option<PerCpuVar<Arc<DeviceId>>>,
694 /// 中断处理程序
695 handler: Option<&'static dyn IrqHandler>,
696 /// interrupt handler function for threaded interrupts
697 thread_fn: Option<&'static dyn IrqHandler>,
698 /// thread pointer for threaded interrupts
699 thread: Option<Arc<ProcessControlBlock>>,
700 /// pointer to secondary irqaction (force threading)
701 secondary: Option<Arc<IrqAction>>,
702 /// 中断号
703 irq: IrqNumber,
704 flags: IrqHandleFlags,
705 /// 中断线程的标志
706 thread_flags: ThreadedHandlerFlags,
707 /// name of the device
708 name: String,
709 }
710
711 impl InnerIrqAction {
dev_id(&self) -> &Option<Arc<DeviceId>>712 pub fn dev_id(&self) -> &Option<Arc<DeviceId>> {
713 &self.dev_id
714 }
715
dev_id_mut(&mut self) -> &mut Option<Arc<DeviceId>>716 pub fn dev_id_mut(&mut self) -> &mut Option<Arc<DeviceId>> {
717 &mut self.dev_id
718 }
719
per_cpu_dev_id(&self) -> Option<&Arc<DeviceId>>720 pub fn per_cpu_dev_id(&self) -> Option<&Arc<DeviceId>> {
721 self.per_cpu_dev_id.as_ref().map(|v| v.get())
722 }
723
724 #[allow(dead_code)]
per_cpu_dev_id_mut(&mut self) -> Option<&mut Arc<DeviceId>>725 pub fn per_cpu_dev_id_mut(&mut self) -> Option<&mut Arc<DeviceId>> {
726 self.per_cpu_dev_id.as_mut().map(|v| v.get_mut())
727 }
728
handler(&self) -> Option<&'static dyn IrqHandler>729 pub fn handler(&self) -> Option<&'static dyn IrqHandler> {
730 self.handler
731 }
732
set_handler(&mut self, handler: Option<&'static dyn IrqHandler>)733 pub fn set_handler(&mut self, handler: Option<&'static dyn IrqHandler>) {
734 self.handler = handler;
735 }
736
thread_fn(&self) -> Option<&'static dyn IrqHandler>737 pub fn thread_fn(&self) -> Option<&'static dyn IrqHandler> {
738 self.thread_fn
739 }
740
thread(&self) -> Option<Arc<ProcessControlBlock>>741 pub fn thread(&self) -> Option<Arc<ProcessControlBlock>> {
742 self.thread.clone()
743 }
744
set_thread(&mut self, thread: Option<Arc<ProcessControlBlock>>)745 pub fn set_thread(&mut self, thread: Option<Arc<ProcessControlBlock>>) {
746 self.thread = thread;
747 }
748
749 #[allow(dead_code)]
thread_flags(&self) -> &ThreadedHandlerFlags750 pub fn thread_flags(&self) -> &ThreadedHandlerFlags {
751 &self.thread_flags
752 }
753
thread_flags_mut(&mut self) -> &mut ThreadedHandlerFlags754 pub fn thread_flags_mut(&mut self) -> &mut ThreadedHandlerFlags {
755 &mut self.thread_flags
756 }
757
secondary(&self) -> Option<Arc<IrqAction>>758 pub fn secondary(&self) -> Option<Arc<IrqAction>> {
759 self.secondary.clone()
760 }
761
762 #[allow(dead_code)]
irq(&self) -> IrqNumber763 pub fn irq(&self) -> IrqNumber {
764 self.irq
765 }
766
767 #[allow(dead_code)]
set_irq(&mut self, irq: IrqNumber)768 pub fn set_irq(&mut self, irq: IrqNumber) {
769 self.irq = irq;
770 }
771
flags(&self) -> &IrqHandleFlags772 pub fn flags(&self) -> &IrqHandleFlags {
773 &self.flags
774 }
775
flags_mut(&mut self) -> &mut IrqHandleFlags776 pub fn flags_mut(&mut self) -> &mut IrqHandleFlags {
777 &mut self.flags
778 }
779
name(&self) -> &String780 pub fn name(&self) -> &String {
781 &self.name
782 }
783 }
784
785 bitflags! {
786 /// 这些标志由线程处理程序使用
787 pub struct ThreadedHandlerFlags: u32 {
788 /// IRQTF_RUNTHREAD - 表示应运行中断处理程序线程
789 const IRQTF_RUNTHREAD = 1 << 0;
790 /// IRQTF_WARNED - 已打印警告 "IRQ_WAKE_THREAD w/o thread_fn"
791 const IRQTF_WARNED = 1 << 1;
792 /// IRQTF_AFFINITY - 请求irq线程调整亲和性
793 const IRQTF_AFFINITY = 1 << 2;
794 /// IRQTF_FORCED_THREAD - irq操作被强制线程化
795 const IRQTF_FORCED_THREAD = 1 << 3;
796 /// IRQTF_READY - 表示irq线程已准备就绪
797 const IRQTF_READY = 1 << 4;
798 }
799 }
800
801 /// Implements the `ThreadedHandlerFlags` structure.
802 impl ThreadedHandlerFlags {
803 /// 在 `ThreadedHandlerFlags` 结构中测试并设置特定的位。
804 ///
805 /// # 参数
806 ///
807 /// * `bit` - 要测试并设置的位。
808 ///
809 /// # 返回
810 ///
811 /// 如果操作前该位已被设置，则返回 `true`，否则返回 `false`。
test_and_set_bit(&mut self, bit: ThreadedHandlerFlags) -> bool812 pub fn test_and_set_bit(&mut self, bit: ThreadedHandlerFlags) -> bool {
813 let res = (self.bits & bit.bits) != 0;
814 self.bits |= bit.bits;
815 return res;
816 }
817 }
818
819 // 定义IrqFlags位标志
820
821 bitflags! {
822 /// 这些标志仅由内核在中断处理例程中使用。
823 #[allow(clippy::bad_bit_mask)]
824 pub struct IrqHandleFlags: u32 {
825
826 const IRQF_TRIGGER_NONE = IrqLineStatus::IRQ_TYPE_NONE.bits();
827 const IRQF_TRIGGER_RISING = IrqLineStatus::IRQ_TYPE_EDGE_RISING.bits();
828 const IRQF_TRIGGER_FALLING = IrqLineStatus::IRQ_TYPE_EDGE_FALLING.bits();
829 const IRQF_TRIGGER_HIGH = IrqLineStatus::IRQ_TYPE_LEVEL_HIGH.bits();
830 const IRQF_TRIGGER_LOW = IrqLineStatus::IRQ_TYPE_LEVEL_LOW.bits();
831 const IRQF_TRIGGER_MASK = Self::IRQF_TRIGGER_HIGH.bits | Self::IRQF_TRIGGER_LOW.bits | Self::IRQF_TRIGGER_RISING.bits | Self::IRQF_TRIGGER_FALLING.bits;
832 /// IRQF_SHARED - 允许多个设备共享中断
833 const IRQF_SHARED = 0x00000080;
834 /// IRQF_PROBE_SHARED - 当预期出现共享不匹配时，由调用者设置
835 const IRQF_PROBE_SHARED = 0x00000100;
836 /// IRQF_TIMER - 标记此中断为定时器中断
837 const __IRQF_TIMER = 0x00000200;
838 /// IRQF_PERCPU - 中断是每个CPU的
839 const IRQF_PERCPU = 0x00000400;
840 /// IRQF_NOBALANCING - 将此中断从中断平衡中排除
841 const IRQF_NOBALANCING = 0x00000800;
842 /// IRQF_IRQPOLL - 中断用于轮询（出于性能原因，只有在共享中断中首次注册的中断会被考虑）
843 const IRQF_IRQPOLL = 0x00001000;
844 /// IRQF_ONESHOT - 在硬中断处理程序完成后，不会重新启用中断。由需要在运行线程处理程序之前保持中断线路禁用的线程中断使用。
845 const IRQF_ONESHOT = 0x00002000;
846 /// IRQF_NO_SUSPEND - 在挂起期间不禁用此IRQ。不能保证此中断会从挂起状态唤醒系统。
847 const IRQF_NO_SUSPEND = 0x00004000;
848 /// IRQF_FORCE_RESUME - 即使设置了IRQF_NO_SUSPEND，也强制在恢复时启用它
849 const IRQF_FORCE_RESUME = 0x00008000;
850 /// IRQF_NO_THREAD - 中断不能被线程化
851 const IRQF_NO_THREAD = 0x00010000;
852 /// IRQF_EARLY_RESUME - 在syscore而不是在设备恢复时间早期恢复IRQ。
853 const IRQF_EARLY_RESUME = 0x00020000;
854 /// IRQF_COND_SUSPEND - 如果IRQ与NO_SUSPEND用户共享，则在挂起中断后执行此中断处理程序。对于系统唤醒设备用户，需要在他们的中断处理程序中实现唤醒检测。
855 const IRQF_COND_SUSPEND = 0x00040000;
856 /// IRQF_NO_AUTOEN - 当用户请求时，不会自动启用IRQ或NMI。用户稍后会通过enable_irq()或enable_nmi()显式启用它。
857 const IRQF_NO_AUTOEN = 0x00080000;
858 /// IRQF_NO_DEBUG - 从IPI和类似处理程序的逃逸检测中排除，取决于IRQF_PERCPU。
859 const IRQF_NO_DEBUG = 0x00100000;
860 const IRQF_TIMER = Self::__IRQF_TIMER.bits | Self::IRQF_NO_SUSPEND.bits | Self::IRQF_NO_THREAD.bits;
861 }
862 }
863
864 impl IrqHandleFlags {
865 /// 检查是否指定了触发类型
866 #[inline(always)]
trigger_type_specified(&self) -> bool867 pub fn trigger_type_specified(&self) -> bool {
868 (self.bits & Self::IRQF_TRIGGER_MASK.bits) != 0
869 }
870
871 /// 插入触发类型
insert_trigger_type(&mut self, trigger: IrqLineStatus)872 pub fn insert_trigger_type(&mut self, trigger: IrqLineStatus) {
873 self.bits |= trigger.trigger_bits() & IrqHandleFlags::IRQF_TRIGGER_MASK.bits;
874 }
875
876 #[allow(dead_code)]
remove_trigger_type(&mut self, trigger: IrqLineStatus)877 pub fn remove_trigger_type(&mut self, trigger: IrqLineStatus) {
878 self.bits &= !(trigger.trigger_bits() & IrqHandleFlags::IRQF_TRIGGER_MASK.bits);
879 }
880
trigger_type(&self) -> IrqLineStatus881 pub fn trigger_type(&self) -> IrqLineStatus {
882 IrqLineStatus::from_bits_truncate(self.bits & IrqHandleFlags::IRQF_TRIGGER_MASK.bits)
883 }
884 }
885
886 #[inline(never)]
early_irq_init() -> Result<(), SystemError>887 pub(super) fn early_irq_init() -> Result<(), SystemError> {
888 let irqcnt = CurrentIrqArch::probe_total_irq_num();
889 let mut manager = IrqDescManager::new();
890 for i in 0..irqcnt {
891 let irq_desc = IrqDesc::new(IrqNumber::new(i), None, IrqStatus::empty());
892 manager.insert(IrqNumber::new(i), irq_desc);
893 }
894
895 unsafe {
896 IRQ_DESC_MANAGER = Some(manager);
897 }
898
899 return CurrentIrqArch::arch_early_irq_init();
900 }
901
902 static mut IRQ_DESC_MANAGER: Option<IrqDescManager> = None;
903
904 /// 获取中断描述符管理器的引用
905 #[inline(always)]
irq_desc_manager() -> &'static IrqDescManager906 pub fn irq_desc_manager() -> &'static IrqDescManager {
907 return unsafe { IRQ_DESC_MANAGER.as_ref().unwrap() };
908 }
909
910 pub struct IrqDescManager {
911 irq_descs: BTreeMap<IrqNumber, Arc<IrqDesc>>,
912 }
913
914 impl IrqDescManager {
new() -> Self915 fn new() -> Self {
916 IrqDescManager {
917 irq_descs: BTreeMap::new(),
918 }
919 }
920
921 /// 查找中断描述符
lookup(&self, irq: IrqNumber) -> Option<Arc<IrqDesc>>922 pub fn lookup(&self, irq: IrqNumber) -> Option<Arc<IrqDesc>> {
923 self.irq_descs.get(&irq).cloned()
924 }
925
926 /// 查找中断描述符并锁定总线(没有对irqdesc进行加锁)
927 #[allow(dead_code)]
lookup_and_lock_bus( &self, irq: IrqNumber, check_global: bool, check_percpu: bool, ) -> Option<Arc<IrqDesc>>928 pub fn lookup_and_lock_bus(
929 &self,
930 irq: IrqNumber,
931 check_global: bool,
932 check_percpu: bool,
933 ) -> Option<Arc<IrqDesc>> {
934 self.do_lookup_and_lock(irq, true, check_global, check_percpu)
935 }
936
do_lookup_and_lock( &self, irq: IrqNumber, lock_bus: bool, check_global: bool, check_percpu: bool, ) -> Option<Arc<IrqDesc>>937 fn do_lookup_and_lock(
938 &self,
939 irq: IrqNumber,
940 lock_bus: bool,
941 check_global: bool,
942 check_percpu: bool,
943 ) -> Option<Arc<IrqDesc>> {
944 let desc = self.lookup(irq)?;
945 if check_global || check_percpu {
946 if check_percpu && !desc.inner().line_status().is_per_cpu_devid() {
947 return None;
948 }
949
950 if check_global && desc.inner().line_status().is_per_cpu_devid() {
951 return None;
952 }
953 }
954
955 if lock_bus {
956 desc.chip_bus_lock();
957 }
958
959 return Some(desc);
960 }
961
insert(&mut self, irq: IrqNumber, desc: Arc<IrqDesc>)962 fn insert(&mut self, irq: IrqNumber, desc: Arc<IrqDesc>) {
963 self.irq_descs.insert(irq, desc);
964 }
965
966 /// 释放中断描述符
967 #[allow(dead_code)]
free_desc(&mut self, irq: IrqNumber)968 fn free_desc(&mut self, irq: IrqNumber) {
969 if let Some(desc) = self.irq_descs.get(&irq) {
970 irq_sysfs_del(desc);
971 self.irq_descs.remove(&irq);
972 }
973 }
974
975 /// 迭代中断描述符
iter_descs(&self) -> btree_map::Iter<'_, IrqNumber, Arc<IrqDesc>>976 pub fn iter_descs(&self) -> btree_map::Iter<'_, IrqNumber, Arc<IrqDesc>> {
977 self.irq_descs.iter()
978 }
979
980 /// 设置指定irq的可用cpu为所有cpu
981 #[allow(dead_code)]
set_percpu_devid_all(&self, irq: IrqNumber) -> Result<(), SystemError>982 pub fn set_percpu_devid_all(&self, irq: IrqNumber) -> Result<(), SystemError> {
983 self.set_percpu_devid(irq, None)
984 }
985
986 /// 设置指定irq的可用cpu
987 ///
988 /// 如果affinity为None，则表示设置为所有cpu
set_percpu_devid( &self, irq: IrqNumber, affinity: Option<&CpuMask>, ) -> Result<(), SystemError>989 pub fn set_percpu_devid(
990 &self,
991 irq: IrqNumber,
992 affinity: Option<&CpuMask>,
993 ) -> Result<(), SystemError> {
994 let desc = self.lookup(irq).ok_or(SystemError::EINVAL)?;
995 let mut desc_inner = desc.inner();
996
997 if desc_inner.percpu_enabled().is_some() {
998 return Err(SystemError::EINVAL);
999 }
1000
1001 *desc_inner.percpu_enabled_mut() = Some(CpuMask::new());
1002
1003 if let Some(affinity) = affinity {
1004 desc_inner.percpu_affinity_mut().replace(affinity.clone());
1005 } else {
1006 desc_inner
1007 .percpu_affinity_mut()
1008 .replace(smp_cpu_manager().possible_cpus().clone());
1009 }
1010
1011 drop(desc_inner);
1012
1013 desc.set_percpu_devid_flags();
1014
1015 return Ok(());
1016 }
1017 }
1018
1019 pub struct GenericIrqHandler;
1020
1021 #[allow(dead_code)]
1022 impl GenericIrqHandler {
1023 /// `handle_domain_irq` - 调用属于某个中断域的硬件中断的处理程序
1024 ///
1025 /// # 参数
1026 ///
1027 /// * `domain`: 执行查找的域
1028 /// * `hwirq`: 要转换为逻辑中断的硬件中断号
1029 ///
1030 /// # 返回
1031 ///
1032 /// 成功时返回 `Ok(())`，如果转换失败则返回 `Err(SystemError)`
1033 ///
1034 /// 此函数必须在初始化了中断寄存器的中断上下文中调用
1035 ///
1036 /// 参考 https://code.dragonos.org.cn/xref/linux-6.6.21/kernel/irq/irqdesc.c?fi=generic_handle_domain_irq#726
handle_domain_irq( domain: Arc<IrqDomain>, hwirq: HardwareIrqNumber, trap_frame: &mut TrapFrame, ) -> Result<(), SystemError>1037 pub fn handle_domain_irq(
1038 domain: Arc<IrqDomain>,
1039 hwirq: HardwareIrqNumber,
1040 trap_frame: &mut TrapFrame,
1041 ) -> Result<(), SystemError> {
1042 let (irq_desc, _) =
1043 irq_domain_manager().resolve_irq_mapping(Some(domain.clone()), hwirq)?;
1044
1045 irq_desc.handler().unwrap().handle(&irq_desc, trap_frame);
1046
1047 return Ok(());
1048 }
1049 }
1050