1 use core::{intrinsics::unlikely, ops::BitAnd};
2
3 use alloc::sync::Arc;
4 use log::{debug, error, warn};
5 use system_error::SystemError;
6
7 use crate::{
8 arch::{interrupt::TrapFrame, CurrentIrqArch},
9 exception::{irqchip::IrqChipFlags, irqdesc::InnerIrqDesc},
10 libs::{once::Once, spinlock::SpinLockGuard},
11 process::{ProcessFlags, ProcessManager},
12 smp::core::smp_get_processor_id,
13 };
14
15 use super::{
16 irqchip::IrqChip,
17 irqdata::{IrqData, IrqHandlerData, IrqStatus},
18 irqdesc::{
19 InnerIrqAction, IrqDesc, IrqDescState, IrqFlowHandler, IrqReturn, ThreadedHandlerFlags,
20 },
21 manage::{irq_manager, IrqManager},
22 InterruptArch, IrqNumber,
23 };
24
25 /// 获取用于处理错误的中断的处理程序
26 #[inline(always)]
bad_irq_handler() -> &'static dyn IrqFlowHandler27 pub fn bad_irq_handler() -> &'static dyn IrqFlowHandler {
28 &HandleBadIrq
29 }
30
31 /// 获取用于处理快速EOI的中断的处理程序
32 #[inline(always)]
fast_eoi_irq_handler() -> &'static dyn IrqFlowHandler33 pub fn fast_eoi_irq_handler() -> &'static dyn IrqFlowHandler {
34 &FastEOIIrqHandler
35 }
36
37 /// 获取用于处理边沿触发中断的处理程序
38 #[inline(always)]
39 #[allow(dead_code)]
edge_irq_handler() -> &'static dyn IrqFlowHandler40 pub fn edge_irq_handler() -> &'static dyn IrqFlowHandler {
41 &EdgeIrqHandler
42 }
43
44 /// handle spurious and unhandled irqs
45 #[derive(Debug)]
46 struct HandleBadIrq;
47
48 impl IrqFlowHandler for HandleBadIrq {
49 /// 参考: https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/handle.c?fi=handle_bad_irq#33
handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame)50 fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) {
51 // todo: print_irq_desc
52 // todo: 增加kstat计数
53 CurrentIrqArch::ack_bad_irq(irq_desc.irq());
54 }
55 }
56
57 #[derive(Debug)]
58 struct FastEOIIrqHandler;
59
60 impl IrqFlowHandler for FastEOIIrqHandler {
61 /// https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/chip.c?r=&mo=17578&fi=689#689
handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame)62 fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) {
63 let chip = irq_desc.irq_data().chip_info_read_irqsave().chip();
64
65 let mut desc_inner = irq_desc.inner();
66 let out = |din: SpinLockGuard<InnerIrqDesc>| {
67 if !chip.flags().contains(IrqChipFlags::IRQCHIP_EOI_IF_HANDLED) {
68 chip.irq_eoi(din.irq_data());
69 }
70 };
71 if !irq_may_run(&desc_inner) {
72 out(desc_inner);
73 return;
74 }
75
76 desc_inner
77 .internal_state_mut()
78 .remove(IrqDescState::IRQS_REPLAY | IrqDescState::IRQS_WAITING);
79
80 if desc_inner.actions().is_empty() || desc_inner.common_data().disabled() {
81 desc_inner
82 .internal_state_mut()
83 .insert(IrqDescState::IRQS_PENDING);
84 mask_irq(desc_inner.irq_data());
85 out(desc_inner);
86 return;
87 }
88
89 desc_inner = handle_irq_event(irq_desc, desc_inner);
90 cond_unmask_eoi_irq(&desc_inner, &chip);
91
92 return;
93 }
94 }
95
96 #[derive(Debug)]
97 struct EdgeIrqHandler;
98
99 impl IrqFlowHandler for EdgeIrqHandler {
100 // https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/chip.c?fi=handle_edge_irq#775
handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame)101 fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) {
102 let mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc> = irq_desc.inner();
103 if !irq_may_run(&desc_inner_guard) {
104 // debug!("!irq_may_run");
105 desc_inner_guard
106 .internal_state_mut()
107 .insert(IrqDescState::IRQS_PENDING);
108 mask_ack_irq(desc_inner_guard.irq_data());
109 return;
110 }
111
112 if desc_inner_guard.common_data().disabled() {
113 // debug!("desc_inner_guard.common_data().disabled()");
114 desc_inner_guard
115 .internal_state_mut()
116 .insert(IrqDescState::IRQS_PENDING);
117 mask_ack_irq(desc_inner_guard.irq_data());
118 return;
119 }
120
121 let irq_data = desc_inner_guard.irq_data().clone();
122
123 irq_data.chip_info_read_irqsave().chip().irq_ack(&irq_data);
124
125 loop {
126 if unlikely(desc_inner_guard.actions().is_empty()) {
127 debug!("no action for irq {}", irq_data.irq().data());
128 irq_manager().mask_irq(&irq_data);
129 return;
130 }
131
132 // 当我们在处理一个中断时，如果另一个中断到来，我们本可以屏蔽它.
133 // 如果在此期间没有被禁用，请重新启用它。
134 if desc_inner_guard
135 .internal_state()
136 .contains(IrqDescState::IRQS_PENDING)
137 {
138 let status = desc_inner_guard.common_data().status();
139 if !status.disabled() && status.masked() {
140 // debug!("re-enable irq");
141 irq_manager().unmask_irq(&desc_inner_guard);
142 }
143 }
144
145 // debug!("handle_irq_event");
146
147 desc_inner_guard = handle_irq_event(irq_desc, desc_inner_guard);
148
149 if !desc_inner_guard
150 .internal_state()
151 .contains(IrqDescState::IRQS_PENDING)
152 || desc_inner_guard.common_data().disabled()
153 {
154 break;
155 }
156 }
157 }
158 }
159
160 /// 判断中断是否可以运行
irq_may_run(desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) -> bool161 fn irq_may_run(desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>) -> bool {
162 let mask = IrqStatus::IRQD_IRQ_INPROGRESS | IrqStatus::IRQD_WAKEUP_ARMED;
163 let status = desc_inner_guard.common_data().status();
164
165 // 如果中断不在处理中并且没有被唤醒，则可以运行
166 if status.bitand(mask).is_empty() {
167 return true;
168 }
169
170 // todo: 检查其他处理器是否在轮询当前中断
171 return false;
172 }
173
mask_ack_irq(irq_data: &Arc<IrqData>)174 pub(super) fn mask_ack_irq(irq_data: &Arc<IrqData>) {
175 let chip = irq_data.chip_info_read_irqsave().chip();
176 if chip.can_mask_ack() {
177 chip.irq_mask_ack(irq_data);
178 irq_data.common_data().set_masked();
179 } else {
180 irq_manager().mask_irq(irq_data);
181 chip.irq_ack(irq_data);
182 }
183 }
184
mask_irq(irq_data: &Arc<IrqData>)185 pub(super) fn mask_irq(irq_data: &Arc<IrqData>) {
186 if irq_data.common_data().masked() {
187 return;
188 }
189
190 let chip = irq_data.chip_info_read_irqsave().chip();
191 if chip.irq_mask(irq_data).is_ok() {
192 irq_data.irqd_set(IrqStatus::IRQD_IRQ_MASKED);
193 }
194 }
195
unmask_irq(irq_data: &Arc<IrqData>)196 pub(super) fn unmask_irq(irq_data: &Arc<IrqData>) {
197 if !irq_data.common_data().masked() {
198 return;
199 }
200
201 let chip = irq_data.chip_info_read_irqsave().chip();
202
203 if chip.irq_unmask(irq_data).is_ok() {
204 irq_data.irqd_clear(IrqStatus::IRQD_IRQ_MASKED);
205 }
206 }
207
208 impl IrqManager {
do_irq_wake_thread( &self, desc: &Arc<IrqDesc>, action_inner: &mut SpinLockGuard<'_, InnerIrqAction>, )209 pub(super) fn do_irq_wake_thread(
210 &self,
211 desc: &Arc<IrqDesc>,
212 action_inner: &mut SpinLockGuard<'_, InnerIrqAction>,
213 ) {
214 let thread = action_inner.thread();
215
216 if thread.is_none() {
217 return;
218 }
219
220 let thread = thread.unwrap();
221 if thread.flags().contains(ProcessFlags::EXITING) {
222 return;
223 }
224
225 // 如果线程已经在运行，我们不需要唤醒它
226 if action_inner
227 .thread_flags_mut()
228 .test_and_set_bit(ThreadedHandlerFlags::IRQTF_RUNTHREAD)
229 {
230 return;
231 }
232
233 desc.inc_threads_active();
234
235 ProcessManager::wakeup(&thread).ok();
236 }
237 }
238
handle_irq_event<'a>( irq_desc: &'a Arc<IrqDesc>, mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc>, ) -> SpinLockGuard<'a, InnerIrqDesc>239 fn handle_irq_event<'a>(
240 irq_desc: &'a Arc<IrqDesc>,
241 mut desc_inner_guard: SpinLockGuard<'_, InnerIrqDesc>,
242 ) -> SpinLockGuard<'a, InnerIrqDesc> {
243 desc_inner_guard
244 .internal_state_mut()
245 .remove(IrqDescState::IRQS_PENDING);
246 desc_inner_guard.common_data().set_inprogress();
247
248 drop(desc_inner_guard);
249
250 let _r = do_handle_irq_event(irq_desc);
251
252 let desc_inner_guard = irq_desc.inner();
253 desc_inner_guard.common_data().clear_inprogress();
254
255 return desc_inner_guard;
256 }
257 /// 处理中断事件
258 ///
259 /// https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/handle.c?fi=handle_irq_event#139
260 #[inline(never)]
do_handle_irq_event(desc: &Arc<IrqDesc>) -> Result<(), SystemError>261 fn do_handle_irq_event(desc: &Arc<IrqDesc>) -> Result<(), SystemError> {
262 let desc_inner_guard = desc.inner();
263 let irq_data = desc_inner_guard.irq_data().clone();
264 let actions = desc_inner_guard.actions().clone();
265 drop(desc_inner_guard);
266
267 let irq = irq_data.irq();
268 let mut r = Ok(IrqReturn::NotHandled);
269
270 for action in actions {
271 let mut action_inner: SpinLockGuard<'_, InnerIrqAction> = action.inner();
272 // debug!("do_handle_irq_event: action: {:?}", action_inner.name());
273 let dynamic_data = action_inner
274 .dev_id()
275 .clone()
276 .map(|d| d as Arc<dyn IrqHandlerData>);
277 r = action_inner
278 .handler()
279 .unwrap()
280 .handle(irq, None, dynamic_data);
281
282 if let Ok(IrqReturn::WakeThread) = r {
283 if unlikely(action_inner.thread_fn().is_none()) {
284 warn_no_thread(irq, &mut action_inner);
285 } else {
286 irq_manager().do_irq_wake_thread(desc, &mut action_inner);
287 }
288 };
289 }
290
291 return r.map(|_| ());
292 }
293
294 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/kernel/irq/chip.c?r=&mo=17578&fi=659
cond_unmask_eoi_irq( desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>, chip: &Arc<dyn IrqChip>, )295 fn cond_unmask_eoi_irq(
296 desc_inner_guard: &SpinLockGuard<'_, InnerIrqDesc>,
297 chip: &Arc<dyn IrqChip>,
298 ) {
299 if !desc_inner_guard
300 .internal_state()
301 .contains(IrqDescState::IRQS_ONESHOT)
302 {
303 chip.irq_eoi(desc_inner_guard.irq_data());
304 return;
305 }
306
307 /*
308 * We need to unmask in the following cases:
309 * - Oneshot irq which did not wake the thread (caused by a
310 * spurious interrupt or a primary handler handling it
311 * completely).
312 */
313
314 if !desc_inner_guard.common_data().disabled()
315 && desc_inner_guard.common_data().masked()
316 && desc_inner_guard.threads_oneshot() == 0
317 {
318 debug!(
319 "eoi unmask irq {}",
320 desc_inner_guard.irq_data().irq().data()
321 );
322 chip.irq_eoi(desc_inner_guard.irq_data());
323 unmask_irq(desc_inner_guard.irq_data());
324 } else if !chip.flags().contains(IrqChipFlags::IRQCHIP_EOI_THREADED) {
325 debug!("eoi irq {}", desc_inner_guard.irq_data().irq().data());
326 chip.irq_eoi(desc_inner_guard.irq_data());
327 } else {
328 warn!(
329 "irq {} eoi failed",
330 desc_inner_guard.irq_data().irq().data()
331 );
332 }
333 }
334
warn_no_thread(irq: IrqNumber, action_inner: &mut SpinLockGuard<'_, InnerIrqAction>)335 fn warn_no_thread(irq: IrqNumber, action_inner: &mut SpinLockGuard<'_, InnerIrqAction>) {
336 // warn on once
337 if action_inner
338 .thread_flags_mut()
339 .test_and_set_bit(ThreadedHandlerFlags::IRQTF_WARNED)
340 {
341 return;
342 }
343
344 warn!(
345 "irq {}, device {} returned IRQ_WAKE_THREAD, but no threaded handler",
346 irq.data(),
347 action_inner.name()
348 );
349 }
350
351 /// `handle_percpu_devid_irq` - 带有per-CPU设备id的perCPU本地中断处理程序
352 ///
353 ///
354 /// * `desc`: 此中断的中断描述结构
355 ///
356 /// 在没有锁定要求的SMP机器上的每个CPU中断。与linux的`handle_percpu_irq()`相同，但有以下额外内容：
357 ///
358 /// `action->percpu_dev_id`是一个指向per-cpu变量的指针，这些变量
359 /// 包含调用此处理程序的cpu的真实设备id
360 #[derive(Debug)]
361 pub struct PerCpuDevIdIrqHandler;
362
363 impl IrqFlowHandler for PerCpuDevIdIrqHandler {
handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame)364 fn handle(&self, irq_desc: &Arc<IrqDesc>, _trap_frame: &mut TrapFrame) {
365 let desc_inner_guard = irq_desc.inner();
366 let irq_data = desc_inner_guard.irq_data().clone();
367 let chip = irq_data.chip_info_read().chip();
368
369 chip.irq_ack(&irq_data);
370
371 let irq = irq_data.irq();
372
373 let action = desc_inner_guard.actions().first().cloned();
374
375 drop(desc_inner_guard);
376
377 if let Some(action) = action {
378 let action_inner = action.inner();
379 let per_cpu_devid = action_inner.per_cpu_dev_id().cloned();
380
381 let handler = action_inner.handler().unwrap();
382 drop(action_inner);
383
384 let _r = handler.handle(
385 irq,
386 None,
387 per_cpu_devid.map(|d| d as Arc<dyn IrqHandlerData>),
388 );
389 } else {
390 let cpu = smp_get_processor_id();
391
392 let enabled = irq_desc
393 .inner()
394 .percpu_enabled()
395 .as_ref()
396 .unwrap()
397 .get(cpu)
398 .unwrap_or(false);
399
400 if enabled {
401 irq_manager().irq_percpu_disable(irq_desc, &irq_data, &chip, cpu);
402 }
403 static ONCE: Once = Once::new();
404
405 ONCE.call_once(|| {
406 error!(
407 "Spurious percpu irq {} on cpu {:?}, enabled: {}",
408 irq.data(),
409 cpu,
410 enabled
411 );
412 });
413 }
414
415 chip.irq_eoi(&irq_data);
416 }
417 }
418