1 use core::intrinsics::unlikely;
2
3 use alloc::{string::ToString, sync::Arc};
4 use intertrait::CastFrom;
5 use log::warn;
6 use system_error::SystemError;
7
8 use crate::{
9 arch::{
10 driver::apic::{
11 apic_timer::{local_apic_timer_irq_desc_init, APIC_TIMER_IRQ_NUM},
12 ioapic::ioapic_init,
13 },
14 interrupt::{
15 entry::arch_setup_interrupt_gate,
16 ipi::{arch_ipi_handler_init, send_ipi, IPI_NUM_FLUSH_TLB, IPI_NUM_KICK_CPU},
17 msi::{X86MsiAddrHi, X86MsiAddrLoNormal, X86MsiDataNormal, X86_MSI_BASE_ADDRESS_LOW},
18 },
19 },
20 driver::open_firmware::device_node::DeviceNode,
21 exception::{
22 ipi::{IpiKind, IpiTarget},
23 irqchip::{IrqChip, IrqChipData, IrqChipFlags},
24 irqdata::IrqData,
25 irqdomain::{irq_domain_manager, IrqDomain, IrqDomainBusToken, IrqDomainOps},
26 msi::MsiMsg,
27 HardwareIrqNumber, IrqNumber,
28 },
29 libs::spinlock::{SpinLock, SpinLockGuard},
30 smp::{core::smp_get_processor_id, cpu::ProcessorId},
31 };
32
33 use super::{hw_irq::HardwareIrqConfig, CurrentApic, LocalAPIC};
34
35 static mut LOCAL_APIC_CHIP: Option<Arc<LocalApicChip>> = None;
36
local_apic_chip() -> &'static Arc<LocalApicChip>37 pub fn local_apic_chip() -> &'static Arc<LocalApicChip> {
38 unsafe { LOCAL_APIC_CHIP.as_ref().unwrap() }
39 }
40
41 #[derive(Debug)]
42 pub struct LocalApicChip {
43 inner: SpinLock<InnerIrqChip>,
44 }
45
46 impl LocalApicChip {
new() -> Self47 pub fn new() -> Self {
48 Self {
49 inner: SpinLock::new(InnerIrqChip {
50 flags: IrqChipFlags::empty(),
51 }),
52 }
53 }
54 }
55
56 impl IrqChip for LocalApicChip {
name(&self) -> &'static str57 fn name(&self) -> &'static str {
58 "APIC"
59 }
60
can_set_flow_type(&self) -> bool61 fn can_set_flow_type(&self) -> bool {
62 false
63 }
64
irq_disable(&self, _irq: &Arc<IrqData>)65 fn irq_disable(&self, _irq: &Arc<IrqData>) {}
66
irq_ack(&self, _irq: &Arc<IrqData>)67 fn irq_ack(&self, _irq: &Arc<IrqData>) {
68 CurrentApic.send_eoi();
69 }
70
can_set_affinity(&self) -> bool71 fn can_set_affinity(&self) -> bool {
72 false
73 }
74
can_mask_ack(&self) -> bool75 fn can_mask_ack(&self) -> bool {
76 false
77 }
78
irq_enable(&self, _irq: &Arc<IrqData>) -> Result<(), SystemError>79 fn irq_enable(&self, _irq: &Arc<IrqData>) -> Result<(), SystemError> {
80 // 这里临时处理，后续需要修改
81 return Ok(());
82 }
83
irq_unmask(&self, _irq: &Arc<IrqData>) -> Result<(), SystemError>84 fn irq_unmask(&self, _irq: &Arc<IrqData>) -> Result<(), SystemError> {
85 Ok(())
86 }
87
irq_compose_msi_msg(&self, irq: &Arc<IrqData>, msg: &mut MsiMsg)88 fn irq_compose_msi_msg(&self, irq: &Arc<IrqData>, msg: &mut MsiMsg) {
89 let chip_data = irq.chip_info_read_irqsave().chip_data().unwrap();
90 let apicd = chip_data.ref_any().downcast_ref::<ApicChipData>().unwrap();
91 let cfg = &apicd.inner().hw_irq_cfg;
92 irq_msi_compose_msg(cfg, msg, false);
93 }
94
retrigger(&self, irq: &Arc<IrqData>) -> Result<(), SystemError>95 fn retrigger(&self, irq: &Arc<IrqData>) -> Result<(), SystemError> {
96 let chip_data = irq
97 .chip_info_read_irqsave()
98 .chip_data()
99 .ok_or(SystemError::EINVAL)?;
100 let apicd = chip_data
101 .ref_any()
102 .downcast_ref::<ApicChipData>()
103 .ok_or(SystemError::EINVAL)?;
104 let inner = apicd.inner();
105
106 send_ipi(
107 IpiKind::SpecVector(inner.vector),
108 IpiTarget::Specified(inner.cpu),
109 );
110
111 Ok(())
112 }
113
flags(&self) -> IrqChipFlags114 fn flags(&self) -> IrqChipFlags {
115 self.inner.lock_irqsave().flags
116 }
117 }
118
119 #[derive(Debug)]
120 struct InnerIrqChip {
121 flags: IrqChipFlags,
122 }
123
124 #[derive(Debug)]
125 struct ApicChipData {
126 inner: SpinLock<InnerApicChipData>,
127 }
128
129 impl ApicChipData {
130 #[allow(dead_code)]
new( hw_irq_cfg: HardwareIrqConfig, irq: IrqNumber, vector: HardwareIrqNumber, cpu: ProcessorId, ) -> Self131 pub fn new(
132 hw_irq_cfg: HardwareIrqConfig,
133 irq: IrqNumber,
134 vector: HardwareIrqNumber,
135 cpu: ProcessorId,
136 ) -> Self {
137 Self {
138 inner: SpinLock::new(InnerApicChipData {
139 hw_irq_cfg,
140 irq,
141 vector,
142 prev_vector: None,
143 cpu,
144 prev_cpu: None,
145 status: ApicChipStatus::empty(),
146 }),
147 }
148 }
149
inner(&self) -> SpinLockGuard<InnerApicChipData>150 pub fn inner(&self) -> SpinLockGuard<InnerApicChipData> {
151 self.inner.lock_irqsave()
152 }
153 }
154
155 #[allow(dead_code)]
156 #[derive(Debug)]
157 struct InnerApicChipData {
158 hw_irq_cfg: HardwareIrqConfig,
159 irq: IrqNumber,
160 vector: HardwareIrqNumber,
161 prev_vector: Option<HardwareIrqNumber>,
162 cpu: ProcessorId,
163 prev_cpu: Option<ProcessorId>,
164 status: ApicChipStatus,
165 }
166
167 impl IrqChipData for ApicChipData {
as_any_ref(&self) -> &dyn core::any::Any168 fn as_any_ref(&self) -> &dyn core::any::Any {
169 self
170 }
171 }
172
173 bitflags! {
174 pub struct ApicChipStatus: u32 {
175 const MOVE_IN_PROGRESS = 1 << 0;
176 const IS_MANAGED = 1 << 1;
177 const CAN_RESERVE = 1 << 2;
178 const HAS_RESERVED = 1 << 3;
179 }
180 }
181
182 #[allow(dead_code)]
irq_msi_compose_msg(cfg: &HardwareIrqConfig, msg: &mut MsiMsg, dmar: bool)183 pub(super) fn irq_msi_compose_msg(cfg: &HardwareIrqConfig, msg: &mut MsiMsg, dmar: bool) {
184 *msg = MsiMsg::new_zeroed();
185
186 let arch_data = X86MsiDataNormal::new()
187 .with_delivery_mode(x86::apic::DeliveryMode::Fixed as u8)
188 .with_vector((cfg.vector.data() & 0xff) as u8);
189 let mut address_lo = X86MsiAddrLoNormal::new()
190 .with_base_address(X86_MSI_BASE_ADDRESS_LOW)
191 .with_dest_mode_logical(false)
192 .with_destid_0_7(cfg.apic_id.data() & 0xff);
193
194 let mut address_hi = X86MsiAddrHi::new();
195
196 /*
197 * 只有IOMMU本身可以使用将目标APIC ID放入地址的高位的技术。
198 * 任何其他尝试这样做的东西都只是在写内存，并且需要IR来
199 * 寻址不能在正常的32位地址范围内0xFFExxxxx寻址的APIC。
200 * 这通常是8位，但一些虚拟化程序允许在位5-11使用扩展的目的地ID字段，
201 * 总共支持15位的APIC ID。
202 */
203 if dmar {
204 address_hi.set_destid_8_31(cfg.apic_id.data() >> 8);
205 } else if cfg.apic_id.data() < 0x8000 {
206 // todo: 判断vmx是否支持 extended destination mode
207 // 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/arch/x86/kernel/apic/apic.c?fi=__irq_msi_compose_msg#2580
208 address_lo.set_virt_destid_8_14(cfg.apic_id.data() >> 8);
209 } else if unlikely(cfg.apic_id.data() > 0xff) {
210 warn!(
211 "irq_msi_compose_msg: Invalid APIC ID: {}",
212 cfg.apic_id.data()
213 );
214 }
215
216 msg.address_hi = address_hi.into();
217 msg.address_lo = address_lo.into();
218 msg.data = arch_data.into();
219 }
220
221 static mut X86_VECTOR_DOMAIN: Option<Arc<IrqDomain>> = None;
222
223 #[inline(always)]
224 #[allow(dead_code)]
x86_vector_domain() -> &'static Arc<IrqDomain>225 pub fn x86_vector_domain() -> &'static Arc<IrqDomain> {
226 unsafe { X86_VECTOR_DOMAIN.as_ref().unwrap() }
227 }
228
229 #[inline(never)]
arch_early_irq_init() -> Result<(), SystemError>230 pub fn arch_early_irq_init() -> Result<(), SystemError> {
231 const IRQ_SIZE: u32 = 223;
232 let vec_domain = irq_domain_manager()
233 .create_and_add(
234 "VECTOR".to_string(),
235 &X86VectorDomainOps,
236 IrqNumber::new(32),
237 HardwareIrqNumber::new(32),
238 IRQ_SIZE,
239 )
240 .ok_or(SystemError::ENOMEM)?;
241 irq_domain_manager().set_default_domain(vec_domain.clone());
242 irq_domain_manager().domain_associate_many(
243 &vec_domain,
244 IrqNumber::new(0),
245 HardwareIrqNumber::new(0),
246 IRQ_SIZE,
247 );
248 unsafe { X86_VECTOR_DOMAIN = Some(vec_domain) };
249
250 let apic_chip = Arc::new(LocalApicChip::new());
251
252 unsafe { LOCAL_APIC_CHIP = Some(apic_chip) };
253
254 // todo: add vector matrix
255 // 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/arch/x86/kernel/apic/vector.c#803
256 warn!("arch_early_irq_init: todo: add vector matrix");
257
258 local_apic_timer_irq_desc_init();
259 arch_ipi_handler_init();
260 CurrentApic.init_current_cpu();
261 if smp_get_processor_id().data() == 0 {
262 unsafe { arch_setup_interrupt_gate() };
263 ioapic_init(&[APIC_TIMER_IRQ_NUM, IPI_NUM_KICK_CPU, IPI_NUM_FLUSH_TLB]);
264 }
265 return Ok(());
266 }
267
268 /// x86的中断域操作
269 ///
270 /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/arch/x86/kernel/apic/vector.c#693
271 #[derive(Debug)]
272 struct X86VectorDomainOps;
273
274 impl IrqDomainOps for X86VectorDomainOps {
match_node( &self, _irq_domain: &Arc<IrqDomain>, _device_node: &Arc<DeviceNode>, _bus_token: IrqDomainBusToken, ) -> bool275 fn match_node(
276 &self,
277 _irq_domain: &Arc<IrqDomain>,
278 _device_node: &Arc<DeviceNode>,
279 _bus_token: IrqDomainBusToken,
280 ) -> bool {
281 todo!()
282 }
283
map( &self, _irq_domain: &Arc<IrqDomain>, _hwirq: HardwareIrqNumber, _virq: IrqNumber, ) -> Result<(), SystemError>284 fn map(
285 &self,
286 _irq_domain: &Arc<IrqDomain>,
287 _hwirq: HardwareIrqNumber,
288 _virq: IrqNumber,
289 ) -> Result<(), SystemError> {
290 Err(SystemError::ENOSYS)
291 }
292
unmap(&self, _irq_domain: &Arc<IrqDomain>, _virq: IrqNumber)293 fn unmap(&self, _irq_domain: &Arc<IrqDomain>, _virq: IrqNumber) {
294 todo!()
295 }
296 }
297