1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *	Intel IO-APIC support for multi-Pentium hosts.
4  *
5  *	Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
6  *
7  *	Many thanks to Stig Venaas for trying out countless experimental
8  *	patches and reporting/debugging problems patiently!
9  *
10  *	(c) 1999, Multiple IO-APIC support, developed by
11  *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
12  *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
13  *	further tested and cleaned up by Zach Brown <zab@redhat.com>
14  *	and Ingo Molnar <mingo@redhat.com>
15  *
16  *	Fixes
17  *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
18  *					thanks to Eric Gilmore
19  *					and Rolf G. Tews
20  *					for testing these extensively
21  *	Paul Diefenbaugh	:	Added full ACPI support
22  *
23  * Historical information which is worth to be preserved:
24  *
25  * - SiS APIC rmw bug:
26  *
27  *	We used to have a workaround for a bug in SiS chips which
28  *	required to rewrite the index register for a read-modify-write
29  *	operation as the chip lost the index information which was
30  *	setup for the read already. We cache the data now, so that
31  *	workaround has been removed.
32  */
33 
34 #include <linux/mm.h>
35 #include <linux/interrupt.h>
36 #include <linux/irq.h>
37 #include <linux/init.h>
38 #include <linux/delay.h>
39 #include <linux/sched.h>
40 #include <linux/pci.h>
41 #include <linux/mc146818rtc.h>
42 #include <linux/compiler.h>
43 #include <linux/acpi.h>
44 #include <linux/export.h>
45 #include <linux/syscore_ops.h>
46 #include <linux/freezer.h>
47 #include <linux/kthread.h>
48 #include <linux/jiffies.h>	/* time_after() */
49 #include <linux/slab.h>
50 #include <linux/memblock.h>
51 #include <linux/msi.h>
52 
53 #include <asm/irqdomain.h>
54 #include <asm/io.h>
55 #include <asm/smp.h>
56 #include <asm/cpu.h>
57 #include <asm/desc.h>
58 #include <asm/proto.h>
59 #include <asm/acpi.h>
60 #include <asm/dma.h>
61 #include <asm/timer.h>
62 #include <asm/time.h>
63 #include <asm/i8259.h>
64 #include <asm/setup.h>
65 #include <asm/irq_remapping.h>
66 #include <asm/hw_irq.h>
67 #include <asm/apic.h>
68 #include <asm/pgtable.h>
69 #include <asm/x86_init.h>
70 
71 #define	for_each_ioapic(idx)		\
72 	for ((idx) = 0; (idx) < nr_ioapics; (idx)++)
73 #define	for_each_ioapic_reverse(idx)	\
74 	for ((idx) = nr_ioapics - 1; (idx) >= 0; (idx)--)
75 #define	for_each_pin(idx, pin)		\
76 	for ((pin) = 0; (pin) < ioapics[(idx)].nr_registers; (pin)++)
77 #define	for_each_ioapic_pin(idx, pin)	\
78 	for_each_ioapic((idx))		\
79 		for_each_pin((idx), (pin))
80 #define for_each_irq_pin(entry, head) \
81 	list_for_each_entry(entry, &head, list)
82 
83 static DEFINE_RAW_SPINLOCK(ioapic_lock);
84 static DEFINE_MUTEX(ioapic_mutex);
85 static unsigned int ioapic_dynirq_base;
86 static int ioapic_initialized;
87 
88 struct irq_pin_list {
89 	struct list_head list;
90 	int apic, pin;
91 };
92 
93 struct mp_chip_data {
94 	struct list_head		irq_2_pin;
95 	struct IO_APIC_route_entry	entry;
96 	bool				is_level;
97 	bool				active_low;
98 	bool				isa_irq;
99 	u32 count;
100 };
101 
102 struct mp_ioapic_gsi {
103 	u32 gsi_base;
104 	u32 gsi_end;
105 };
106 
107 static struct ioapic {
108 	/*
109 	 * # of IRQ routing registers
110 	 */
111 	int nr_registers;
112 	/*
113 	 * Saved state during suspend/resume, or while enabling intr-remap.
114 	 */
115 	struct IO_APIC_route_entry *saved_registers;
116 	/* I/O APIC config */
117 	struct mpc_ioapic mp_config;
118 	/* IO APIC gsi routing info */
119 	struct mp_ioapic_gsi  gsi_config;
120 	struct ioapic_domain_cfg irqdomain_cfg;
121 	struct irq_domain *irqdomain;
122 	struct resource *iomem_res;
123 } ioapics[MAX_IO_APICS];
124 
125 #define mpc_ioapic_ver(ioapic_idx)	ioapics[ioapic_idx].mp_config.apicver
126 
mpc_ioapic_id(int ioapic_idx)127 int mpc_ioapic_id(int ioapic_idx)
128 {
129 	return ioapics[ioapic_idx].mp_config.apicid;
130 }
131 
mpc_ioapic_addr(int ioapic_idx)132 unsigned int mpc_ioapic_addr(int ioapic_idx)
133 {
134 	return ioapics[ioapic_idx].mp_config.apicaddr;
135 }
136 
mp_ioapic_gsi_routing(int ioapic_idx)137 static inline struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
138 {
139 	return &ioapics[ioapic_idx].gsi_config;
140 }
141 
mp_ioapic_pin_count(int ioapic)142 static inline int mp_ioapic_pin_count(int ioapic)
143 {
144 	struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
145 
146 	return gsi_cfg->gsi_end - gsi_cfg->gsi_base + 1;
147 }
148 
mp_pin_to_gsi(int ioapic,int pin)149 static inline u32 mp_pin_to_gsi(int ioapic, int pin)
150 {
151 	return mp_ioapic_gsi_routing(ioapic)->gsi_base + pin;
152 }
153 
mp_is_legacy_irq(int irq)154 static inline bool mp_is_legacy_irq(int irq)
155 {
156 	return irq >= 0 && irq < nr_legacy_irqs();
157 }
158 
mp_ioapic_irqdomain(int ioapic)159 static inline struct irq_domain *mp_ioapic_irqdomain(int ioapic)
160 {
161 	return ioapics[ioapic].irqdomain;
162 }
163 
164 int nr_ioapics;
165 
166 /* The one past the highest gsi number used */
167 u32 gsi_top;
168 
169 /* MP IRQ source entries */
170 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
171 
172 /* # of MP IRQ source entries */
173 int mp_irq_entries;
174 
175 #ifdef CONFIG_EISA
176 int mp_bus_id_to_type[MAX_MP_BUSSES];
177 #endif
178 
179 DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
180 
181 bool ioapic_is_disabled __ro_after_init;
182 
183 /**
184  * disable_ioapic_support() - disables ioapic support at runtime
185  */
disable_ioapic_support(void)186 void disable_ioapic_support(void)
187 {
188 #ifdef CONFIG_PCI
189 	noioapicquirk = 1;
190 	noioapicreroute = -1;
191 #endif
192 	ioapic_is_disabled = true;
193 }
194 
parse_noapic(char * str)195 static int __init parse_noapic(char *str)
196 {
197 	/* disable IO-APIC */
198 	disable_ioapic_support();
199 	return 0;
200 }
201 early_param("noapic", parse_noapic);
202 
203 /* Will be called in mpparse/ACPI codes for saving IRQ info */
mp_save_irq(struct mpc_intsrc * m)204 void mp_save_irq(struct mpc_intsrc *m)
205 {
206 	int i;
207 
208 	apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
209 		" IRQ %02x, APIC ID %x, APIC INT %02x\n",
210 		m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
211 		m->srcbusirq, m->dstapic, m->dstirq);
212 
213 	for (i = 0; i < mp_irq_entries; i++) {
214 		if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
215 			return;
216 	}
217 
218 	memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
219 	if (++mp_irq_entries == MAX_IRQ_SOURCES)
220 		panic("Max # of irq sources exceeded!!\n");
221 }
222 
alloc_ioapic_saved_registers(int idx)223 static void alloc_ioapic_saved_registers(int idx)
224 {
225 	size_t size;
226 
227 	if (ioapics[idx].saved_registers)
228 		return;
229 
230 	size = sizeof(struct IO_APIC_route_entry) * ioapics[idx].nr_registers;
231 	ioapics[idx].saved_registers = kzalloc(size, GFP_KERNEL);
232 	if (!ioapics[idx].saved_registers)
233 		pr_err("IOAPIC %d: suspend/resume impossible!\n", idx);
234 }
235 
free_ioapic_saved_registers(int idx)236 static void free_ioapic_saved_registers(int idx)
237 {
238 	kfree(ioapics[idx].saved_registers);
239 	ioapics[idx].saved_registers = NULL;
240 }
241 
arch_early_ioapic_init(void)242 int __init arch_early_ioapic_init(void)
243 {
244 	int i;
245 
246 	if (!nr_legacy_irqs())
247 		io_apic_irqs = ~0UL;
248 
249 	for_each_ioapic(i)
250 		alloc_ioapic_saved_registers(i);
251 
252 	return 0;
253 }
254 
255 struct io_apic {
256 	unsigned int index;
257 	unsigned int unused[3];
258 	unsigned int data;
259 	unsigned int unused2[11];
260 	unsigned int eoi;
261 };
262 
io_apic_base(int idx)263 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
264 {
265 	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
266 		+ (mpc_ioapic_addr(idx) & ~PAGE_MASK);
267 }
268 
io_apic_eoi(unsigned int apic,unsigned int vector)269 static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
270 {
271 	struct io_apic __iomem *io_apic = io_apic_base(apic);
272 	writel(vector, &io_apic->eoi);
273 }
274 
native_io_apic_read(unsigned int apic,unsigned int reg)275 unsigned int native_io_apic_read(unsigned int apic, unsigned int reg)
276 {
277 	struct io_apic __iomem *io_apic = io_apic_base(apic);
278 	writel(reg, &io_apic->index);
279 	return readl(&io_apic->data);
280 }
281 
io_apic_write(unsigned int apic,unsigned int reg,unsigned int value)282 static void io_apic_write(unsigned int apic, unsigned int reg,
283 			  unsigned int value)
284 {
285 	struct io_apic __iomem *io_apic = io_apic_base(apic);
286 
287 	writel(reg, &io_apic->index);
288 	writel(value, &io_apic->data);
289 }
290 
__ioapic_read_entry(int apic,int pin)291 static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
292 {
293 	struct IO_APIC_route_entry entry;
294 
295 	entry.w1 = io_apic_read(apic, 0x10 + 2 * pin);
296 	entry.w2 = io_apic_read(apic, 0x11 + 2 * pin);
297 
298 	return entry;
299 }
300 
ioapic_read_entry(int apic,int pin)301 static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
302 {
303 	struct IO_APIC_route_entry entry;
304 	unsigned long flags;
305 
306 	raw_spin_lock_irqsave(&ioapic_lock, flags);
307 	entry = __ioapic_read_entry(apic, pin);
308 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
309 
310 	return entry;
311 }
312 
313 /*
314  * When we write a new IO APIC routing entry, we need to write the high
315  * word first! If the mask bit in the low word is clear, we will enable
316  * the interrupt, and we need to make sure the entry is fully populated
317  * before that happens.
318  */
__ioapic_write_entry(int apic,int pin,struct IO_APIC_route_entry e)319 static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
320 {
321 	io_apic_write(apic, 0x11 + 2*pin, e.w2);
322 	io_apic_write(apic, 0x10 + 2*pin, e.w1);
323 }
324 
ioapic_write_entry(int apic,int pin,struct IO_APIC_route_entry e)325 static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
326 {
327 	unsigned long flags;
328 
329 	raw_spin_lock_irqsave(&ioapic_lock, flags);
330 	__ioapic_write_entry(apic, pin, e);
331 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
332 }
333 
334 /*
335  * When we mask an IO APIC routing entry, we need to write the low
336  * word first, in order to set the mask bit before we change the
337  * high bits!
338  */
ioapic_mask_entry(int apic,int pin)339 static void ioapic_mask_entry(int apic, int pin)
340 {
341 	struct IO_APIC_route_entry e = { .masked = true };
342 	unsigned long flags;
343 
344 	raw_spin_lock_irqsave(&ioapic_lock, flags);
345 	io_apic_write(apic, 0x10 + 2*pin, e.w1);
346 	io_apic_write(apic, 0x11 + 2*pin, e.w2);
347 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
348 }
349 
350 /*
351  * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
352  * shared ISA-space IRQs, so we have to support them. We are super
353  * fast in the common case, and fast for shared ISA-space IRQs.
354  */
__add_pin_to_irq_node(struct mp_chip_data * data,int node,int apic,int pin)355 static int __add_pin_to_irq_node(struct mp_chip_data *data,
356 				 int node, int apic, int pin)
357 {
358 	struct irq_pin_list *entry;
359 
360 	/* don't allow duplicates */
361 	for_each_irq_pin(entry, data->irq_2_pin)
362 		if (entry->apic == apic && entry->pin == pin)
363 			return 0;
364 
365 	entry = kzalloc_node(sizeof(struct irq_pin_list), GFP_ATOMIC, node);
366 	if (!entry) {
367 		pr_err("can not alloc irq_pin_list (%d,%d,%d)\n",
368 		       node, apic, pin);
369 		return -ENOMEM;
370 	}
371 	entry->apic = apic;
372 	entry->pin = pin;
373 	list_add_tail(&entry->list, &data->irq_2_pin);
374 
375 	return 0;
376 }
377 
__remove_pin_from_irq(struct mp_chip_data * data,int apic,int pin)378 static void __remove_pin_from_irq(struct mp_chip_data *data, int apic, int pin)
379 {
380 	struct irq_pin_list *tmp, *entry;
381 
382 	list_for_each_entry_safe(entry, tmp, &data->irq_2_pin, list)
383 		if (entry->apic == apic && entry->pin == pin) {
384 			list_del(&entry->list);
385 			kfree(entry);
386 			return;
387 		}
388 }
389 
add_pin_to_irq_node(struct mp_chip_data * data,int node,int apic,int pin)390 static void add_pin_to_irq_node(struct mp_chip_data *data,
391 				int node, int apic, int pin)
392 {
393 	if (__add_pin_to_irq_node(data, node, apic, pin))
394 		panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
395 }
396 
397 /*
398  * Reroute an IRQ to a different pin.
399  */
replace_pin_at_irq_node(struct mp_chip_data * data,int node,int oldapic,int oldpin,int newapic,int newpin)400 static void __init replace_pin_at_irq_node(struct mp_chip_data *data, int node,
401 					   int oldapic, int oldpin,
402 					   int newapic, int newpin)
403 {
404 	struct irq_pin_list *entry;
405 
406 	for_each_irq_pin(entry, data->irq_2_pin) {
407 		if (entry->apic == oldapic && entry->pin == oldpin) {
408 			entry->apic = newapic;
409 			entry->pin = newpin;
410 			/* every one is different, right? */
411 			return;
412 		}
413 	}
414 
415 	/* old apic/pin didn't exist, so just add new ones */
416 	add_pin_to_irq_node(data, node, newapic, newpin);
417 }
418 
io_apic_modify_irq(struct mp_chip_data * data,bool masked,void (* final)(struct irq_pin_list * entry))419 static void io_apic_modify_irq(struct mp_chip_data *data, bool masked,
420 			       void (*final)(struct irq_pin_list *entry))
421 {
422 	struct irq_pin_list *entry;
423 
424 	data->entry.masked = masked;
425 
426 	for_each_irq_pin(entry, data->irq_2_pin) {
427 		io_apic_write(entry->apic, 0x10 + 2 * entry->pin, data->entry.w1);
428 		if (final)
429 			final(entry);
430 	}
431 }
432 
io_apic_sync(struct irq_pin_list * entry)433 static void io_apic_sync(struct irq_pin_list *entry)
434 {
435 	/*
436 	 * Synchronize the IO-APIC and the CPU by doing
437 	 * a dummy read from the IO-APIC
438 	 */
439 	struct io_apic __iomem *io_apic;
440 
441 	io_apic = io_apic_base(entry->apic);
442 	readl(&io_apic->data);
443 }
444 
mask_ioapic_irq(struct irq_data * irq_data)445 static void mask_ioapic_irq(struct irq_data *irq_data)
446 {
447 	struct mp_chip_data *data = irq_data->chip_data;
448 	unsigned long flags;
449 
450 	raw_spin_lock_irqsave(&ioapic_lock, flags);
451 	io_apic_modify_irq(data, true, &io_apic_sync);
452 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
453 }
454 
__unmask_ioapic(struct mp_chip_data * data)455 static void __unmask_ioapic(struct mp_chip_data *data)
456 {
457 	io_apic_modify_irq(data, false, NULL);
458 }
459 
unmask_ioapic_irq(struct irq_data * irq_data)460 static void unmask_ioapic_irq(struct irq_data *irq_data)
461 {
462 	struct mp_chip_data *data = irq_data->chip_data;
463 	unsigned long flags;
464 
465 	raw_spin_lock_irqsave(&ioapic_lock, flags);
466 	__unmask_ioapic(data);
467 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
468 }
469 
470 /*
471  * IO-APIC versions below 0x20 don't support EOI register.
472  * For the record, here is the information about various versions:
473  *     0Xh     82489DX
474  *     1Xh     I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
475  *     2Xh     I/O(x)APIC which is PCI 2.2 Compliant
476  *     30h-FFh Reserved
477  *
478  * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
479  * version as 0x2. This is an error with documentation and these ICH chips
480  * use io-apic's of version 0x20.
481  *
482  * For IO-APIC's with EOI register, we use that to do an explicit EOI.
483  * Otherwise, we simulate the EOI message manually by changing the trigger
484  * mode to edge and then back to level, with RTE being masked during this.
485  */
__eoi_ioapic_pin(int apic,int pin,int vector)486 static void __eoi_ioapic_pin(int apic, int pin, int vector)
487 {
488 	if (mpc_ioapic_ver(apic) >= 0x20) {
489 		io_apic_eoi(apic, vector);
490 	} else {
491 		struct IO_APIC_route_entry entry, entry1;
492 
493 		entry = entry1 = __ioapic_read_entry(apic, pin);
494 
495 		/*
496 		 * Mask the entry and change the trigger mode to edge.
497 		 */
498 		entry1.masked = true;
499 		entry1.is_level = false;
500 
501 		__ioapic_write_entry(apic, pin, entry1);
502 
503 		/*
504 		 * Restore the previous level triggered entry.
505 		 */
506 		__ioapic_write_entry(apic, pin, entry);
507 	}
508 }
509 
eoi_ioapic_pin(int vector,struct mp_chip_data * data)510 static void eoi_ioapic_pin(int vector, struct mp_chip_data *data)
511 {
512 	unsigned long flags;
513 	struct irq_pin_list *entry;
514 
515 	raw_spin_lock_irqsave(&ioapic_lock, flags);
516 	for_each_irq_pin(entry, data->irq_2_pin)
517 		__eoi_ioapic_pin(entry->apic, entry->pin, vector);
518 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
519 }
520 
clear_IO_APIC_pin(unsigned int apic,unsigned int pin)521 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
522 {
523 	struct IO_APIC_route_entry entry;
524 
525 	/* Check delivery_mode to be sure we're not clearing an SMI pin */
526 	entry = ioapic_read_entry(apic, pin);
527 	if (entry.delivery_mode == APIC_DELIVERY_MODE_SMI)
528 		return;
529 
530 	/*
531 	 * Make sure the entry is masked and re-read the contents to check
532 	 * if it is a level triggered pin and if the remote-IRR is set.
533 	 */
534 	if (!entry.masked) {
535 		entry.masked = true;
536 		ioapic_write_entry(apic, pin, entry);
537 		entry = ioapic_read_entry(apic, pin);
538 	}
539 
540 	if (entry.irr) {
541 		unsigned long flags;
542 
543 		/*
544 		 * Make sure the trigger mode is set to level. Explicit EOI
545 		 * doesn't clear the remote-IRR if the trigger mode is not
546 		 * set to level.
547 		 */
548 		if (!entry.is_level) {
549 			entry.is_level = true;
550 			ioapic_write_entry(apic, pin, entry);
551 		}
552 		raw_spin_lock_irqsave(&ioapic_lock, flags);
553 		__eoi_ioapic_pin(apic, pin, entry.vector);
554 		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
555 	}
556 
557 	/*
558 	 * Clear the rest of the bits in the IO-APIC RTE except for the mask
559 	 * bit.
560 	 */
561 	ioapic_mask_entry(apic, pin);
562 	entry = ioapic_read_entry(apic, pin);
563 	if (entry.irr)
564 		pr_err("Unable to reset IRR for apic: %d, pin :%d\n",
565 		       mpc_ioapic_id(apic), pin);
566 }
567 
clear_IO_APIC(void)568 void clear_IO_APIC (void)
569 {
570 	int apic, pin;
571 
572 	for_each_ioapic_pin(apic, pin)
573 		clear_IO_APIC_pin(apic, pin);
574 }
575 
576 #ifdef CONFIG_X86_32
577 /*
578  * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
579  * specific CPU-side IRQs.
580  */
581 
582 #define MAX_PIRQS 8
583 static int pirq_entries[MAX_PIRQS] = {
584 	[0 ... MAX_PIRQS - 1] = -1
585 };
586 
ioapic_pirq_setup(char * str)587 static int __init ioapic_pirq_setup(char *str)
588 {
589 	int i, max;
590 	int ints[MAX_PIRQS+1];
591 
592 	get_options(str, ARRAY_SIZE(ints), ints);
593 
594 	apic_printk(APIC_VERBOSE, KERN_INFO
595 			"PIRQ redirection, working around broken MP-BIOS.\n");
596 	max = MAX_PIRQS;
597 	if (ints[0] < MAX_PIRQS)
598 		max = ints[0];
599 
600 	for (i = 0; i < max; i++) {
601 		apic_printk(APIC_VERBOSE, KERN_DEBUG
602 				"... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
603 		/*
604 		 * PIRQs are mapped upside down, usually.
605 		 */
606 		pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
607 	}
608 	return 1;
609 }
610 
611 __setup("pirq=", ioapic_pirq_setup);
612 #endif /* CONFIG_X86_32 */
613 
614 /*
615  * Saves all the IO-APIC RTE's
616  */
save_ioapic_entries(void)617 int save_ioapic_entries(void)
618 {
619 	int apic, pin;
620 	int err = 0;
621 
622 	for_each_ioapic(apic) {
623 		if (!ioapics[apic].saved_registers) {
624 			err = -ENOMEM;
625 			continue;
626 		}
627 
628 		for_each_pin(apic, pin)
629 			ioapics[apic].saved_registers[pin] =
630 				ioapic_read_entry(apic, pin);
631 	}
632 
633 	return err;
634 }
635 
636 /*
637  * Mask all IO APIC entries.
638  */
mask_ioapic_entries(void)639 void mask_ioapic_entries(void)
640 {
641 	int apic, pin;
642 
643 	for_each_ioapic(apic) {
644 		if (!ioapics[apic].saved_registers)
645 			continue;
646 
647 		for_each_pin(apic, pin) {
648 			struct IO_APIC_route_entry entry;
649 
650 			entry = ioapics[apic].saved_registers[pin];
651 			if (!entry.masked) {
652 				entry.masked = true;
653 				ioapic_write_entry(apic, pin, entry);
654 			}
655 		}
656 	}
657 }
658 
659 /*
660  * Restore IO APIC entries which was saved in the ioapic structure.
661  */
restore_ioapic_entries(void)662 int restore_ioapic_entries(void)
663 {
664 	int apic, pin;
665 
666 	for_each_ioapic(apic) {
667 		if (!ioapics[apic].saved_registers)
668 			continue;
669 
670 		for_each_pin(apic, pin)
671 			ioapic_write_entry(apic, pin,
672 					   ioapics[apic].saved_registers[pin]);
673 	}
674 	return 0;
675 }
676 
677 /*
678  * Find the IRQ entry number of a certain pin.
679  */
find_irq_entry(int ioapic_idx,int pin,int type)680 static int find_irq_entry(int ioapic_idx, int pin, int type)
681 {
682 	int i;
683 
684 	for (i = 0; i < mp_irq_entries; i++)
685 		if (mp_irqs[i].irqtype == type &&
686 		    (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
687 		     mp_irqs[i].dstapic == MP_APIC_ALL) &&
688 		    mp_irqs[i].dstirq == pin)
689 			return i;
690 
691 	return -1;
692 }
693 
694 /*
695  * Find the pin to which IRQ[irq] (ISA) is connected
696  */
find_isa_irq_pin(int irq,int type)697 static int __init find_isa_irq_pin(int irq, int type)
698 {
699 	int i;
700 
701 	for (i = 0; i < mp_irq_entries; i++) {
702 		int lbus = mp_irqs[i].srcbus;
703 
704 		if (test_bit(lbus, mp_bus_not_pci) &&
705 		    (mp_irqs[i].irqtype == type) &&
706 		    (mp_irqs[i].srcbusirq == irq))
707 
708 			return mp_irqs[i].dstirq;
709 	}
710 	return -1;
711 }
712 
find_isa_irq_apic(int irq,int type)713 static int __init find_isa_irq_apic(int irq, int type)
714 {
715 	int i;
716 
717 	for (i = 0; i < mp_irq_entries; i++) {
718 		int lbus = mp_irqs[i].srcbus;
719 
720 		if (test_bit(lbus, mp_bus_not_pci) &&
721 		    (mp_irqs[i].irqtype == type) &&
722 		    (mp_irqs[i].srcbusirq == irq))
723 			break;
724 	}
725 
726 	if (i < mp_irq_entries) {
727 		int ioapic_idx;
728 
729 		for_each_ioapic(ioapic_idx)
730 			if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
731 				return ioapic_idx;
732 	}
733 
734 	return -1;
735 }
736 
irq_active_low(int idx)737 static bool irq_active_low(int idx)
738 {
739 	int bus = mp_irqs[idx].srcbus;
740 
741 	/*
742 	 * Determine IRQ line polarity (high active or low active):
743 	 */
744 	switch (mp_irqs[idx].irqflag & MP_IRQPOL_MASK) {
745 	case MP_IRQPOL_DEFAULT:
746 		/*
747 		 * Conforms to spec, ie. bus-type dependent polarity.  PCI
748 		 * defaults to low active. [E]ISA defaults to high active.
749 		 */
750 		return !test_bit(bus, mp_bus_not_pci);
751 	case MP_IRQPOL_ACTIVE_HIGH:
752 		return false;
753 	case MP_IRQPOL_RESERVED:
754 		pr_warn("IOAPIC: Invalid polarity: 2, defaulting to low\n");
755 		fallthrough;
756 	case MP_IRQPOL_ACTIVE_LOW:
757 	default: /* Pointless default required due to do gcc stupidity */
758 		return true;
759 	}
760 }
761 
762 #ifdef CONFIG_EISA
763 /*
764  * EISA Edge/Level control register, ELCR
765  */
EISA_ELCR(unsigned int irq)766 static bool EISA_ELCR(unsigned int irq)
767 {
768 	if (irq < nr_legacy_irqs()) {
769 		unsigned int port = PIC_ELCR1 + (irq >> 3);
770 		return (inb(port) >> (irq & 7)) & 1;
771 	}
772 	apic_printk(APIC_VERBOSE, KERN_INFO
773 			"Broken MPtable reports ISA irq %d\n", irq);
774 	return false;
775 }
776 
777 /*
778  * EISA interrupts are always active high and can be edge or level
779  * triggered depending on the ELCR value.  If an interrupt is listed as
780  * EISA conforming in the MP table, that means its trigger type must be
781  * read in from the ELCR.
782  */
eisa_irq_is_level(int idx,int bus,bool level)783 static bool eisa_irq_is_level(int idx, int bus, bool level)
784 {
785 	switch (mp_bus_id_to_type[bus]) {
786 	case MP_BUS_PCI:
787 	case MP_BUS_ISA:
788 		return level;
789 	case MP_BUS_EISA:
790 		return EISA_ELCR(mp_irqs[idx].srcbusirq);
791 	}
792 	pr_warn("IOAPIC: Invalid srcbus: %d defaulting to level\n", bus);
793 	return true;
794 }
795 #else
eisa_irq_is_level(int idx,int bus,bool level)796 static inline int eisa_irq_is_level(int idx, int bus, bool level)
797 {
798 	return level;
799 }
800 #endif
801 
irq_is_level(int idx)802 static bool irq_is_level(int idx)
803 {
804 	int bus = mp_irqs[idx].srcbus;
805 	bool level;
806 
807 	/*
808 	 * Determine IRQ trigger mode (edge or level sensitive):
809 	 */
810 	switch (mp_irqs[idx].irqflag & MP_IRQTRIG_MASK) {
811 	case MP_IRQTRIG_DEFAULT:
812 		/*
813 		 * Conforms to spec, ie. bus-type dependent trigger
814 		 * mode. PCI defaults to level, ISA to edge.
815 		 */
816 		level = !test_bit(bus, mp_bus_not_pci);
817 		/* Take EISA into account */
818 		return eisa_irq_is_level(idx, bus, level);
819 	case MP_IRQTRIG_EDGE:
820 		return false;
821 	case MP_IRQTRIG_RESERVED:
822 		pr_warn("IOAPIC: Invalid trigger mode 2 defaulting to level\n");
823 		fallthrough;
824 	case MP_IRQTRIG_LEVEL:
825 	default: /* Pointless default required due to do gcc stupidity */
826 		return true;
827 	}
828 }
829 
__acpi_get_override_irq(u32 gsi,bool * trigger,bool * polarity)830 static int __acpi_get_override_irq(u32 gsi, bool *trigger, bool *polarity)
831 {
832 	int ioapic, pin, idx;
833 
834 	if (ioapic_is_disabled)
835 		return -1;
836 
837 	ioapic = mp_find_ioapic(gsi);
838 	if (ioapic < 0)
839 		return -1;
840 
841 	pin = mp_find_ioapic_pin(ioapic, gsi);
842 	if (pin < 0)
843 		return -1;
844 
845 	idx = find_irq_entry(ioapic, pin, mp_INT);
846 	if (idx < 0)
847 		return -1;
848 
849 	*trigger = irq_is_level(idx);
850 	*polarity = irq_active_low(idx);
851 	return 0;
852 }
853 
854 #ifdef CONFIG_ACPI
acpi_get_override_irq(u32 gsi,int * is_level,int * active_low)855 int acpi_get_override_irq(u32 gsi, int *is_level, int *active_low)
856 {
857 	*is_level = *active_low = 0;
858 	return __acpi_get_override_irq(gsi, (bool *)is_level,
859 				       (bool *)active_low);
860 }
861 #endif
862 
ioapic_set_alloc_attr(struct irq_alloc_info * info,int node,int trigger,int polarity)863 void ioapic_set_alloc_attr(struct irq_alloc_info *info, int node,
864 			   int trigger, int polarity)
865 {
866 	init_irq_alloc_info(info, NULL);
867 	info->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
868 	info->ioapic.node = node;
869 	info->ioapic.is_level = trigger;
870 	info->ioapic.active_low = polarity;
871 	info->ioapic.valid = 1;
872 }
873 
ioapic_copy_alloc_attr(struct irq_alloc_info * dst,struct irq_alloc_info * src,u32 gsi,int ioapic_idx,int pin)874 static void ioapic_copy_alloc_attr(struct irq_alloc_info *dst,
875 				   struct irq_alloc_info *src,
876 				   u32 gsi, int ioapic_idx, int pin)
877 {
878 	bool level, pol_low;
879 
880 	copy_irq_alloc_info(dst, src);
881 	dst->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
882 	dst->devid = mpc_ioapic_id(ioapic_idx);
883 	dst->ioapic.pin = pin;
884 	dst->ioapic.valid = 1;
885 	if (src && src->ioapic.valid) {
886 		dst->ioapic.node = src->ioapic.node;
887 		dst->ioapic.is_level = src->ioapic.is_level;
888 		dst->ioapic.active_low = src->ioapic.active_low;
889 	} else {
890 		dst->ioapic.node = NUMA_NO_NODE;
891 		if (__acpi_get_override_irq(gsi, &level, &pol_low) >= 0) {
892 			dst->ioapic.is_level = level;
893 			dst->ioapic.active_low = pol_low;
894 		} else {
895 			/*
896 			 * PCI interrupts are always active low level
897 			 * triggered.
898 			 */
899 			dst->ioapic.is_level = true;
900 			dst->ioapic.active_low = true;
901 		}
902 	}
903 }
904 
ioapic_alloc_attr_node(struct irq_alloc_info * info)905 static int ioapic_alloc_attr_node(struct irq_alloc_info *info)
906 {
907 	return (info && info->ioapic.valid) ? info->ioapic.node : NUMA_NO_NODE;
908 }
909 
mp_register_handler(unsigned int irq,bool level)910 static void mp_register_handler(unsigned int irq, bool level)
911 {
912 	irq_flow_handler_t hdl;
913 	bool fasteoi;
914 
915 	if (level) {
916 		irq_set_status_flags(irq, IRQ_LEVEL);
917 		fasteoi = true;
918 	} else {
919 		irq_clear_status_flags(irq, IRQ_LEVEL);
920 		fasteoi = false;
921 	}
922 
923 	hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
924 	__irq_set_handler(irq, hdl, 0, fasteoi ? "fasteoi" : "edge");
925 }
926 
mp_check_pin_attr(int irq,struct irq_alloc_info * info)927 static bool mp_check_pin_attr(int irq, struct irq_alloc_info *info)
928 {
929 	struct mp_chip_data *data = irq_get_chip_data(irq);
930 
931 	/*
932 	 * setup_IO_APIC_irqs() programs all legacy IRQs with default trigger
933 	 * and polarity attributes. So allow the first user to reprogram the
934 	 * pin with real trigger and polarity attributes.
935 	 */
936 	if (irq < nr_legacy_irqs() && data->count == 1) {
937 		if (info->ioapic.is_level != data->is_level)
938 			mp_register_handler(irq, info->ioapic.is_level);
939 		data->entry.is_level = data->is_level = info->ioapic.is_level;
940 		data->entry.active_low = data->active_low = info->ioapic.active_low;
941 	}
942 
943 	return data->is_level == info->ioapic.is_level &&
944 	       data->active_low == info->ioapic.active_low;
945 }
946 
alloc_irq_from_domain(struct irq_domain * domain,int ioapic,u32 gsi,struct irq_alloc_info * info)947 static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi,
948 				 struct irq_alloc_info *info)
949 {
950 	bool legacy = false;
951 	int irq = -1;
952 	int type = ioapics[ioapic].irqdomain_cfg.type;
953 
954 	switch (type) {
955 	case IOAPIC_DOMAIN_LEGACY:
956 		/*
957 		 * Dynamically allocate IRQ number for non-ISA IRQs in the first
958 		 * 16 GSIs on some weird platforms.
959 		 */
960 		if (!ioapic_initialized || gsi >= nr_legacy_irqs())
961 			irq = gsi;
962 		legacy = mp_is_legacy_irq(irq);
963 		break;
964 	case IOAPIC_DOMAIN_STRICT:
965 		irq = gsi;
966 		break;
967 	case IOAPIC_DOMAIN_DYNAMIC:
968 		break;
969 	default:
970 		WARN(1, "ioapic: unknown irqdomain type %d\n", type);
971 		return -1;
972 	}
973 
974 	return __irq_domain_alloc_irqs(domain, irq, 1,
975 				       ioapic_alloc_attr_node(info),
976 				       info, legacy, NULL);
977 }
978 
979 /*
980  * Need special handling for ISA IRQs because there may be multiple IOAPIC pins
981  * sharing the same ISA IRQ number and irqdomain only supports 1:1 mapping
982  * between IOAPIC pin and IRQ number. A typical IOAPIC has 24 pins, pin 0-15 are
983  * used for legacy IRQs and pin 16-23 are used for PCI IRQs (PIRQ A-H).
984  * When ACPI is disabled, only legacy IRQ numbers (IRQ0-15) are available, and
985  * some BIOSes may use MP Interrupt Source records to override IRQ numbers for
986  * PIRQs instead of reprogramming the interrupt routing logic. Thus there may be
987  * multiple pins sharing the same legacy IRQ number when ACPI is disabled.
988  */
alloc_isa_irq_from_domain(struct irq_domain * domain,int irq,int ioapic,int pin,struct irq_alloc_info * info)989 static int alloc_isa_irq_from_domain(struct irq_domain *domain,
990 				     int irq, int ioapic, int pin,
991 				     struct irq_alloc_info *info)
992 {
993 	struct mp_chip_data *data;
994 	struct irq_data *irq_data = irq_get_irq_data(irq);
995 	int node = ioapic_alloc_attr_node(info);
996 
997 	/*
998 	 * Legacy ISA IRQ has already been allocated, just add pin to
999 	 * the pin list associated with this IRQ and program the IOAPIC
1000 	 * entry. The IOAPIC entry
1001 	 */
1002 	if (irq_data && irq_data->parent_data) {
1003 		if (!mp_check_pin_attr(irq, info))
1004 			return -EBUSY;
1005 		if (__add_pin_to_irq_node(irq_data->chip_data, node, ioapic,
1006 					  info->ioapic.pin))
1007 			return -ENOMEM;
1008 	} else {
1009 		info->flags |= X86_IRQ_ALLOC_LEGACY;
1010 		irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true,
1011 					      NULL);
1012 		if (irq >= 0) {
1013 			irq_data = irq_domain_get_irq_data(domain, irq);
1014 			data = irq_data->chip_data;
1015 			data->isa_irq = true;
1016 		}
1017 	}
1018 
1019 	return irq;
1020 }
1021 
mp_map_pin_to_irq(u32 gsi,int idx,int ioapic,int pin,unsigned int flags,struct irq_alloc_info * info)1022 static int mp_map_pin_to_irq(u32 gsi, int idx, int ioapic, int pin,
1023 			     unsigned int flags, struct irq_alloc_info *info)
1024 {
1025 	int irq;
1026 	bool legacy = false;
1027 	struct irq_alloc_info tmp;
1028 	struct mp_chip_data *data;
1029 	struct irq_domain *domain = mp_ioapic_irqdomain(ioapic);
1030 
1031 	if (!domain)
1032 		return -ENOSYS;
1033 
1034 	if (idx >= 0 && test_bit(mp_irqs[idx].srcbus, mp_bus_not_pci)) {
1035 		irq = mp_irqs[idx].srcbusirq;
1036 		legacy = mp_is_legacy_irq(irq);
1037 		/*
1038 		 * IRQ2 is unusable for historical reasons on systems which
1039 		 * have a legacy PIC. See the comment vs. IRQ2 further down.
1040 		 *
1041 		 * If this gets removed at some point then the related code
1042 		 * in lapic_assign_system_vectors() needs to be adjusted as
1043 		 * well.
1044 		 */
1045 		if (legacy && irq == PIC_CASCADE_IR)
1046 			return -EINVAL;
1047 	}
1048 
1049 	mutex_lock(&ioapic_mutex);
1050 	if (!(flags & IOAPIC_MAP_ALLOC)) {
1051 		if (!legacy) {
1052 			irq = irq_find_mapping(domain, pin);
1053 			if (irq == 0)
1054 				irq = -ENOENT;
1055 		}
1056 	} else {
1057 		ioapic_copy_alloc_attr(&tmp, info, gsi, ioapic, pin);
1058 		if (legacy)
1059 			irq = alloc_isa_irq_from_domain(domain, irq,
1060 							ioapic, pin, &tmp);
1061 		else if ((irq = irq_find_mapping(domain, pin)) == 0)
1062 			irq = alloc_irq_from_domain(domain, ioapic, gsi, &tmp);
1063 		else if (!mp_check_pin_attr(irq, &tmp))
1064 			irq = -EBUSY;
1065 		if (irq >= 0) {
1066 			data = irq_get_chip_data(irq);
1067 			data->count++;
1068 		}
1069 	}
1070 	mutex_unlock(&ioapic_mutex);
1071 
1072 	return irq;
1073 }
1074 
pin_2_irq(int idx,int ioapic,int pin,unsigned int flags)1075 static int pin_2_irq(int idx, int ioapic, int pin, unsigned int flags)
1076 {
1077 	u32 gsi = mp_pin_to_gsi(ioapic, pin);
1078 
1079 	/*
1080 	 * Debugging check, we are in big trouble if this message pops up!
1081 	 */
1082 	if (mp_irqs[idx].dstirq != pin)
1083 		pr_err("broken BIOS or MPTABLE parser, ayiee!!\n");
1084 
1085 #ifdef CONFIG_X86_32
1086 	/*
1087 	 * PCI IRQ command line redirection. Yes, limits are hardcoded.
1088 	 */
1089 	if ((pin >= 16) && (pin <= 23)) {
1090 		if (pirq_entries[pin-16] != -1) {
1091 			if (!pirq_entries[pin-16]) {
1092 				apic_printk(APIC_VERBOSE, KERN_DEBUG
1093 						"disabling PIRQ%d\n", pin-16);
1094 			} else {
1095 				int irq = pirq_entries[pin-16];
1096 				apic_printk(APIC_VERBOSE, KERN_DEBUG
1097 						"using PIRQ%d -> IRQ %d\n",
1098 						pin-16, irq);
1099 				return irq;
1100 			}
1101 		}
1102 	}
1103 #endif
1104 
1105 	return  mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, NULL);
1106 }
1107 
mp_map_gsi_to_irq(u32 gsi,unsigned int flags,struct irq_alloc_info * info)1108 int mp_map_gsi_to_irq(u32 gsi, unsigned int flags, struct irq_alloc_info *info)
1109 {
1110 	int ioapic, pin, idx;
1111 
1112 	ioapic = mp_find_ioapic(gsi);
1113 	if (ioapic < 0)
1114 		return -ENODEV;
1115 
1116 	pin = mp_find_ioapic_pin(ioapic, gsi);
1117 	idx = find_irq_entry(ioapic, pin, mp_INT);
1118 	if ((flags & IOAPIC_MAP_CHECK) && idx < 0)
1119 		return -ENODEV;
1120 
1121 	return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, info);
1122 }
1123 
mp_unmap_irq(int irq)1124 void mp_unmap_irq(int irq)
1125 {
1126 	struct irq_data *irq_data = irq_get_irq_data(irq);
1127 	struct mp_chip_data *data;
1128 
1129 	if (!irq_data || !irq_data->domain)
1130 		return;
1131 
1132 	data = irq_data->chip_data;
1133 	if (!data || data->isa_irq)
1134 		return;
1135 
1136 	mutex_lock(&ioapic_mutex);
1137 	if (--data->count == 0)
1138 		irq_domain_free_irqs(irq, 1);
1139 	mutex_unlock(&ioapic_mutex);
1140 }
1141 
1142 /*
1143  * Find a specific PCI IRQ entry.
1144  * Not an __init, possibly needed by modules
1145  */
IO_APIC_get_PCI_irq_vector(int bus,int slot,int pin)1146 int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
1147 {
1148 	int irq, i, best_ioapic = -1, best_idx = -1;
1149 
1150 	apic_printk(APIC_DEBUG,
1151 		    "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1152 		    bus, slot, pin);
1153 	if (test_bit(bus, mp_bus_not_pci)) {
1154 		apic_printk(APIC_VERBOSE,
1155 			    "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1156 		return -1;
1157 	}
1158 
1159 	for (i = 0; i < mp_irq_entries; i++) {
1160 		int lbus = mp_irqs[i].srcbus;
1161 		int ioapic_idx, found = 0;
1162 
1163 		if (bus != lbus || mp_irqs[i].irqtype != mp_INT ||
1164 		    slot != ((mp_irqs[i].srcbusirq >> 2) & 0x1f))
1165 			continue;
1166 
1167 		for_each_ioapic(ioapic_idx)
1168 			if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1169 			    mp_irqs[i].dstapic == MP_APIC_ALL) {
1170 				found = 1;
1171 				break;
1172 			}
1173 		if (!found)
1174 			continue;
1175 
1176 		/* Skip ISA IRQs */
1177 		irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq, 0);
1178 		if (irq > 0 && !IO_APIC_IRQ(irq))
1179 			continue;
1180 
1181 		if (pin == (mp_irqs[i].srcbusirq & 3)) {
1182 			best_idx = i;
1183 			best_ioapic = ioapic_idx;
1184 			goto out;
1185 		}
1186 
1187 		/*
1188 		 * Use the first all-but-pin matching entry as a
1189 		 * best-guess fuzzy result for broken mptables.
1190 		 */
1191 		if (best_idx < 0) {
1192 			best_idx = i;
1193 			best_ioapic = ioapic_idx;
1194 		}
1195 	}
1196 	if (best_idx < 0)
1197 		return -1;
1198 
1199 out:
1200 	return pin_2_irq(best_idx, best_ioapic, mp_irqs[best_idx].dstirq,
1201 			 IOAPIC_MAP_ALLOC);
1202 }
1203 EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1204 
1205 static struct irq_chip ioapic_chip, ioapic_ir_chip;
1206 
setup_IO_APIC_irqs(void)1207 static void __init setup_IO_APIC_irqs(void)
1208 {
1209 	unsigned int ioapic, pin;
1210 	int idx;
1211 
1212 	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1213 
1214 	for_each_ioapic_pin(ioapic, pin) {
1215 		idx = find_irq_entry(ioapic, pin, mp_INT);
1216 		if (idx < 0)
1217 			apic_printk(APIC_VERBOSE,
1218 				    KERN_DEBUG " apic %d pin %d not connected\n",
1219 				    mpc_ioapic_id(ioapic), pin);
1220 		else
1221 			pin_2_irq(idx, ioapic, pin,
1222 				  ioapic ? 0 : IOAPIC_MAP_ALLOC);
1223 	}
1224 }
1225 
ioapic_zap_locks(void)1226 void ioapic_zap_locks(void)
1227 {
1228 	raw_spin_lock_init(&ioapic_lock);
1229 }
1230 
io_apic_print_entries(unsigned int apic,unsigned int nr_entries)1231 static void io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
1232 {
1233 	struct IO_APIC_route_entry entry;
1234 	char buf[256];
1235 	int i;
1236 
1237 	printk(KERN_DEBUG "IOAPIC %d:\n", apic);
1238 	for (i = 0; i <= nr_entries; i++) {
1239 		entry = ioapic_read_entry(apic, i);
1240 		snprintf(buf, sizeof(buf),
1241 			 " pin%02x, %s, %s, %s, V(%02X), IRR(%1d), S(%1d)",
1242 			 i,
1243 			 entry.masked ? "disabled" : "enabled ",
1244 			 entry.is_level ? "level" : "edge ",
1245 			 entry.active_low ? "low " : "high",
1246 			 entry.vector, entry.irr, entry.delivery_status);
1247 		if (entry.ir_format) {
1248 			printk(KERN_DEBUG "%s, remapped, I(%04X),  Z(%X)\n",
1249 			       buf,
1250 			       (entry.ir_index_15 << 15) | entry.ir_index_0_14,
1251 				entry.ir_zero);
1252 		} else {
1253 			printk(KERN_DEBUG "%s, %s, D(%02X%02X), M(%1d)\n", buf,
1254 			       entry.dest_mode_logical ? "logical " : "physical",
1255 			       entry.virt_destid_8_14, entry.destid_0_7,
1256 			       entry.delivery_mode);
1257 		}
1258 	}
1259 }
1260 
print_IO_APIC(int ioapic_idx)1261 static void __init print_IO_APIC(int ioapic_idx)
1262 {
1263 	union IO_APIC_reg_00 reg_00;
1264 	union IO_APIC_reg_01 reg_01;
1265 	union IO_APIC_reg_02 reg_02;
1266 	union IO_APIC_reg_03 reg_03;
1267 	unsigned long flags;
1268 
1269 	raw_spin_lock_irqsave(&ioapic_lock, flags);
1270 	reg_00.raw = io_apic_read(ioapic_idx, 0);
1271 	reg_01.raw = io_apic_read(ioapic_idx, 1);
1272 	if (reg_01.bits.version >= 0x10)
1273 		reg_02.raw = io_apic_read(ioapic_idx, 2);
1274 	if (reg_01.bits.version >= 0x20)
1275 		reg_03.raw = io_apic_read(ioapic_idx, 3);
1276 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1277 
1278 	printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
1279 	printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1280 	printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
1281 	printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
1282 	printk(KERN_DEBUG ".......    : LTS          : %X\n", reg_00.bits.LTS);
1283 
1284 	printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
1285 	printk(KERN_DEBUG ".......     : max redirection entries: %02X\n",
1286 		reg_01.bits.entries);
1287 
1288 	printk(KERN_DEBUG ".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
1289 	printk(KERN_DEBUG ".......     : IO APIC version: %02X\n",
1290 		reg_01.bits.version);
1291 
1292 	/*
1293 	 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1294 	 * but the value of reg_02 is read as the previous read register
1295 	 * value, so ignore it if reg_02 == reg_01.
1296 	 */
1297 	if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1298 		printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1299 		printk(KERN_DEBUG ".......     : arbitration: %02X\n", reg_02.bits.arbitration);
1300 	}
1301 
1302 	/*
1303 	 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1304 	 * or reg_03, but the value of reg_0[23] is read as the previous read
1305 	 * register value, so ignore it if reg_03 == reg_0[12].
1306 	 */
1307 	if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1308 	    reg_03.raw != reg_01.raw) {
1309 		printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1310 		printk(KERN_DEBUG ".......     : Boot DT    : %X\n", reg_03.bits.boot_DT);
1311 	}
1312 
1313 	printk(KERN_DEBUG ".... IRQ redirection table:\n");
1314 	io_apic_print_entries(ioapic_idx, reg_01.bits.entries);
1315 }
1316 
print_IO_APICs(void)1317 void __init print_IO_APICs(void)
1318 {
1319 	int ioapic_idx;
1320 	unsigned int irq;
1321 
1322 	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1323 	for_each_ioapic(ioapic_idx)
1324 		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1325 		       mpc_ioapic_id(ioapic_idx),
1326 		       ioapics[ioapic_idx].nr_registers);
1327 
1328 	/*
1329 	 * We are a bit conservative about what we expect.  We have to
1330 	 * know about every hardware change ASAP.
1331 	 */
1332 	printk(KERN_INFO "testing the IO APIC.......................\n");
1333 
1334 	for_each_ioapic(ioapic_idx)
1335 		print_IO_APIC(ioapic_idx);
1336 
1337 	printk(KERN_DEBUG "IRQ to pin mappings:\n");
1338 	for_each_active_irq(irq) {
1339 		struct irq_pin_list *entry;
1340 		struct irq_chip *chip;
1341 		struct mp_chip_data *data;
1342 
1343 		chip = irq_get_chip(irq);
1344 		if (chip != &ioapic_chip && chip != &ioapic_ir_chip)
1345 			continue;
1346 		data = irq_get_chip_data(irq);
1347 		if (!data)
1348 			continue;
1349 		if (list_empty(&data->irq_2_pin))
1350 			continue;
1351 
1352 		printk(KERN_DEBUG "IRQ%d ", irq);
1353 		for_each_irq_pin(entry, data->irq_2_pin)
1354 			pr_cont("-> %d:%d", entry->apic, entry->pin);
1355 		pr_cont("\n");
1356 	}
1357 
1358 	printk(KERN_INFO ".................................... done.\n");
1359 }
1360 
1361 /* Where if anywhere is the i8259 connect in external int mode */
1362 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1363 
enable_IO_APIC(void)1364 void __init enable_IO_APIC(void)
1365 {
1366 	int i8259_apic, i8259_pin;
1367 	int apic, pin;
1368 
1369 	if (ioapic_is_disabled)
1370 		nr_ioapics = 0;
1371 
1372 	if (!nr_legacy_irqs() || !nr_ioapics)
1373 		return;
1374 
1375 	for_each_ioapic_pin(apic, pin) {
1376 		/* See if any of the pins is in ExtINT mode */
1377 		struct IO_APIC_route_entry entry = ioapic_read_entry(apic, pin);
1378 
1379 		/* If the interrupt line is enabled and in ExtInt mode
1380 		 * I have found the pin where the i8259 is connected.
1381 		 */
1382 		if (!entry.masked &&
1383 		    entry.delivery_mode == APIC_DELIVERY_MODE_EXTINT) {
1384 			ioapic_i8259.apic = apic;
1385 			ioapic_i8259.pin  = pin;
1386 			goto found_i8259;
1387 		}
1388 	}
1389  found_i8259:
1390 	/* Look to see what if the MP table has reported the ExtINT */
1391 	/* If we could not find the appropriate pin by looking at the ioapic
1392 	 * the i8259 probably is not connected the ioapic but give the
1393 	 * mptable a chance anyway.
1394 	 */
1395 	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
1396 	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1397 	/* Trust the MP table if nothing is setup in the hardware */
1398 	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1399 		printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1400 		ioapic_i8259.pin  = i8259_pin;
1401 		ioapic_i8259.apic = i8259_apic;
1402 	}
1403 	/* Complain if the MP table and the hardware disagree */
1404 	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1405 		(i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1406 	{
1407 		printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1408 	}
1409 
1410 	/*
1411 	 * Do not trust the IO-APIC being empty at bootup
1412 	 */
1413 	clear_IO_APIC();
1414 }
1415 
native_restore_boot_irq_mode(void)1416 void native_restore_boot_irq_mode(void)
1417 {
1418 	/*
1419 	 * If the i8259 is routed through an IOAPIC
1420 	 * Put that IOAPIC in virtual wire mode
1421 	 * so legacy interrupts can be delivered.
1422 	 */
1423 	if (ioapic_i8259.pin != -1) {
1424 		struct IO_APIC_route_entry entry;
1425 		u32 apic_id = read_apic_id();
1426 
1427 		memset(&entry, 0, sizeof(entry));
1428 		entry.masked		= false;
1429 		entry.is_level		= false;
1430 		entry.active_low	= false;
1431 		entry.dest_mode_logical	= false;
1432 		entry.delivery_mode	= APIC_DELIVERY_MODE_EXTINT;
1433 		entry.destid_0_7	= apic_id & 0xFF;
1434 		entry.virt_destid_8_14	= apic_id >> 8;
1435 
1436 		/*
1437 		 * Add it to the IO-APIC irq-routing table:
1438 		 */
1439 		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1440 	}
1441 
1442 	if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config())
1443 		disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1444 }
1445 
restore_boot_irq_mode(void)1446 void restore_boot_irq_mode(void)
1447 {
1448 	if (!nr_legacy_irqs())
1449 		return;
1450 
1451 	x86_apic_ops.restore();
1452 }
1453 
1454 #ifdef CONFIG_X86_32
1455 /*
1456  * function to set the IO-APIC physical IDs based on the
1457  * values stored in the MPC table.
1458  *
1459  * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
1460  */
setup_ioapic_ids_from_mpc_nocheck(void)1461 void __init setup_ioapic_ids_from_mpc_nocheck(void)
1462 {
1463 	union IO_APIC_reg_00 reg_00;
1464 	physid_mask_t phys_id_present_map;
1465 	int ioapic_idx;
1466 	int i;
1467 	unsigned char old_id;
1468 	unsigned long flags;
1469 
1470 	/*
1471 	 * This is broken; anything with a real cpu count has to
1472 	 * circumvent this idiocy regardless.
1473 	 */
1474 	apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
1475 
1476 	/*
1477 	 * Set the IOAPIC ID to the value stored in the MPC table.
1478 	 */
1479 	for_each_ioapic(ioapic_idx) {
1480 		/* Read the register 0 value */
1481 		raw_spin_lock_irqsave(&ioapic_lock, flags);
1482 		reg_00.raw = io_apic_read(ioapic_idx, 0);
1483 		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1484 
1485 		old_id = mpc_ioapic_id(ioapic_idx);
1486 
1487 		if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
1488 			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1489 				ioapic_idx, mpc_ioapic_id(ioapic_idx));
1490 			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1491 				reg_00.bits.ID);
1492 			ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
1493 		}
1494 
1495 		/*
1496 		 * Sanity check, is the ID really free? Every APIC in a
1497 		 * system must have a unique ID or we get lots of nice
1498 		 * 'stuck on smp_invalidate_needed IPI wait' messages.
1499 		 */
1500 		if (apic->check_apicid_used(&phys_id_present_map,
1501 					    mpc_ioapic_id(ioapic_idx))) {
1502 			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1503 				ioapic_idx, mpc_ioapic_id(ioapic_idx));
1504 			for (i = 0; i < get_physical_broadcast(); i++)
1505 				if (!physid_isset(i, phys_id_present_map))
1506 					break;
1507 			if (i >= get_physical_broadcast())
1508 				panic("Max APIC ID exceeded!\n");
1509 			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1510 				i);
1511 			physid_set(i, phys_id_present_map);
1512 			ioapics[ioapic_idx].mp_config.apicid = i;
1513 		} else {
1514 			apic_printk(APIC_VERBOSE, "Setting %d in the phys_id_present_map\n",
1515 				    mpc_ioapic_id(ioapic_idx));
1516 			physid_set(mpc_ioapic_id(ioapic_idx), phys_id_present_map);
1517 		}
1518 
1519 		/*
1520 		 * We need to adjust the IRQ routing table
1521 		 * if the ID changed.
1522 		 */
1523 		if (old_id != mpc_ioapic_id(ioapic_idx))
1524 			for (i = 0; i < mp_irq_entries; i++)
1525 				if (mp_irqs[i].dstapic == old_id)
1526 					mp_irqs[i].dstapic
1527 						= mpc_ioapic_id(ioapic_idx);
1528 
1529 		/*
1530 		 * Update the ID register according to the right value
1531 		 * from the MPC table if they are different.
1532 		 */
1533 		if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
1534 			continue;
1535 
1536 		apic_printk(APIC_VERBOSE, KERN_INFO
1537 			"...changing IO-APIC physical APIC ID to %d ...",
1538 			mpc_ioapic_id(ioapic_idx));
1539 
1540 		reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
1541 		raw_spin_lock_irqsave(&ioapic_lock, flags);
1542 		io_apic_write(ioapic_idx, 0, reg_00.raw);
1543 		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1544 
1545 		/*
1546 		 * Sanity check
1547 		 */
1548 		raw_spin_lock_irqsave(&ioapic_lock, flags);
1549 		reg_00.raw = io_apic_read(ioapic_idx, 0);
1550 		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1551 		if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
1552 			pr_cont("could not set ID!\n");
1553 		else
1554 			apic_printk(APIC_VERBOSE, " ok.\n");
1555 	}
1556 }
1557 
setup_ioapic_ids_from_mpc(void)1558 void __init setup_ioapic_ids_from_mpc(void)
1559 {
1560 
1561 	if (acpi_ioapic)
1562 		return;
1563 	/*
1564 	 * Don't check I/O APIC IDs for xAPIC systems.  They have
1565 	 * no meaning without the serial APIC bus.
1566 	 */
1567 	if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1568 		|| APIC_XAPIC(boot_cpu_apic_version))
1569 		return;
1570 	setup_ioapic_ids_from_mpc_nocheck();
1571 }
1572 #endif
1573 
1574 int no_timer_check __initdata;
1575 
notimercheck(char * s)1576 static int __init notimercheck(char *s)
1577 {
1578 	no_timer_check = 1;
1579 	return 1;
1580 }
1581 __setup("no_timer_check", notimercheck);
1582 
delay_with_tsc(void)1583 static void __init delay_with_tsc(void)
1584 {
1585 	unsigned long long start, now;
1586 	unsigned long end = jiffies + 4;
1587 
1588 	start = rdtsc();
1589 
1590 	/*
1591 	 * We don't know the TSC frequency yet, but waiting for
1592 	 * 40000000000/HZ TSC cycles is safe:
1593 	 * 4 GHz == 10 jiffies
1594 	 * 1 GHz == 40 jiffies
1595 	 */
1596 	do {
1597 		rep_nop();
1598 		now = rdtsc();
1599 	} while ((now - start) < 40000000000ULL / HZ &&
1600 		time_before_eq(jiffies, end));
1601 }
1602 
delay_without_tsc(void)1603 static void __init delay_without_tsc(void)
1604 {
1605 	unsigned long end = jiffies + 4;
1606 	int band = 1;
1607 
1608 	/*
1609 	 * We don't know any frequency yet, but waiting for
1610 	 * 40940000000/HZ cycles is safe:
1611 	 * 4 GHz == 10 jiffies
1612 	 * 1 GHz == 40 jiffies
1613 	 * 1 << 1 + 1 << 2 +...+ 1 << 11 = 4094
1614 	 */
1615 	do {
1616 		__delay(((1U << band++) * 10000000UL) / HZ);
1617 	} while (band < 12 && time_before_eq(jiffies, end));
1618 }
1619 
1620 /*
1621  * There is a nasty bug in some older SMP boards, their mptable lies
1622  * about the timer IRQ. We do the following to work around the situation:
1623  *
1624  *	- timer IRQ defaults to IO-APIC IRQ
1625  *	- if this function detects that timer IRQs are defunct, then we fall
1626  *	  back to ISA timer IRQs
1627  */
timer_irq_works(void)1628 static int __init timer_irq_works(void)
1629 {
1630 	unsigned long t1 = jiffies;
1631 
1632 	if (no_timer_check)
1633 		return 1;
1634 
1635 	local_irq_enable();
1636 	if (boot_cpu_has(X86_FEATURE_TSC))
1637 		delay_with_tsc();
1638 	else
1639 		delay_without_tsc();
1640 
1641 	/*
1642 	 * Expect a few ticks at least, to be sure some possible
1643 	 * glue logic does not lock up after one or two first
1644 	 * ticks in a non-ExtINT mode.  Also the local APIC
1645 	 * might have cached one ExtINT interrupt.  Finally, at
1646 	 * least one tick may be lost due to delays.
1647 	 */
1648 
1649 	local_irq_disable();
1650 
1651 	/* Did jiffies advance? */
1652 	return time_after(jiffies, t1 + 4);
1653 }
1654 
1655 /*
1656  * In the SMP+IOAPIC case it might happen that there are an unspecified
1657  * number of pending IRQ events unhandled. These cases are very rare,
1658  * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
1659  * better to do it this way as thus we do not have to be aware of
1660  * 'pending' interrupts in the IRQ path, except at this point.
1661  */
1662 /*
1663  * Edge triggered needs to resend any interrupt
1664  * that was delayed but this is now handled in the device
1665  * independent code.
1666  */
1667 
1668 /*
1669  * Starting up a edge-triggered IO-APIC interrupt is
1670  * nasty - we need to make sure that we get the edge.
1671  * If it is already asserted for some reason, we need
1672  * return 1 to indicate that is was pending.
1673  *
1674  * This is not complete - we should be able to fake
1675  * an edge even if it isn't on the 8259A...
1676  */
startup_ioapic_irq(struct irq_data * data)1677 static unsigned int startup_ioapic_irq(struct irq_data *data)
1678 {
1679 	int was_pending = 0, irq = data->irq;
1680 	unsigned long flags;
1681 
1682 	raw_spin_lock_irqsave(&ioapic_lock, flags);
1683 	if (irq < nr_legacy_irqs()) {
1684 		legacy_pic->mask(irq);
1685 		if (legacy_pic->irq_pending(irq))
1686 			was_pending = 1;
1687 	}
1688 	__unmask_ioapic(data->chip_data);
1689 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1690 
1691 	return was_pending;
1692 }
1693 
1694 atomic_t irq_mis_count;
1695 
1696 #ifdef CONFIG_GENERIC_PENDING_IRQ
io_apic_level_ack_pending(struct mp_chip_data * data)1697 static bool io_apic_level_ack_pending(struct mp_chip_data *data)
1698 {
1699 	struct irq_pin_list *entry;
1700 	unsigned long flags;
1701 
1702 	raw_spin_lock_irqsave(&ioapic_lock, flags);
1703 	for_each_irq_pin(entry, data->irq_2_pin) {
1704 		struct IO_APIC_route_entry e;
1705 		int pin;
1706 
1707 		pin = entry->pin;
1708 		e.w1 = io_apic_read(entry->apic, 0x10 + pin*2);
1709 		/* Is the remote IRR bit set? */
1710 		if (e.irr) {
1711 			raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1712 			return true;
1713 		}
1714 	}
1715 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1716 
1717 	return false;
1718 }
1719 
ioapic_prepare_move(struct irq_data * data)1720 static inline bool ioapic_prepare_move(struct irq_data *data)
1721 {
1722 	/* If we are moving the IRQ we need to mask it */
1723 	if (unlikely(irqd_is_setaffinity_pending(data))) {
1724 		if (!irqd_irq_masked(data))
1725 			mask_ioapic_irq(data);
1726 		return true;
1727 	}
1728 	return false;
1729 }
1730 
ioapic_finish_move(struct irq_data * data,bool moveit)1731 static inline void ioapic_finish_move(struct irq_data *data, bool moveit)
1732 {
1733 	if (unlikely(moveit)) {
1734 		/* Only migrate the irq if the ack has been received.
1735 		 *
1736 		 * On rare occasions the broadcast level triggered ack gets
1737 		 * delayed going to ioapics, and if we reprogram the
1738 		 * vector while Remote IRR is still set the irq will never
1739 		 * fire again.
1740 		 *
1741 		 * To prevent this scenario we read the Remote IRR bit
1742 		 * of the ioapic.  This has two effects.
1743 		 * - On any sane system the read of the ioapic will
1744 		 *   flush writes (and acks) going to the ioapic from
1745 		 *   this cpu.
1746 		 * - We get to see if the ACK has actually been delivered.
1747 		 *
1748 		 * Based on failed experiments of reprogramming the
1749 		 * ioapic entry from outside of irq context starting
1750 		 * with masking the ioapic entry and then polling until
1751 		 * Remote IRR was clear before reprogramming the
1752 		 * ioapic I don't trust the Remote IRR bit to be
1753 		 * completely accurate.
1754 		 *
1755 		 * However there appears to be no other way to plug
1756 		 * this race, so if the Remote IRR bit is not
1757 		 * accurate and is causing problems then it is a hardware bug
1758 		 * and you can go talk to the chipset vendor about it.
1759 		 */
1760 		if (!io_apic_level_ack_pending(data->chip_data))
1761 			irq_move_masked_irq(data);
1762 		/* If the IRQ is masked in the core, leave it: */
1763 		if (!irqd_irq_masked(data))
1764 			unmask_ioapic_irq(data);
1765 	}
1766 }
1767 #else
ioapic_prepare_move(struct irq_data * data)1768 static inline bool ioapic_prepare_move(struct irq_data *data)
1769 {
1770 	return false;
1771 }
ioapic_finish_move(struct irq_data * data,bool moveit)1772 static inline void ioapic_finish_move(struct irq_data *data, bool moveit)
1773 {
1774 }
1775 #endif
1776 
ioapic_ack_level(struct irq_data * irq_data)1777 static void ioapic_ack_level(struct irq_data *irq_data)
1778 {
1779 	struct irq_cfg *cfg = irqd_cfg(irq_data);
1780 	unsigned long v;
1781 	bool moveit;
1782 	int i;
1783 
1784 	irq_complete_move(cfg);
1785 	moveit = ioapic_prepare_move(irq_data);
1786 
1787 	/*
1788 	 * It appears there is an erratum which affects at least version 0x11
1789 	 * of I/O APIC (that's the 82093AA and cores integrated into various
1790 	 * chipsets).  Under certain conditions a level-triggered interrupt is
1791 	 * erroneously delivered as edge-triggered one but the respective IRR
1792 	 * bit gets set nevertheless.  As a result the I/O unit expects an EOI
1793 	 * message but it will never arrive and further interrupts are blocked
1794 	 * from the source.  The exact reason is so far unknown, but the
1795 	 * phenomenon was observed when two consecutive interrupt requests
1796 	 * from a given source get delivered to the same CPU and the source is
1797 	 * temporarily disabled in between.
1798 	 *
1799 	 * A workaround is to simulate an EOI message manually.  We achieve it
1800 	 * by setting the trigger mode to edge and then to level when the edge
1801 	 * trigger mode gets detected in the TMR of a local APIC for a
1802 	 * level-triggered interrupt.  We mask the source for the time of the
1803 	 * operation to prevent an edge-triggered interrupt escaping meanwhile.
1804 	 * The idea is from Manfred Spraul.  --macro
1805 	 *
1806 	 * Also in the case when cpu goes offline, fixup_irqs() will forward
1807 	 * any unhandled interrupt on the offlined cpu to the new cpu
1808 	 * destination that is handling the corresponding interrupt. This
1809 	 * interrupt forwarding is done via IPI's. Hence, in this case also
1810 	 * level-triggered io-apic interrupt will be seen as an edge
1811 	 * interrupt in the IRR. And we can't rely on the cpu's EOI
1812 	 * to be broadcasted to the IO-APIC's which will clear the remoteIRR
1813 	 * corresponding to the level-triggered interrupt. Hence on IO-APIC's
1814 	 * supporting EOI register, we do an explicit EOI to clear the
1815 	 * remote IRR and on IO-APIC's which don't have an EOI register,
1816 	 * we use the above logic (mask+edge followed by unmask+level) from
1817 	 * Manfred Spraul to clear the remote IRR.
1818 	 */
1819 	i = cfg->vector;
1820 	v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
1821 
1822 	/*
1823 	 * We must acknowledge the irq before we move it or the acknowledge will
1824 	 * not propagate properly.
1825 	 */
1826 	apic_eoi();
1827 
1828 	/*
1829 	 * Tail end of clearing remote IRR bit (either by delivering the EOI
1830 	 * message via io-apic EOI register write or simulating it using
1831 	 * mask+edge followed by unmask+level logic) manually when the
1832 	 * level triggered interrupt is seen as the edge triggered interrupt
1833 	 * at the cpu.
1834 	 */
1835 	if (!(v & (1 << (i & 0x1f)))) {
1836 		atomic_inc(&irq_mis_count);
1837 		eoi_ioapic_pin(cfg->vector, irq_data->chip_data);
1838 	}
1839 
1840 	ioapic_finish_move(irq_data, moveit);
1841 }
1842 
ioapic_ir_ack_level(struct irq_data * irq_data)1843 static void ioapic_ir_ack_level(struct irq_data *irq_data)
1844 {
1845 	struct mp_chip_data *data = irq_data->chip_data;
1846 
1847 	/*
1848 	 * Intr-remapping uses pin number as the virtual vector
1849 	 * in the RTE. Actual vector is programmed in
1850 	 * intr-remapping table entry. Hence for the io-apic
1851 	 * EOI we use the pin number.
1852 	 */
1853 	apic_ack_irq(irq_data);
1854 	eoi_ioapic_pin(data->entry.vector, data);
1855 }
1856 
1857 /*
1858  * The I/OAPIC is just a device for generating MSI messages from legacy
1859  * interrupt pins. Various fields of the RTE translate into bits of the
1860  * resulting MSI which had a historical meaning.
1861  *
1862  * With interrupt remapping, many of those bits have different meanings
1863  * in the underlying MSI, but the way that the I/OAPIC transforms them
1864  * from its RTE to the MSI message is the same. This function allows
1865  * the parent IRQ domain to compose the MSI message, then takes the
1866  * relevant bits to put them in the appropriate places in the RTE in
1867  * order to generate that message when the IRQ happens.
1868  *
1869  * The setup here relies on a preconfigured route entry (is_level,
1870  * active_low, masked) because the parent domain is merely composing the
1871  * generic message routing information which is used for the MSI.
1872  */
ioapic_setup_msg_from_msi(struct irq_data * irq_data,struct IO_APIC_route_entry * entry)1873 static void ioapic_setup_msg_from_msi(struct irq_data *irq_data,
1874 				      struct IO_APIC_route_entry *entry)
1875 {
1876 	struct msi_msg msg;
1877 
1878 	/* Let the parent domain compose the MSI message */
1879 	irq_chip_compose_msi_msg(irq_data, &msg);
1880 
1881 	/*
1882 	 * - Real vector
1883 	 * - DMAR/IR: 8bit subhandle (ioapic.pin)
1884 	 * - AMD/IR:  8bit IRTE index
1885 	 */
1886 	entry->vector			= msg.arch_data.vector;
1887 	/* Delivery mode (for DMAR/IR all 0) */
1888 	entry->delivery_mode		= msg.arch_data.delivery_mode;
1889 	/* Destination mode or DMAR/IR index bit 15 */
1890 	entry->dest_mode_logical	= msg.arch_addr_lo.dest_mode_logical;
1891 	/* DMAR/IR: 1, 0 for all other modes */
1892 	entry->ir_format		= msg.arch_addr_lo.dmar_format;
1893 	/*
1894 	 * - DMAR/IR: index bit 0-14.
1895 	 *
1896 	 * - Virt: If the host supports x2apic without a virtualized IR
1897 	 *	   unit then bit 0-6 of dmar_index_0_14 are providing bit
1898 	 *	   8-14 of the destination id.
1899 	 *
1900 	 * All other modes have bit 0-6 of dmar_index_0_14 cleared and the
1901 	 * topmost 8 bits are destination id bit 0-7 (entry::destid_0_7).
1902 	 */
1903 	entry->ir_index_0_14		= msg.arch_addr_lo.dmar_index_0_14;
1904 }
1905 
ioapic_configure_entry(struct irq_data * irqd)1906 static void ioapic_configure_entry(struct irq_data *irqd)
1907 {
1908 	struct mp_chip_data *mpd = irqd->chip_data;
1909 	struct irq_pin_list *entry;
1910 
1911 	ioapic_setup_msg_from_msi(irqd, &mpd->entry);
1912 
1913 	for_each_irq_pin(entry, mpd->irq_2_pin)
1914 		__ioapic_write_entry(entry->apic, entry->pin, mpd->entry);
1915 }
1916 
ioapic_set_affinity(struct irq_data * irq_data,const struct cpumask * mask,bool force)1917 static int ioapic_set_affinity(struct irq_data *irq_data,
1918 			       const struct cpumask *mask, bool force)
1919 {
1920 	struct irq_data *parent = irq_data->parent_data;
1921 	unsigned long flags;
1922 	int ret;
1923 
1924 	ret = parent->chip->irq_set_affinity(parent, mask, force);
1925 	raw_spin_lock_irqsave(&ioapic_lock, flags);
1926 	if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE)
1927 		ioapic_configure_entry(irq_data);
1928 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1929 
1930 	return ret;
1931 }
1932 
1933 /*
1934  * Interrupt shutdown masks the ioapic pin, but the interrupt might already
1935  * be in flight, but not yet serviced by the target CPU. That means
1936  * __synchronize_hardirq() would return and claim that everything is calmed
1937  * down. So free_irq() would proceed and deactivate the interrupt and free
1938  * resources.
1939  *
1940  * Once the target CPU comes around to service it it will find a cleared
1941  * vector and complain. While the spurious interrupt is harmless, the full
1942  * release of resources might prevent the interrupt from being acknowledged
1943  * which keeps the hardware in a weird state.
1944  *
1945  * Verify that the corresponding Remote-IRR bits are clear.
1946  */
ioapic_irq_get_chip_state(struct irq_data * irqd,enum irqchip_irq_state which,bool * state)1947 static int ioapic_irq_get_chip_state(struct irq_data *irqd,
1948 				   enum irqchip_irq_state which,
1949 				   bool *state)
1950 {
1951 	struct mp_chip_data *mcd = irqd->chip_data;
1952 	struct IO_APIC_route_entry rentry;
1953 	struct irq_pin_list *p;
1954 
1955 	if (which != IRQCHIP_STATE_ACTIVE)
1956 		return -EINVAL;
1957 
1958 	*state = false;
1959 	raw_spin_lock(&ioapic_lock);
1960 	for_each_irq_pin(p, mcd->irq_2_pin) {
1961 		rentry = __ioapic_read_entry(p->apic, p->pin);
1962 		/*
1963 		 * The remote IRR is only valid in level trigger mode. It's
1964 		 * meaning is undefined for edge triggered interrupts and
1965 		 * irrelevant because the IO-APIC treats them as fire and
1966 		 * forget.
1967 		 */
1968 		if (rentry.irr && rentry.is_level) {
1969 			*state = true;
1970 			break;
1971 		}
1972 	}
1973 	raw_spin_unlock(&ioapic_lock);
1974 	return 0;
1975 }
1976 
1977 static struct irq_chip ioapic_chip __read_mostly = {
1978 	.name			= "IO-APIC",
1979 	.irq_startup		= startup_ioapic_irq,
1980 	.irq_mask		= mask_ioapic_irq,
1981 	.irq_unmask		= unmask_ioapic_irq,
1982 	.irq_ack		= irq_chip_ack_parent,
1983 	.irq_eoi		= ioapic_ack_level,
1984 	.irq_set_affinity	= ioapic_set_affinity,
1985 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
1986 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
1987 	.flags			= IRQCHIP_SKIP_SET_WAKE |
1988 				  IRQCHIP_AFFINITY_PRE_STARTUP,
1989 };
1990 
1991 static struct irq_chip ioapic_ir_chip __read_mostly = {
1992 	.name			= "IR-IO-APIC",
1993 	.irq_startup		= startup_ioapic_irq,
1994 	.irq_mask		= mask_ioapic_irq,
1995 	.irq_unmask		= unmask_ioapic_irq,
1996 	.irq_ack		= irq_chip_ack_parent,
1997 	.irq_eoi		= ioapic_ir_ack_level,
1998 	.irq_set_affinity	= ioapic_set_affinity,
1999 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
2000 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
2001 	.flags			= IRQCHIP_SKIP_SET_WAKE |
2002 				  IRQCHIP_AFFINITY_PRE_STARTUP,
2003 };
2004 
init_IO_APIC_traps(void)2005 static inline void init_IO_APIC_traps(void)
2006 {
2007 	struct irq_cfg *cfg;
2008 	unsigned int irq;
2009 
2010 	for_each_active_irq(irq) {
2011 		cfg = irq_cfg(irq);
2012 		if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2013 			/*
2014 			 * Hmm.. We don't have an entry for this,
2015 			 * so default to an old-fashioned 8259
2016 			 * interrupt if we can..
2017 			 */
2018 			if (irq < nr_legacy_irqs())
2019 				legacy_pic->make_irq(irq);
2020 			else
2021 				/* Strange. Oh, well.. */
2022 				irq_set_chip(irq, &no_irq_chip);
2023 		}
2024 	}
2025 }
2026 
2027 /*
2028  * The local APIC irq-chip implementation:
2029  */
2030 
mask_lapic_irq(struct irq_data * data)2031 static void mask_lapic_irq(struct irq_data *data)
2032 {
2033 	unsigned long v;
2034 
2035 	v = apic_read(APIC_LVT0);
2036 	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2037 }
2038 
unmask_lapic_irq(struct irq_data * data)2039 static void unmask_lapic_irq(struct irq_data *data)
2040 {
2041 	unsigned long v;
2042 
2043 	v = apic_read(APIC_LVT0);
2044 	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2045 }
2046 
ack_lapic_irq(struct irq_data * data)2047 static void ack_lapic_irq(struct irq_data *data)
2048 {
2049 	apic_eoi();
2050 }
2051 
2052 static struct irq_chip lapic_chip __read_mostly = {
2053 	.name		= "local-APIC",
2054 	.irq_mask	= mask_lapic_irq,
2055 	.irq_unmask	= unmask_lapic_irq,
2056 	.irq_ack	= ack_lapic_irq,
2057 };
2058 
lapic_register_intr(int irq)2059 static void lapic_register_intr(int irq)
2060 {
2061 	irq_clear_status_flags(irq, IRQ_LEVEL);
2062 	irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2063 				      "edge");
2064 }
2065 
2066 /*
2067  * This looks a bit hackish but it's about the only one way of sending
2068  * a few INTA cycles to 8259As and any associated glue logic.  ICR does
2069  * not support the ExtINT mode, unfortunately.  We need to send these
2070  * cycles as some i82489DX-based boards have glue logic that keeps the
2071  * 8259A interrupt line asserted until INTA.  --macro
2072  */
unlock_ExtINT_logic(void)2073 static inline void __init unlock_ExtINT_logic(void)
2074 {
2075 	int apic, pin, i;
2076 	struct IO_APIC_route_entry entry0, entry1;
2077 	unsigned char save_control, save_freq_select;
2078 	u32 apic_id;
2079 
2080 	pin  = find_isa_irq_pin(8, mp_INT);
2081 	if (pin == -1) {
2082 		WARN_ON_ONCE(1);
2083 		return;
2084 	}
2085 	apic = find_isa_irq_apic(8, mp_INT);
2086 	if (apic == -1) {
2087 		WARN_ON_ONCE(1);
2088 		return;
2089 	}
2090 
2091 	entry0 = ioapic_read_entry(apic, pin);
2092 	clear_IO_APIC_pin(apic, pin);
2093 
2094 	apic_id = read_apic_id();
2095 	memset(&entry1, 0, sizeof(entry1));
2096 
2097 	entry1.dest_mode_logical	= true;
2098 	entry1.masked			= false;
2099 	entry1.destid_0_7		= apic_id & 0xFF;
2100 	entry1.virt_destid_8_14		= apic_id >> 8;
2101 	entry1.delivery_mode		= APIC_DELIVERY_MODE_EXTINT;
2102 	entry1.active_low		= entry0.active_low;
2103 	entry1.is_level			= false;
2104 	entry1.vector = 0;
2105 
2106 	ioapic_write_entry(apic, pin, entry1);
2107 
2108 	save_control = CMOS_READ(RTC_CONTROL);
2109 	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2110 	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2111 		   RTC_FREQ_SELECT);
2112 	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2113 
2114 	i = 100;
2115 	while (i-- > 0) {
2116 		mdelay(10);
2117 		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2118 			i -= 10;
2119 	}
2120 
2121 	CMOS_WRITE(save_control, RTC_CONTROL);
2122 	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2123 	clear_IO_APIC_pin(apic, pin);
2124 
2125 	ioapic_write_entry(apic, pin, entry0);
2126 }
2127 
2128 static int disable_timer_pin_1 __initdata;
2129 /* Actually the next is obsolete, but keep it for paranoid reasons -AK */
disable_timer_pin_setup(char * arg)2130 static int __init disable_timer_pin_setup(char *arg)
2131 {
2132 	disable_timer_pin_1 = 1;
2133 	return 0;
2134 }
2135 early_param("disable_timer_pin_1", disable_timer_pin_setup);
2136 
mp_alloc_timer_irq(int ioapic,int pin)2137 static int mp_alloc_timer_irq(int ioapic, int pin)
2138 {
2139 	int irq = -1;
2140 	struct irq_domain *domain = mp_ioapic_irqdomain(ioapic);
2141 
2142 	if (domain) {
2143 		struct irq_alloc_info info;
2144 
2145 		ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 0, 0);
2146 		info.devid = mpc_ioapic_id(ioapic);
2147 		info.ioapic.pin = pin;
2148 		mutex_lock(&ioapic_mutex);
2149 		irq = alloc_isa_irq_from_domain(domain, 0, ioapic, pin, &info);
2150 		mutex_unlock(&ioapic_mutex);
2151 	}
2152 
2153 	return irq;
2154 }
2155 
2156 /*
2157  * This code may look a bit paranoid, but it's supposed to cooperate with
2158  * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
2159  * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
2160  * fanatically on his truly buggy board.
2161  *
2162  * FIXME: really need to revamp this for all platforms.
2163  */
check_timer(void)2164 static inline void __init check_timer(void)
2165 {
2166 	struct irq_data *irq_data = irq_get_irq_data(0);
2167 	struct mp_chip_data *data = irq_data->chip_data;
2168 	struct irq_cfg *cfg = irqd_cfg(irq_data);
2169 	int node = cpu_to_node(0);
2170 	int apic1, pin1, apic2, pin2;
2171 	int no_pin1 = 0;
2172 
2173 	if (!global_clock_event)
2174 		return;
2175 
2176 	local_irq_disable();
2177 
2178 	/*
2179 	 * get/set the timer IRQ vector:
2180 	 */
2181 	legacy_pic->mask(0);
2182 
2183 	/*
2184 	 * As IRQ0 is to be enabled in the 8259A, the virtual
2185 	 * wire has to be disabled in the local APIC.  Also
2186 	 * timer interrupts need to be acknowledged manually in
2187 	 * the 8259A for the i82489DX when using the NMI
2188 	 * watchdog as that APIC treats NMIs as level-triggered.
2189 	 * The AEOI mode will finish them in the 8259A
2190 	 * automatically.
2191 	 */
2192 	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2193 	legacy_pic->init(1);
2194 
2195 	pin1  = find_isa_irq_pin(0, mp_INT);
2196 	apic1 = find_isa_irq_apic(0, mp_INT);
2197 	pin2  = ioapic_i8259.pin;
2198 	apic2 = ioapic_i8259.apic;
2199 
2200 	apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2201 		    "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2202 		    cfg->vector, apic1, pin1, apic2, pin2);
2203 
2204 	/*
2205 	 * Some BIOS writers are clueless and report the ExtINTA
2206 	 * I/O APIC input from the cascaded 8259A as the timer
2207 	 * interrupt input.  So just in case, if only one pin
2208 	 * was found above, try it both directly and through the
2209 	 * 8259A.
2210 	 */
2211 	if (pin1 == -1) {
2212 		panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC");
2213 		pin1 = pin2;
2214 		apic1 = apic2;
2215 		no_pin1 = 1;
2216 	} else if (pin2 == -1) {
2217 		pin2 = pin1;
2218 		apic2 = apic1;
2219 	}
2220 
2221 	if (pin1 != -1) {
2222 		/* Ok, does IRQ0 through the IOAPIC work? */
2223 		if (no_pin1) {
2224 			mp_alloc_timer_irq(apic1, pin1);
2225 		} else {
2226 			/*
2227 			 * for edge trigger, it's already unmasked,
2228 			 * so only need to unmask if it is level-trigger
2229 			 * do we really have level trigger timer?
2230 			 */
2231 			int idx = find_irq_entry(apic1, pin1, mp_INT);
2232 
2233 			if (idx != -1 && irq_is_level(idx))
2234 				unmask_ioapic_irq(irq_get_irq_data(0));
2235 		}
2236 		irq_domain_deactivate_irq(irq_data);
2237 		irq_domain_activate_irq(irq_data, false);
2238 		if (timer_irq_works()) {
2239 			if (disable_timer_pin_1 > 0)
2240 				clear_IO_APIC_pin(0, pin1);
2241 			goto out;
2242 		}
2243 		panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
2244 		clear_IO_APIC_pin(apic1, pin1);
2245 		if (!no_pin1)
2246 			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2247 				    "8254 timer not connected to IO-APIC\n");
2248 
2249 		apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2250 			    "(IRQ0) through the 8259A ...\n");
2251 		apic_printk(APIC_QUIET, KERN_INFO
2252 			    "..... (found apic %d pin %d) ...\n", apic2, pin2);
2253 		/*
2254 		 * legacy devices should be connected to IO APIC #0
2255 		 */
2256 		replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2);
2257 		irq_domain_deactivate_irq(irq_data);
2258 		irq_domain_activate_irq(irq_data, false);
2259 		legacy_pic->unmask(0);
2260 		if (timer_irq_works()) {
2261 			apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2262 			goto out;
2263 		}
2264 		/*
2265 		 * Cleanup, just in case ...
2266 		 */
2267 		legacy_pic->mask(0);
2268 		clear_IO_APIC_pin(apic2, pin2);
2269 		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2270 	}
2271 
2272 	apic_printk(APIC_QUIET, KERN_INFO
2273 		    "...trying to set up timer as Virtual Wire IRQ...\n");
2274 
2275 	lapic_register_intr(0);
2276 	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
2277 	legacy_pic->unmask(0);
2278 
2279 	if (timer_irq_works()) {
2280 		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2281 		goto out;
2282 	}
2283 	legacy_pic->mask(0);
2284 	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2285 	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2286 
2287 	apic_printk(APIC_QUIET, KERN_INFO
2288 		    "...trying to set up timer as ExtINT IRQ...\n");
2289 
2290 	legacy_pic->init(0);
2291 	legacy_pic->make_irq(0);
2292 	apic_write(APIC_LVT0, APIC_DM_EXTINT);
2293 	legacy_pic->unmask(0);
2294 
2295 	unlock_ExtINT_logic();
2296 
2297 	if (timer_irq_works()) {
2298 		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2299 		goto out;
2300 	}
2301 	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2302 	if (apic_is_x2apic_enabled())
2303 		apic_printk(APIC_QUIET, KERN_INFO
2304 			    "Perhaps problem with the pre-enabled x2apic mode\n"
2305 			    "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2306 	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
2307 		"report.  Then try booting with the 'noapic' option.\n");
2308 out:
2309 	local_irq_enable();
2310 }
2311 
2312 /*
2313  * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2314  * to devices.  However there may be an I/O APIC pin available for
2315  * this interrupt regardless.  The pin may be left unconnected, but
2316  * typically it will be reused as an ExtINT cascade interrupt for
2317  * the master 8259A.  In the MPS case such a pin will normally be
2318  * reported as an ExtINT interrupt in the MP table.  With ACPI
2319  * there is no provision for ExtINT interrupts, and in the absence
2320  * of an override it would be treated as an ordinary ISA I/O APIC
2321  * interrupt, that is edge-triggered and unmasked by default.  We
2322  * used to do this, but it caused problems on some systems because
2323  * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2324  * the same ExtINT cascade interrupt to drive the local APIC of the
2325  * bootstrap processor.  Therefore we refrain from routing IRQ2 to
2326  * the I/O APIC in all cases now.  No actual device should request
2327  * it anyway.  --macro
2328  */
2329 #define PIC_IRQS	(1UL << PIC_CASCADE_IR)
2330 
mp_irqdomain_create(int ioapic)2331 static int mp_irqdomain_create(int ioapic)
2332 {
2333 	struct irq_domain *parent;
2334 	int hwirqs = mp_ioapic_pin_count(ioapic);
2335 	struct ioapic *ip = &ioapics[ioapic];
2336 	struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg;
2337 	struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2338 	struct fwnode_handle *fn;
2339 	struct irq_fwspec fwspec;
2340 
2341 	if (cfg->type == IOAPIC_DOMAIN_INVALID)
2342 		return 0;
2343 
2344 	/* Handle device tree enumerated APICs proper */
2345 	if (cfg->dev) {
2346 		fn = of_node_to_fwnode(cfg->dev);
2347 	} else {
2348 		fn = irq_domain_alloc_named_id_fwnode("IO-APIC", mpc_ioapic_id(ioapic));
2349 		if (!fn)
2350 			return -ENOMEM;
2351 	}
2352 
2353 	fwspec.fwnode = fn;
2354 	fwspec.param_count = 1;
2355 	fwspec.param[0] = mpc_ioapic_id(ioapic);
2356 
2357 	parent = irq_find_matching_fwspec(&fwspec, DOMAIN_BUS_ANY);
2358 	if (!parent) {
2359 		if (!cfg->dev)
2360 			irq_domain_free_fwnode(fn);
2361 		return -ENODEV;
2362 	}
2363 
2364 	ip->irqdomain = irq_domain_create_hierarchy(parent, 0, hwirqs, fn, cfg->ops,
2365 						    (void *)(long)ioapic);
2366 	if (!ip->irqdomain) {
2367 		/* Release fw handle if it was allocated above */
2368 		if (!cfg->dev)
2369 			irq_domain_free_fwnode(fn);
2370 		return -ENOMEM;
2371 	}
2372 
2373 	if (cfg->type == IOAPIC_DOMAIN_LEGACY ||
2374 	    cfg->type == IOAPIC_DOMAIN_STRICT)
2375 		ioapic_dynirq_base = max(ioapic_dynirq_base,
2376 					 gsi_cfg->gsi_end + 1);
2377 
2378 	return 0;
2379 }
2380 
ioapic_destroy_irqdomain(int idx)2381 static void ioapic_destroy_irqdomain(int idx)
2382 {
2383 	struct ioapic_domain_cfg *cfg = &ioapics[idx].irqdomain_cfg;
2384 	struct fwnode_handle *fn = ioapics[idx].irqdomain->fwnode;
2385 
2386 	if (ioapics[idx].irqdomain) {
2387 		irq_domain_remove(ioapics[idx].irqdomain);
2388 		if (!cfg->dev)
2389 			irq_domain_free_fwnode(fn);
2390 		ioapics[idx].irqdomain = NULL;
2391 	}
2392 }
2393 
setup_IO_APIC(void)2394 void __init setup_IO_APIC(void)
2395 {
2396 	int ioapic;
2397 
2398 	if (ioapic_is_disabled || !nr_ioapics)
2399 		return;
2400 
2401 	io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL;
2402 
2403 	apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2404 	for_each_ioapic(ioapic)
2405 		BUG_ON(mp_irqdomain_create(ioapic));
2406 
2407 	/*
2408          * Set up IO-APIC IRQ routing.
2409          */
2410 	x86_init.mpparse.setup_ioapic_ids();
2411 
2412 	sync_Arb_IDs();
2413 	setup_IO_APIC_irqs();
2414 	init_IO_APIC_traps();
2415 	if (nr_legacy_irqs())
2416 		check_timer();
2417 
2418 	ioapic_initialized = 1;
2419 }
2420 
resume_ioapic_id(int ioapic_idx)2421 static void resume_ioapic_id(int ioapic_idx)
2422 {
2423 	unsigned long flags;
2424 	union IO_APIC_reg_00 reg_00;
2425 
2426 	raw_spin_lock_irqsave(&ioapic_lock, flags);
2427 	reg_00.raw = io_apic_read(ioapic_idx, 0);
2428 	if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
2429 		reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2430 		io_apic_write(ioapic_idx, 0, reg_00.raw);
2431 	}
2432 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2433 }
2434 
ioapic_resume(void)2435 static void ioapic_resume(void)
2436 {
2437 	int ioapic_idx;
2438 
2439 	for_each_ioapic_reverse(ioapic_idx)
2440 		resume_ioapic_id(ioapic_idx);
2441 
2442 	restore_ioapic_entries();
2443 }
2444 
2445 static struct syscore_ops ioapic_syscore_ops = {
2446 	.suspend = save_ioapic_entries,
2447 	.resume = ioapic_resume,
2448 };
2449 
ioapic_init_ops(void)2450 static int __init ioapic_init_ops(void)
2451 {
2452 	register_syscore_ops(&ioapic_syscore_ops);
2453 
2454 	return 0;
2455 }
2456 
2457 device_initcall(ioapic_init_ops);
2458 
io_apic_get_redir_entries(int ioapic)2459 static int io_apic_get_redir_entries(int ioapic)
2460 {
2461 	union IO_APIC_reg_01	reg_01;
2462 	unsigned long flags;
2463 
2464 	raw_spin_lock_irqsave(&ioapic_lock, flags);
2465 	reg_01.raw = io_apic_read(ioapic, 1);
2466 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2467 
2468 	/* The register returns the maximum index redir index
2469 	 * supported, which is one less than the total number of redir
2470 	 * entries.
2471 	 */
2472 	return reg_01.bits.entries + 1;
2473 }
2474 
arch_dynirq_lower_bound(unsigned int from)2475 unsigned int arch_dynirq_lower_bound(unsigned int from)
2476 {
2477 	unsigned int ret;
2478 
2479 	/*
2480 	 * dmar_alloc_hwirq() may be called before setup_IO_APIC(), so use
2481 	 * gsi_top if ioapic_dynirq_base hasn't been initialized yet.
2482 	 */
2483 	ret = ioapic_dynirq_base ? : gsi_top;
2484 
2485 	/*
2486 	 * For DT enabled machines ioapic_dynirq_base is irrelevant and
2487 	 * always 0. gsi_top can be 0 if there is no IO/APIC registered.
2488 	 * 0 is an invalid interrupt number for dynamic allocations. Return
2489 	 * @from instead.
2490 	 */
2491 	return ret ? : from;
2492 }
2493 
2494 #ifdef CONFIG_X86_32
io_apic_get_unique_id(int ioapic,int apic_id)2495 static int io_apic_get_unique_id(int ioapic, int apic_id)
2496 {
2497 	union IO_APIC_reg_00 reg_00;
2498 	static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
2499 	physid_mask_t tmp;
2500 	unsigned long flags;
2501 	int i = 0;
2502 
2503 	/*
2504 	 * The P4 platform supports up to 256 APIC IDs on two separate APIC
2505 	 * buses (one for LAPICs, one for IOAPICs), where predecessors only
2506 	 * supports up to 16 on one shared APIC bus.
2507 	 *
2508 	 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
2509 	 *      advantage of new APIC bus architecture.
2510 	 */
2511 
2512 	if (physids_empty(apic_id_map))
2513 		apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
2514 
2515 	raw_spin_lock_irqsave(&ioapic_lock, flags);
2516 	reg_00.raw = io_apic_read(ioapic, 0);
2517 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2518 
2519 	if (apic_id >= get_physical_broadcast()) {
2520 		printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
2521 			"%d\n", ioapic, apic_id, reg_00.bits.ID);
2522 		apic_id = reg_00.bits.ID;
2523 	}
2524 
2525 	/*
2526 	 * Every APIC in a system must have a unique ID or we get lots of nice
2527 	 * 'stuck on smp_invalidate_needed IPI wait' messages.
2528 	 */
2529 	if (apic->check_apicid_used(&apic_id_map, apic_id)) {
2530 
2531 		for (i = 0; i < get_physical_broadcast(); i++) {
2532 			if (!apic->check_apicid_used(&apic_id_map, i))
2533 				break;
2534 		}
2535 
2536 		if (i == get_physical_broadcast())
2537 			panic("Max apic_id exceeded!\n");
2538 
2539 		printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
2540 			"trying %d\n", ioapic, apic_id, i);
2541 
2542 		apic_id = i;
2543 	}
2544 
2545 	physid_set_mask_of_physid(apic_id, &tmp);
2546 	physids_or(apic_id_map, apic_id_map, tmp);
2547 
2548 	if (reg_00.bits.ID != apic_id) {
2549 		reg_00.bits.ID = apic_id;
2550 
2551 		raw_spin_lock_irqsave(&ioapic_lock, flags);
2552 		io_apic_write(ioapic, 0, reg_00.raw);
2553 		reg_00.raw = io_apic_read(ioapic, 0);
2554 		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2555 
2556 		/* Sanity check */
2557 		if (reg_00.bits.ID != apic_id) {
2558 			pr_err("IOAPIC[%d]: Unable to change apic_id!\n",
2559 			       ioapic);
2560 			return -1;
2561 		}
2562 	}
2563 
2564 	apic_printk(APIC_VERBOSE, KERN_INFO
2565 			"IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
2566 
2567 	return apic_id;
2568 }
2569 
io_apic_unique_id(int idx,u8 id)2570 static u8 io_apic_unique_id(int idx, u8 id)
2571 {
2572 	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
2573 	    !APIC_XAPIC(boot_cpu_apic_version))
2574 		return io_apic_get_unique_id(idx, id);
2575 	else
2576 		return id;
2577 }
2578 #else
io_apic_unique_id(int idx,u8 id)2579 static u8 io_apic_unique_id(int idx, u8 id)
2580 {
2581 	union IO_APIC_reg_00 reg_00;
2582 	DECLARE_BITMAP(used, 256);
2583 	unsigned long flags;
2584 	u8 new_id;
2585 	int i;
2586 
2587 	bitmap_zero(used, 256);
2588 	for_each_ioapic(i)
2589 		__set_bit(mpc_ioapic_id(i), used);
2590 
2591 	/* Hand out the requested id if available */
2592 	if (!test_bit(id, used))
2593 		return id;
2594 
2595 	/*
2596 	 * Read the current id from the ioapic and keep it if
2597 	 * available.
2598 	 */
2599 	raw_spin_lock_irqsave(&ioapic_lock, flags);
2600 	reg_00.raw = io_apic_read(idx, 0);
2601 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2602 	new_id = reg_00.bits.ID;
2603 	if (!test_bit(new_id, used)) {
2604 		apic_printk(APIC_VERBOSE, KERN_INFO
2605 			"IOAPIC[%d]: Using reg apic_id %d instead of %d\n",
2606 			 idx, new_id, id);
2607 		return new_id;
2608 	}
2609 
2610 	/*
2611 	 * Get the next free id and write it to the ioapic.
2612 	 */
2613 	new_id = find_first_zero_bit(used, 256);
2614 	reg_00.bits.ID = new_id;
2615 	raw_spin_lock_irqsave(&ioapic_lock, flags);
2616 	io_apic_write(idx, 0, reg_00.raw);
2617 	reg_00.raw = io_apic_read(idx, 0);
2618 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2619 	/* Sanity check */
2620 	BUG_ON(reg_00.bits.ID != new_id);
2621 
2622 	return new_id;
2623 }
2624 #endif
2625 
io_apic_get_version(int ioapic)2626 static int io_apic_get_version(int ioapic)
2627 {
2628 	union IO_APIC_reg_01	reg_01;
2629 	unsigned long flags;
2630 
2631 	raw_spin_lock_irqsave(&ioapic_lock, flags);
2632 	reg_01.raw = io_apic_read(ioapic, 1);
2633 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2634 
2635 	return reg_01.bits.version;
2636 }
2637 
2638 /*
2639  * This function updates target affinity of IOAPIC interrupts to include
2640  * the CPUs which came online during SMP bringup.
2641  */
2642 #define IOAPIC_RESOURCE_NAME_SIZE 11
2643 
2644 static struct resource *ioapic_resources;
2645 
ioapic_setup_resources(void)2646 static struct resource * __init ioapic_setup_resources(void)
2647 {
2648 	unsigned long n;
2649 	struct resource *res;
2650 	char *mem;
2651 	int i;
2652 
2653 	if (nr_ioapics == 0)
2654 		return NULL;
2655 
2656 	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
2657 	n *= nr_ioapics;
2658 
2659 	mem = memblock_alloc(n, SMP_CACHE_BYTES);
2660 	if (!mem)
2661 		panic("%s: Failed to allocate %lu bytes\n", __func__, n);
2662 	res = (void *)mem;
2663 
2664 	mem += sizeof(struct resource) * nr_ioapics;
2665 
2666 	for_each_ioapic(i) {
2667 		res[i].name = mem;
2668 		res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
2669 		snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
2670 		mem += IOAPIC_RESOURCE_NAME_SIZE;
2671 		ioapics[i].iomem_res = &res[i];
2672 	}
2673 
2674 	ioapic_resources = res;
2675 
2676 	return res;
2677 }
2678 
io_apic_set_fixmap(enum fixed_addresses idx,phys_addr_t phys)2679 static void io_apic_set_fixmap(enum fixed_addresses idx, phys_addr_t phys)
2680 {
2681 	pgprot_t flags = FIXMAP_PAGE_NOCACHE;
2682 
2683 	/*
2684 	 * Ensure fixmaps for IO-APIC MMIO respect memory encryption pgprot
2685 	 * bits, just like normal ioremap():
2686 	 */
2687 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
2688 		if (x86_platform.hyper.is_private_mmio(phys))
2689 			flags = pgprot_encrypted(flags);
2690 		else
2691 			flags = pgprot_decrypted(flags);
2692 	}
2693 
2694 	__set_fixmap(idx, phys, flags);
2695 }
2696 
io_apic_init_mappings(void)2697 void __init io_apic_init_mappings(void)
2698 {
2699 	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
2700 	struct resource *ioapic_res;
2701 	int i;
2702 
2703 	ioapic_res = ioapic_setup_resources();
2704 	for_each_ioapic(i) {
2705 		if (smp_found_config) {
2706 			ioapic_phys = mpc_ioapic_addr(i);
2707 #ifdef CONFIG_X86_32
2708 			if (!ioapic_phys) {
2709 				printk(KERN_ERR
2710 				       "WARNING: bogus zero IO-APIC "
2711 				       "address found in MPTABLE, "
2712 				       "disabling IO/APIC support!\n");
2713 				smp_found_config = 0;
2714 				ioapic_is_disabled = true;
2715 				goto fake_ioapic_page;
2716 			}
2717 #endif
2718 		} else {
2719 #ifdef CONFIG_X86_32
2720 fake_ioapic_page:
2721 #endif
2722 			ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE,
2723 								    PAGE_SIZE);
2724 			if (!ioapic_phys)
2725 				panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
2726 				      __func__, PAGE_SIZE, PAGE_SIZE);
2727 			ioapic_phys = __pa(ioapic_phys);
2728 		}
2729 		io_apic_set_fixmap(idx, ioapic_phys);
2730 		apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
2731 			__fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
2732 			ioapic_phys);
2733 		idx++;
2734 
2735 		ioapic_res->start = ioapic_phys;
2736 		ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
2737 		ioapic_res++;
2738 	}
2739 }
2740 
ioapic_insert_resources(void)2741 void __init ioapic_insert_resources(void)
2742 {
2743 	int i;
2744 	struct resource *r = ioapic_resources;
2745 
2746 	if (!r) {
2747 		if (nr_ioapics > 0)
2748 			printk(KERN_ERR
2749 				"IO APIC resources couldn't be allocated.\n");
2750 		return;
2751 	}
2752 
2753 	for_each_ioapic(i) {
2754 		insert_resource(&iomem_resource, r);
2755 		r++;
2756 	}
2757 }
2758 
mp_find_ioapic(u32 gsi)2759 int mp_find_ioapic(u32 gsi)
2760 {
2761 	int i;
2762 
2763 	if (nr_ioapics == 0)
2764 		return -1;
2765 
2766 	/* Find the IOAPIC that manages this GSI. */
2767 	for_each_ioapic(i) {
2768 		struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
2769 		if (gsi >= gsi_cfg->gsi_base && gsi <= gsi_cfg->gsi_end)
2770 			return i;
2771 	}
2772 
2773 	printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
2774 	return -1;
2775 }
2776 
mp_find_ioapic_pin(int ioapic,u32 gsi)2777 int mp_find_ioapic_pin(int ioapic, u32 gsi)
2778 {
2779 	struct mp_ioapic_gsi *gsi_cfg;
2780 
2781 	if (WARN_ON(ioapic < 0))
2782 		return -1;
2783 
2784 	gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2785 	if (WARN_ON(gsi > gsi_cfg->gsi_end))
2786 		return -1;
2787 
2788 	return gsi - gsi_cfg->gsi_base;
2789 }
2790 
bad_ioapic_register(int idx)2791 static int bad_ioapic_register(int idx)
2792 {
2793 	union IO_APIC_reg_00 reg_00;
2794 	union IO_APIC_reg_01 reg_01;
2795 	union IO_APIC_reg_02 reg_02;
2796 
2797 	reg_00.raw = io_apic_read(idx, 0);
2798 	reg_01.raw = io_apic_read(idx, 1);
2799 	reg_02.raw = io_apic_read(idx, 2);
2800 
2801 	if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
2802 		pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
2803 			mpc_ioapic_addr(idx));
2804 		return 1;
2805 	}
2806 
2807 	return 0;
2808 }
2809 
find_free_ioapic_entry(void)2810 static int find_free_ioapic_entry(void)
2811 {
2812 	int idx;
2813 
2814 	for (idx = 0; idx < MAX_IO_APICS; idx++)
2815 		if (ioapics[idx].nr_registers == 0)
2816 			return idx;
2817 
2818 	return MAX_IO_APICS;
2819 }
2820 
2821 /**
2822  * mp_register_ioapic - Register an IOAPIC device
2823  * @id:		hardware IOAPIC ID
2824  * @address:	physical address of IOAPIC register area
2825  * @gsi_base:	base of GSI associated with the IOAPIC
2826  * @cfg:	configuration information for the IOAPIC
2827  */
mp_register_ioapic(int id,u32 address,u32 gsi_base,struct ioapic_domain_cfg * cfg)2828 int mp_register_ioapic(int id, u32 address, u32 gsi_base,
2829 		       struct ioapic_domain_cfg *cfg)
2830 {
2831 	bool hotplug = !!ioapic_initialized;
2832 	struct mp_ioapic_gsi *gsi_cfg;
2833 	int idx, ioapic, entries;
2834 	u32 gsi_end;
2835 
2836 	if (!address) {
2837 		pr_warn("Bogus (zero) I/O APIC address found, skipping!\n");
2838 		return -EINVAL;
2839 	}
2840 	for_each_ioapic(ioapic)
2841 		if (ioapics[ioapic].mp_config.apicaddr == address) {
2842 			pr_warn("address 0x%x conflicts with IOAPIC%d\n",
2843 				address, ioapic);
2844 			return -EEXIST;
2845 		}
2846 
2847 	idx = find_free_ioapic_entry();
2848 	if (idx >= MAX_IO_APICS) {
2849 		pr_warn("Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
2850 			MAX_IO_APICS, idx);
2851 		return -ENOSPC;
2852 	}
2853 
2854 	ioapics[idx].mp_config.type = MP_IOAPIC;
2855 	ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
2856 	ioapics[idx].mp_config.apicaddr = address;
2857 
2858 	io_apic_set_fixmap(FIX_IO_APIC_BASE_0 + idx, address);
2859 	if (bad_ioapic_register(idx)) {
2860 		clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2861 		return -ENODEV;
2862 	}
2863 
2864 	ioapics[idx].mp_config.apicid = io_apic_unique_id(idx, id);
2865 	ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
2866 
2867 	/*
2868 	 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
2869 	 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
2870 	 */
2871 	entries = io_apic_get_redir_entries(idx);
2872 	gsi_end = gsi_base + entries - 1;
2873 	for_each_ioapic(ioapic) {
2874 		gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2875 		if ((gsi_base >= gsi_cfg->gsi_base &&
2876 		     gsi_base <= gsi_cfg->gsi_end) ||
2877 		    (gsi_end >= gsi_cfg->gsi_base &&
2878 		     gsi_end <= gsi_cfg->gsi_end)) {
2879 			pr_warn("GSI range [%u-%u] for new IOAPIC conflicts with GSI[%u-%u]\n",
2880 				gsi_base, gsi_end,
2881 				gsi_cfg->gsi_base, gsi_cfg->gsi_end);
2882 			clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2883 			return -ENOSPC;
2884 		}
2885 	}
2886 	gsi_cfg = mp_ioapic_gsi_routing(idx);
2887 	gsi_cfg->gsi_base = gsi_base;
2888 	gsi_cfg->gsi_end = gsi_end;
2889 
2890 	ioapics[idx].irqdomain = NULL;
2891 	ioapics[idx].irqdomain_cfg = *cfg;
2892 
2893 	/*
2894 	 * If mp_register_ioapic() is called during early boot stage when
2895 	 * walking ACPI/DT tables, it's too early to create irqdomain,
2896 	 * we are still using bootmem allocator. So delay it to setup_IO_APIC().
2897 	 */
2898 	if (hotplug) {
2899 		if (mp_irqdomain_create(idx)) {
2900 			clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2901 			return -ENOMEM;
2902 		}
2903 		alloc_ioapic_saved_registers(idx);
2904 	}
2905 
2906 	if (gsi_cfg->gsi_end >= gsi_top)
2907 		gsi_top = gsi_cfg->gsi_end + 1;
2908 	if (nr_ioapics <= idx)
2909 		nr_ioapics = idx + 1;
2910 
2911 	/* Set nr_registers to mark entry present */
2912 	ioapics[idx].nr_registers = entries;
2913 
2914 	pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
2915 		idx, mpc_ioapic_id(idx),
2916 		mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
2917 		gsi_cfg->gsi_base, gsi_cfg->gsi_end);
2918 
2919 	return 0;
2920 }
2921 
mp_unregister_ioapic(u32 gsi_base)2922 int mp_unregister_ioapic(u32 gsi_base)
2923 {
2924 	int ioapic, pin;
2925 	int found = 0;
2926 
2927 	for_each_ioapic(ioapic)
2928 		if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) {
2929 			found = 1;
2930 			break;
2931 		}
2932 	if (!found) {
2933 		pr_warn("can't find IOAPIC for GSI %d\n", gsi_base);
2934 		return -ENODEV;
2935 	}
2936 
2937 	for_each_pin(ioapic, pin) {
2938 		u32 gsi = mp_pin_to_gsi(ioapic, pin);
2939 		int irq = mp_map_gsi_to_irq(gsi, 0, NULL);
2940 		struct mp_chip_data *data;
2941 
2942 		if (irq >= 0) {
2943 			data = irq_get_chip_data(irq);
2944 			if (data && data->count) {
2945 				pr_warn("pin%d on IOAPIC%d is still in use.\n",
2946 					pin, ioapic);
2947 				return -EBUSY;
2948 			}
2949 		}
2950 	}
2951 
2952 	/* Mark entry not present */
2953 	ioapics[ioapic].nr_registers  = 0;
2954 	ioapic_destroy_irqdomain(ioapic);
2955 	free_ioapic_saved_registers(ioapic);
2956 	if (ioapics[ioapic].iomem_res)
2957 		release_resource(ioapics[ioapic].iomem_res);
2958 	clear_fixmap(FIX_IO_APIC_BASE_0 + ioapic);
2959 	memset(&ioapics[ioapic], 0, sizeof(ioapics[ioapic]));
2960 
2961 	return 0;
2962 }
2963 
mp_ioapic_registered(u32 gsi_base)2964 int mp_ioapic_registered(u32 gsi_base)
2965 {
2966 	int ioapic;
2967 
2968 	for_each_ioapic(ioapic)
2969 		if (ioapics[ioapic].gsi_config.gsi_base == gsi_base)
2970 			return 1;
2971 
2972 	return 0;
2973 }
2974 
mp_irqdomain_get_attr(u32 gsi,struct mp_chip_data * data,struct irq_alloc_info * info)2975 static void mp_irqdomain_get_attr(u32 gsi, struct mp_chip_data *data,
2976 				  struct irq_alloc_info *info)
2977 {
2978 	if (info && info->ioapic.valid) {
2979 		data->is_level = info->ioapic.is_level;
2980 		data->active_low = info->ioapic.active_low;
2981 	} else if (__acpi_get_override_irq(gsi, &data->is_level,
2982 					   &data->active_low) < 0) {
2983 		/* PCI interrupts are always active low level triggered. */
2984 		data->is_level = true;
2985 		data->active_low = true;
2986 	}
2987 }
2988 
2989 /*
2990  * Configure the I/O-APIC specific fields in the routing entry.
2991  *
2992  * This is important to setup the I/O-APIC specific bits (is_level,
2993  * active_low, masked) because the underlying parent domain will only
2994  * provide the routing information and is oblivious of the I/O-APIC
2995  * specific bits.
2996  *
2997  * The entry is just preconfigured at this point and not written into the
2998  * RTE. This happens later during activation which will fill in the actual
2999  * routing information.
3000  */
mp_preconfigure_entry(struct mp_chip_data * data)3001 static void mp_preconfigure_entry(struct mp_chip_data *data)
3002 {
3003 	struct IO_APIC_route_entry *entry = &data->entry;
3004 
3005 	memset(entry, 0, sizeof(*entry));
3006 	entry->is_level		 = data->is_level;
3007 	entry->active_low	 = data->active_low;
3008 	/*
3009 	 * Mask level triggered irqs. Edge triggered irqs are masked
3010 	 * by the irq core code in case they fire.
3011 	 */
3012 	entry->masked		= data->is_level;
3013 }
3014 
mp_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)3015 int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
3016 		       unsigned int nr_irqs, void *arg)
3017 {
3018 	struct irq_alloc_info *info = arg;
3019 	struct mp_chip_data *data;
3020 	struct irq_data *irq_data;
3021 	int ret, ioapic, pin;
3022 	unsigned long flags;
3023 
3024 	if (!info || nr_irqs > 1)
3025 		return -EINVAL;
3026 	irq_data = irq_domain_get_irq_data(domain, virq);
3027 	if (!irq_data)
3028 		return -EINVAL;
3029 
3030 	ioapic = mp_irqdomain_ioapic_idx(domain);
3031 	pin = info->ioapic.pin;
3032 	if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0)
3033 		return -EEXIST;
3034 
3035 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3036 	if (!data)
3037 		return -ENOMEM;
3038 
3039 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, info);
3040 	if (ret < 0) {
3041 		kfree(data);
3042 		return ret;
3043 	}
3044 
3045 	INIT_LIST_HEAD(&data->irq_2_pin);
3046 	irq_data->hwirq = info->ioapic.pin;
3047 	irq_data->chip = (domain->parent == x86_vector_domain) ?
3048 			  &ioapic_chip : &ioapic_ir_chip;
3049 	irq_data->chip_data = data;
3050 	mp_irqdomain_get_attr(mp_pin_to_gsi(ioapic, pin), data, info);
3051 
3052 	add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin);
3053 
3054 	mp_preconfigure_entry(data);
3055 	mp_register_handler(virq, data->is_level);
3056 
3057 	local_irq_save(flags);
3058 	if (virq < nr_legacy_irqs())
3059 		legacy_pic->mask(virq);
3060 	local_irq_restore(flags);
3061 
3062 	apic_printk(APIC_VERBOSE, KERN_DEBUG
3063 		    "IOAPIC[%d]: Preconfigured routing entry (%d-%d -> IRQ %d Level:%i ActiveLow:%i)\n",
3064 		    ioapic, mpc_ioapic_id(ioapic), pin, virq,
3065 		    data->is_level, data->active_low);
3066 	return 0;
3067 }
3068 
mp_irqdomain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)3069 void mp_irqdomain_free(struct irq_domain *domain, unsigned int virq,
3070 		       unsigned int nr_irqs)
3071 {
3072 	struct irq_data *irq_data;
3073 	struct mp_chip_data *data;
3074 
3075 	BUG_ON(nr_irqs != 1);
3076 	irq_data = irq_domain_get_irq_data(domain, virq);
3077 	if (irq_data && irq_data->chip_data) {
3078 		data = irq_data->chip_data;
3079 		__remove_pin_from_irq(data, mp_irqdomain_ioapic_idx(domain),
3080 				      (int)irq_data->hwirq);
3081 		WARN_ON(!list_empty(&data->irq_2_pin));
3082 		kfree(irq_data->chip_data);
3083 	}
3084 	irq_domain_free_irqs_top(domain, virq, nr_irqs);
3085 }
3086 
mp_irqdomain_activate(struct irq_domain * domain,struct irq_data * irq_data,bool reserve)3087 int mp_irqdomain_activate(struct irq_domain *domain,
3088 			  struct irq_data *irq_data, bool reserve)
3089 {
3090 	unsigned long flags;
3091 
3092 	raw_spin_lock_irqsave(&ioapic_lock, flags);
3093 	ioapic_configure_entry(irq_data);
3094 	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3095 	return 0;
3096 }
3097 
mp_irqdomain_deactivate(struct irq_domain * domain,struct irq_data * irq_data)3098 void mp_irqdomain_deactivate(struct irq_domain *domain,
3099 			     struct irq_data *irq_data)
3100 {
3101 	/* It won't be called for IRQ with multiple IOAPIC pins associated */
3102 	ioapic_mask_entry(mp_irqdomain_ioapic_idx(domain),
3103 			  (int)irq_data->hwirq);
3104 }
3105 
mp_irqdomain_ioapic_idx(struct irq_domain * domain)3106 int mp_irqdomain_ioapic_idx(struct irq_domain *domain)
3107 {
3108 	return (int)(long)domain->host_data;
3109 }
3110 
3111 const struct irq_domain_ops mp_ioapic_irqdomain_ops = {
3112 	.alloc		= mp_irqdomain_alloc,
3113 	.free		= mp_irqdomain_free,
3114 	.activate	= mp_irqdomain_activate,
3115 	.deactivate	= mp_irqdomain_deactivate,
3116 };
3117