1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *    Support for adapter interruptions
4  *
5  *    Copyright IBM Corp. 1999, 2007
6  *    Author(s): Ingo Adlung <adlung@de.ibm.com>
7  *		 Cornelia Huck <cornelia.huck@de.ibm.com>
8  *		 Arnd Bergmann <arndb@de.ibm.com>
9  *		 Peter Oberparleiter <peter.oberparleiter@de.ibm.com>
10  */
11 
12 #include <linux/init.h>
13 #include <linux/irq.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/rculist.h>
18 #include <linux/slab.h>
19 #include <linux/dmapool.h>
20 
21 #include <asm/airq.h>
22 #include <asm/isc.h>
23 #include <asm/cio.h>
24 
25 #include "cio.h"
26 #include "cio_debug.h"
27 #include "ioasm.h"
28 
29 static DEFINE_SPINLOCK(airq_lists_lock);
30 static struct hlist_head airq_lists[MAX_ISC+1];
31 
32 static struct dma_pool *airq_iv_cache;
33 
34 /**
35  * register_adapter_interrupt() - register adapter interrupt handler
36  * @airq: pointer to adapter interrupt descriptor
37  *
38  * Returns 0 on success, or -EINVAL.
39  */
register_adapter_interrupt(struct airq_struct * airq)40 int register_adapter_interrupt(struct airq_struct *airq)
41 {
42 	char dbf_txt[32];
43 
44 	if (!airq->handler || airq->isc > MAX_ISC)
45 		return -EINVAL;
46 	if (!airq->lsi_ptr) {
47 		airq->lsi_ptr = cio_dma_zalloc(1);
48 		if (!airq->lsi_ptr)
49 			return -ENOMEM;
50 		airq->flags |= AIRQ_PTR_ALLOCATED;
51 	}
52 	snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq);
53 	CIO_TRACE_EVENT(4, dbf_txt);
54 	isc_register(airq->isc);
55 	spin_lock(&airq_lists_lock);
56 	hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]);
57 	spin_unlock(&airq_lists_lock);
58 	return 0;
59 }
60 EXPORT_SYMBOL(register_adapter_interrupt);
61 
62 /**
63  * unregister_adapter_interrupt - unregister adapter interrupt handler
64  * @airq: pointer to adapter interrupt descriptor
65  */
unregister_adapter_interrupt(struct airq_struct * airq)66 void unregister_adapter_interrupt(struct airq_struct *airq)
67 {
68 	char dbf_txt[32];
69 
70 	if (hlist_unhashed(&airq->list))
71 		return;
72 	snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq);
73 	CIO_TRACE_EVENT(4, dbf_txt);
74 	spin_lock(&airq_lists_lock);
75 	hlist_del_rcu(&airq->list);
76 	spin_unlock(&airq_lists_lock);
77 	synchronize_rcu();
78 	isc_unregister(airq->isc);
79 	if (airq->flags & AIRQ_PTR_ALLOCATED) {
80 		cio_dma_free(airq->lsi_ptr, 1);
81 		airq->lsi_ptr = NULL;
82 		airq->flags &= ~AIRQ_PTR_ALLOCATED;
83 	}
84 }
85 EXPORT_SYMBOL(unregister_adapter_interrupt);
86 
do_airq_interrupt(int irq,void * dummy)87 static irqreturn_t do_airq_interrupt(int irq, void *dummy)
88 {
89 	struct tpi_info *tpi_info;
90 	struct airq_struct *airq;
91 	struct hlist_head *head;
92 
93 	set_cpu_flag(CIF_NOHZ_DELAY);
94 	tpi_info = &get_irq_regs()->tpi_info;
95 	trace_s390_cio_adapter_int(tpi_info);
96 	head = &airq_lists[tpi_info->isc];
97 	rcu_read_lock();
98 	hlist_for_each_entry_rcu(airq, head, list)
99 		if (*airq->lsi_ptr != 0)
100 			airq->handler(airq, tpi_info);
101 	rcu_read_unlock();
102 
103 	return IRQ_HANDLED;
104 }
105 
init_airq_interrupts(void)106 void __init init_airq_interrupts(void)
107 {
108 	irq_set_chip_and_handler(THIN_INTERRUPT,
109 				 &dummy_irq_chip, handle_percpu_irq);
110 	if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL))
111 		panic("Failed to register AIO interrupt\n");
112 }
113 
iv_size(unsigned long bits)114 static inline unsigned long iv_size(unsigned long bits)
115 {
116 	return BITS_TO_LONGS(bits) * sizeof(unsigned long);
117 }
118 
119 /**
120  * airq_iv_create - create an interrupt vector
121  * @bits: number of bits in the interrupt vector
122  * @flags: allocation flags
123  * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC
124  *
125  * Returns a pointer to an interrupt vector structure
126  */
airq_iv_create(unsigned long bits,unsigned long flags,unsigned long * vec)127 struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags,
128 			       unsigned long *vec)
129 {
130 	struct airq_iv *iv;
131 	unsigned long size;
132 
133 	iv = kzalloc(sizeof(*iv), GFP_KERNEL);
134 	if (!iv)
135 		goto out;
136 	iv->bits = bits;
137 	iv->flags = flags;
138 	size = iv_size(bits);
139 
140 	if (flags & AIRQ_IV_CACHELINE) {
141 		if ((cache_line_size() * BITS_PER_BYTE) < bits
142 				|| !airq_iv_cache)
143 			goto out_free;
144 
145 		iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL,
146 					     &iv->vector_dma);
147 		if (!iv->vector)
148 			goto out_free;
149 	} else if (flags & AIRQ_IV_GUESTVEC) {
150 		iv->vector = vec;
151 	} else {
152 		iv->vector = cio_dma_zalloc(size);
153 		if (!iv->vector)
154 			goto out_free;
155 	}
156 	if (flags & AIRQ_IV_ALLOC) {
157 		iv->avail = kmalloc(size, GFP_KERNEL);
158 		if (!iv->avail)
159 			goto out_free;
160 		memset(iv->avail, 0xff, size);
161 		iv->end = 0;
162 	} else
163 		iv->end = bits;
164 	if (flags & AIRQ_IV_BITLOCK) {
165 		iv->bitlock = kzalloc(size, GFP_KERNEL);
166 		if (!iv->bitlock)
167 			goto out_free;
168 	}
169 	if (flags & AIRQ_IV_PTR) {
170 		size = bits * sizeof(unsigned long);
171 		iv->ptr = kzalloc(size, GFP_KERNEL);
172 		if (!iv->ptr)
173 			goto out_free;
174 	}
175 	if (flags & AIRQ_IV_DATA) {
176 		size = bits * sizeof(unsigned int);
177 		iv->data = kzalloc(size, GFP_KERNEL);
178 		if (!iv->data)
179 			goto out_free;
180 	}
181 	spin_lock_init(&iv->lock);
182 	return iv;
183 
184 out_free:
185 	kfree(iv->ptr);
186 	kfree(iv->bitlock);
187 	kfree(iv->avail);
188 	if (iv->flags & AIRQ_IV_CACHELINE && iv->vector)
189 		dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
190 	else if (!(iv->flags & AIRQ_IV_GUESTVEC))
191 		cio_dma_free(iv->vector, size);
192 	kfree(iv);
193 out:
194 	return NULL;
195 }
196 EXPORT_SYMBOL(airq_iv_create);
197 
198 /**
199  * airq_iv_release - release an interrupt vector
200  * @iv: pointer to interrupt vector structure
201  */
airq_iv_release(struct airq_iv * iv)202 void airq_iv_release(struct airq_iv *iv)
203 {
204 	kfree(iv->data);
205 	kfree(iv->ptr);
206 	kfree(iv->bitlock);
207 	if (iv->flags & AIRQ_IV_CACHELINE)
208 		dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
209 	else if (!(iv->flags & AIRQ_IV_GUESTVEC))
210 		cio_dma_free(iv->vector, iv_size(iv->bits));
211 	kfree(iv->avail);
212 	kfree(iv);
213 }
214 EXPORT_SYMBOL(airq_iv_release);
215 
216 /**
217  * airq_iv_alloc - allocate irq bits from an interrupt vector
218  * @iv: pointer to an interrupt vector structure
219  * @num: number of consecutive irq bits to allocate
220  *
221  * Returns the bit number of the first irq in the allocated block of irqs,
222  * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been
223  * specified
224  */
airq_iv_alloc(struct airq_iv * iv,unsigned long num)225 unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num)
226 {
227 	unsigned long bit, i, flags;
228 
229 	if (!iv->avail || num == 0)
230 		return -1UL;
231 	spin_lock_irqsave(&iv->lock, flags);
232 	bit = find_first_bit_inv(iv->avail, iv->bits);
233 	while (bit + num <= iv->bits) {
234 		for (i = 1; i < num; i++)
235 			if (!test_bit_inv(bit + i, iv->avail))
236 				break;
237 		if (i >= num) {
238 			/* Found a suitable block of irqs */
239 			for (i = 0; i < num; i++)
240 				clear_bit_inv(bit + i, iv->avail);
241 			if (bit + num >= iv->end)
242 				iv->end = bit + num + 1;
243 			break;
244 		}
245 		bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1);
246 	}
247 	if (bit + num > iv->bits)
248 		bit = -1UL;
249 	spin_unlock_irqrestore(&iv->lock, flags);
250 	return bit;
251 }
252 EXPORT_SYMBOL(airq_iv_alloc);
253 
254 /**
255  * airq_iv_free - free irq bits of an interrupt vector
256  * @iv: pointer to interrupt vector structure
257  * @bit: number of the first irq bit to free
258  * @num: number of consecutive irq bits to free
259  */
airq_iv_free(struct airq_iv * iv,unsigned long bit,unsigned long num)260 void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num)
261 {
262 	unsigned long i, flags;
263 
264 	if (!iv->avail || num == 0)
265 		return;
266 	spin_lock_irqsave(&iv->lock, flags);
267 	for (i = 0; i < num; i++) {
268 		/* Clear (possibly left over) interrupt bit */
269 		clear_bit_inv(bit + i, iv->vector);
270 		/* Make the bit positions available again */
271 		set_bit_inv(bit + i, iv->avail);
272 	}
273 	if (bit + num >= iv->end) {
274 		/* Find new end of bit-field */
275 		while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail))
276 			iv->end--;
277 	}
278 	spin_unlock_irqrestore(&iv->lock, flags);
279 }
280 EXPORT_SYMBOL(airq_iv_free);
281 
282 /**
283  * airq_iv_scan - scan interrupt vector for non-zero bits
284  * @iv: pointer to interrupt vector structure
285  * @start: bit number to start the search
286  * @end: bit number to end the search
287  *
288  * Returns the bit number of the next non-zero interrupt bit, or
289  * -1UL if the scan completed without finding any more any non-zero bits.
290  */
airq_iv_scan(struct airq_iv * iv,unsigned long start,unsigned long end)291 unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
292 			   unsigned long end)
293 {
294 	unsigned long bit;
295 
296 	/* Find non-zero bit starting from 'ivs->next'. */
297 	bit = find_next_bit_inv(iv->vector, end, start);
298 	if (bit >= end)
299 		return -1UL;
300 	clear_bit_inv(bit, iv->vector);
301 	return bit;
302 }
303 EXPORT_SYMBOL(airq_iv_scan);
304 
airq_init(void)305 int __init airq_init(void)
306 {
307 	airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(),
308 					cache_line_size(),
309 					cache_line_size(), PAGE_SIZE);
310 	if (!airq_iv_cache)
311 		return -ENOMEM;
312 	return 0;
313 }
314