1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2017 - Cambridge Greys Ltd
4  * Copyright (C) 2011 - 2014 Cisco Systems Inc
5  * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
6  * Derived (i.e. mostly copied) from arch/i386/kernel/irq.c:
7  *	Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
8  */
9 
10 #include <linux/cpumask.h>
11 #include <linux/hardirq.h>
12 #include <linux/interrupt.h>
13 #include <linux/kernel_stat.h>
14 #include <linux/module.h>
15 #include <linux/sched.h>
16 #include <linux/seq_file.h>
17 #include <linux/slab.h>
18 #include <as-layout.h>
19 #include <kern_util.h>
20 #include <os.h>
21 #include <irq_user.h>
22 #include <irq_kern.h>
23 #include <linux/time-internal.h>
24 
25 
26 /* When epoll triggers we do not know why it did so
27  * we can also have different IRQs for read and write.
28  * This is why we keep a small irq_reg array for each fd -
29  * one entry per IRQ type
30  */
31 struct irq_reg {
32 	void *id;
33 	int irq;
34 	/* it's cheaper to store this than to query it */
35 	int events;
36 	bool active;
37 	bool pending;
38 	bool wakeup;
39 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
40 	bool pending_on_resume;
41 	void (*timetravel_handler)(int, int, void *,
42 				   struct time_travel_event *);
43 	struct time_travel_event event;
44 #endif
45 };
46 
47 struct irq_entry {
48 	struct list_head list;
49 	int fd;
50 	struct irq_reg reg[NUM_IRQ_TYPES];
51 	bool suspended;
52 	bool sigio_workaround;
53 };
54 
55 static DEFINE_SPINLOCK(irq_lock);
56 static LIST_HEAD(active_fds);
57 static DECLARE_BITMAP(irqs_allocated, UM_LAST_SIGNAL_IRQ);
58 static bool irqs_suspended;
59 
irq_io_loop(struct irq_reg * irq,struct uml_pt_regs * regs)60 static void irq_io_loop(struct irq_reg *irq, struct uml_pt_regs *regs)
61 {
62 /*
63  * irq->active guards against reentry
64  * irq->pending accumulates pending requests
65  * if pending is raised the irq_handler is re-run
66  * until pending is cleared
67  */
68 	if (irq->active) {
69 		irq->active = false;
70 
71 		do {
72 			irq->pending = false;
73 			do_IRQ(irq->irq, regs);
74 		} while (irq->pending);
75 
76 		irq->active = true;
77 	} else {
78 		irq->pending = true;
79 	}
80 }
81 
82 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
irq_event_handler(struct time_travel_event * ev)83 static void irq_event_handler(struct time_travel_event *ev)
84 {
85 	struct irq_reg *reg = container_of(ev, struct irq_reg, event);
86 
87 	/* do nothing if suspended - just to cause a wakeup */
88 	if (irqs_suspended)
89 		return;
90 
91 	generic_handle_irq(reg->irq);
92 }
93 
irq_do_timetravel_handler(struct irq_entry * entry,enum um_irq_type t)94 static bool irq_do_timetravel_handler(struct irq_entry *entry,
95 				      enum um_irq_type t)
96 {
97 	struct irq_reg *reg = &entry->reg[t];
98 
99 	if (!reg->timetravel_handler)
100 		return false;
101 
102 	/*
103 	 * Handle all messages - we might get multiple even while
104 	 * interrupts are already suspended, due to suspend order
105 	 * etc. Note that time_travel_add_irq_event() will not add
106 	 * an event twice, if it's pending already "first wins".
107 	 */
108 	reg->timetravel_handler(reg->irq, entry->fd, reg->id, &reg->event);
109 
110 	if (!reg->event.pending)
111 		return false;
112 
113 	if (irqs_suspended)
114 		reg->pending_on_resume = true;
115 	return true;
116 }
117 #else
irq_do_timetravel_handler(struct irq_entry * entry,enum um_irq_type t)118 static bool irq_do_timetravel_handler(struct irq_entry *entry,
119 				      enum um_irq_type t)
120 {
121 	return false;
122 }
123 #endif
124 
sigio_reg_handler(int idx,struct irq_entry * entry,enum um_irq_type t,struct uml_pt_regs * regs,bool timetravel_handlers_only)125 static void sigio_reg_handler(int idx, struct irq_entry *entry, enum um_irq_type t,
126 			      struct uml_pt_regs *regs,
127 			      bool timetravel_handlers_only)
128 {
129 	struct irq_reg *reg = &entry->reg[t];
130 
131 	if (!reg->events)
132 		return;
133 
134 	if (os_epoll_triggered(idx, reg->events) <= 0)
135 		return;
136 
137 	if (irq_do_timetravel_handler(entry, t))
138 		return;
139 
140 	/*
141 	 * If we're called to only run time-travel handlers then don't
142 	 * actually proceed but mark sigio as pending (if applicable).
143 	 * For suspend/resume, timetravel_handlers_only may be true
144 	 * despite time-travel not being configured and used.
145 	 */
146 	if (timetravel_handlers_only) {
147 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
148 		mark_sigio_pending();
149 #endif
150 		return;
151 	}
152 
153 	irq_io_loop(reg, regs);
154 }
155 
_sigio_handler(struct uml_pt_regs * regs,bool timetravel_handlers_only)156 static void _sigio_handler(struct uml_pt_regs *regs,
157 			   bool timetravel_handlers_only)
158 {
159 	struct irq_entry *irq_entry;
160 	int n, i;
161 
162 	if (timetravel_handlers_only && !um_irq_timetravel_handler_used())
163 		return;
164 
165 	while (1) {
166 		/* This is now lockless - epoll keeps back-referencesto the irqs
167 		 * which have trigger it so there is no need to walk the irq
168 		 * list and lock it every time. We avoid locking by turning off
169 		 * IO for a specific fd by executing os_del_epoll_fd(fd) before
170 		 * we do any changes to the actual data structures
171 		 */
172 		n = os_waiting_for_events_epoll();
173 
174 		if (n <= 0) {
175 			if (n == -EINTR)
176 				continue;
177 			else
178 				break;
179 		}
180 
181 		for (i = 0; i < n ; i++) {
182 			enum um_irq_type t;
183 
184 			irq_entry = os_epoll_get_data_pointer(i);
185 
186 			for (t = 0; t < NUM_IRQ_TYPES; t++)
187 				sigio_reg_handler(i, irq_entry, t, regs,
188 						  timetravel_handlers_only);
189 		}
190 	}
191 
192 	if (!timetravel_handlers_only)
193 		free_irqs();
194 }
195 
sigio_handler(int sig,struct siginfo * unused_si,struct uml_pt_regs * regs)196 void sigio_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs)
197 {
198 	_sigio_handler(regs, irqs_suspended);
199 }
200 
get_irq_entry_by_fd(int fd)201 static struct irq_entry *get_irq_entry_by_fd(int fd)
202 {
203 	struct irq_entry *walk;
204 
205 	lockdep_assert_held(&irq_lock);
206 
207 	list_for_each_entry(walk, &active_fds, list) {
208 		if (walk->fd == fd)
209 			return walk;
210 	}
211 
212 	return NULL;
213 }
214 
free_irq_entry(struct irq_entry * to_free,bool remove)215 static void free_irq_entry(struct irq_entry *to_free, bool remove)
216 {
217 	if (!to_free)
218 		return;
219 
220 	if (remove)
221 		os_del_epoll_fd(to_free->fd);
222 	list_del(&to_free->list);
223 	kfree(to_free);
224 }
225 
update_irq_entry(struct irq_entry * entry)226 static bool update_irq_entry(struct irq_entry *entry)
227 {
228 	enum um_irq_type i;
229 	int events = 0;
230 
231 	for (i = 0; i < NUM_IRQ_TYPES; i++)
232 		events |= entry->reg[i].events;
233 
234 	if (events) {
235 		/* will modify (instead of add) if needed */
236 		os_add_epoll_fd(events, entry->fd, entry);
237 		return true;
238 	}
239 
240 	os_del_epoll_fd(entry->fd);
241 	return false;
242 }
243 
update_or_free_irq_entry(struct irq_entry * entry)244 static void update_or_free_irq_entry(struct irq_entry *entry)
245 {
246 	if (!update_irq_entry(entry))
247 		free_irq_entry(entry, false);
248 }
249 
activate_fd(int irq,int fd,enum um_irq_type type,void * dev_id,void (* timetravel_handler)(int,int,void *,struct time_travel_event *))250 static int activate_fd(int irq, int fd, enum um_irq_type type, void *dev_id,
251 		       void (*timetravel_handler)(int, int, void *,
252 						  struct time_travel_event *))
253 {
254 	struct irq_entry *irq_entry;
255 	int err, events = os_event_mask(type);
256 	unsigned long flags;
257 
258 	err = os_set_fd_async(fd);
259 	if (err < 0)
260 		goto out;
261 
262 	spin_lock_irqsave(&irq_lock, flags);
263 	irq_entry = get_irq_entry_by_fd(fd);
264 	if (irq_entry) {
265 		/* cannot register the same FD twice with the same type */
266 		if (WARN_ON(irq_entry->reg[type].events)) {
267 			err = -EALREADY;
268 			goto out_unlock;
269 		}
270 
271 		/* temporarily disable to avoid IRQ-side locking */
272 		os_del_epoll_fd(fd);
273 	} else {
274 		irq_entry = kzalloc(sizeof(*irq_entry), GFP_ATOMIC);
275 		if (!irq_entry) {
276 			err = -ENOMEM;
277 			goto out_unlock;
278 		}
279 		irq_entry->fd = fd;
280 		list_add_tail(&irq_entry->list, &active_fds);
281 		maybe_sigio_broken(fd);
282 	}
283 
284 	irq_entry->reg[type].id = dev_id;
285 	irq_entry->reg[type].irq = irq;
286 	irq_entry->reg[type].active = true;
287 	irq_entry->reg[type].events = events;
288 
289 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
290 	if (um_irq_timetravel_handler_used()) {
291 		irq_entry->reg[type].timetravel_handler = timetravel_handler;
292 		irq_entry->reg[type].event.fn = irq_event_handler;
293 	}
294 #endif
295 
296 	WARN_ON(!update_irq_entry(irq_entry));
297 	spin_unlock_irqrestore(&irq_lock, flags);
298 
299 	return 0;
300 out_unlock:
301 	spin_unlock_irqrestore(&irq_lock, flags);
302 out:
303 	return err;
304 }
305 
306 /*
307  * Remove the entry or entries for a specific FD, if you
308  * don't want to remove all the possible entries then use
309  * um_free_irq() or deactivate_fd() instead.
310  */
free_irq_by_fd(int fd)311 void free_irq_by_fd(int fd)
312 {
313 	struct irq_entry *to_free;
314 	unsigned long flags;
315 
316 	spin_lock_irqsave(&irq_lock, flags);
317 	to_free = get_irq_entry_by_fd(fd);
318 	free_irq_entry(to_free, true);
319 	spin_unlock_irqrestore(&irq_lock, flags);
320 }
321 EXPORT_SYMBOL(free_irq_by_fd);
322 
free_irq_by_irq_and_dev(unsigned int irq,void * dev)323 static void free_irq_by_irq_and_dev(unsigned int irq, void *dev)
324 {
325 	struct irq_entry *entry;
326 	unsigned long flags;
327 
328 	spin_lock_irqsave(&irq_lock, flags);
329 	list_for_each_entry(entry, &active_fds, list) {
330 		enum um_irq_type i;
331 
332 		for (i = 0; i < NUM_IRQ_TYPES; i++) {
333 			struct irq_reg *reg = &entry->reg[i];
334 
335 			if (!reg->events)
336 				continue;
337 			if (reg->irq != irq)
338 				continue;
339 			if (reg->id != dev)
340 				continue;
341 
342 			os_del_epoll_fd(entry->fd);
343 			reg->events = 0;
344 			update_or_free_irq_entry(entry);
345 			goto out;
346 		}
347 	}
348 out:
349 	spin_unlock_irqrestore(&irq_lock, flags);
350 }
351 
deactivate_fd(int fd,int irqnum)352 void deactivate_fd(int fd, int irqnum)
353 {
354 	struct irq_entry *entry;
355 	unsigned long flags;
356 	enum um_irq_type i;
357 
358 	os_del_epoll_fd(fd);
359 
360 	spin_lock_irqsave(&irq_lock, flags);
361 	entry = get_irq_entry_by_fd(fd);
362 	if (!entry)
363 		goto out;
364 
365 	for (i = 0; i < NUM_IRQ_TYPES; i++) {
366 		if (!entry->reg[i].events)
367 			continue;
368 		if (entry->reg[i].irq == irqnum)
369 			entry->reg[i].events = 0;
370 	}
371 
372 	update_or_free_irq_entry(entry);
373 out:
374 	spin_unlock_irqrestore(&irq_lock, flags);
375 
376 	ignore_sigio_fd(fd);
377 }
378 EXPORT_SYMBOL(deactivate_fd);
379 
380 /*
381  * Called just before shutdown in order to provide a clean exec
382  * environment in case the system is rebooting.  No locking because
383  * that would cause a pointless shutdown hang if something hadn't
384  * released the lock.
385  */
deactivate_all_fds(void)386 int deactivate_all_fds(void)
387 {
388 	struct irq_entry *entry;
389 
390 	/* Stop IO. The IRQ loop has no lock so this is our
391 	 * only way of making sure we are safe to dispose
392 	 * of all IRQ handlers
393 	 */
394 	os_set_ioignore();
395 
396 	/* we can no longer call kfree() here so just deactivate */
397 	list_for_each_entry(entry, &active_fds, list)
398 		os_del_epoll_fd(entry->fd);
399 	os_close_epoll_fd();
400 	return 0;
401 }
402 
403 /*
404  * do_IRQ handles all normal device IRQs (the special
405  * SMP cross-CPU interrupts have their own specific
406  * handlers).
407  */
do_IRQ(int irq,struct uml_pt_regs * regs)408 unsigned int do_IRQ(int irq, struct uml_pt_regs *regs)
409 {
410 	struct pt_regs *old_regs = set_irq_regs((struct pt_regs *)regs);
411 	irq_enter();
412 	generic_handle_irq(irq);
413 	irq_exit();
414 	set_irq_regs(old_regs);
415 	return 1;
416 }
417 
um_free_irq(int irq,void * dev)418 void um_free_irq(int irq, void *dev)
419 {
420 	if (WARN(irq < 0 || irq > UM_LAST_SIGNAL_IRQ,
421 		 "freeing invalid irq %d", irq))
422 		return;
423 
424 	free_irq_by_irq_and_dev(irq, dev);
425 	free_irq(irq, dev);
426 	clear_bit(irq, irqs_allocated);
427 }
428 EXPORT_SYMBOL(um_free_irq);
429 
430 static int
_um_request_irq(int irq,int fd,enum um_irq_type type,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id,void (* timetravel_handler)(int,int,void *,struct time_travel_event *))431 _um_request_irq(int irq, int fd, enum um_irq_type type,
432 		irq_handler_t handler, unsigned long irqflags,
433 		const char *devname, void *dev_id,
434 		void (*timetravel_handler)(int, int, void *,
435 					   struct time_travel_event *))
436 {
437 	int err;
438 
439 	if (irq == UM_IRQ_ALLOC) {
440 		int i;
441 
442 		for (i = UM_FIRST_DYN_IRQ; i < NR_IRQS; i++) {
443 			if (!test_and_set_bit(i, irqs_allocated)) {
444 				irq = i;
445 				break;
446 			}
447 		}
448 	}
449 
450 	if (irq < 0)
451 		return -ENOSPC;
452 
453 	if (fd != -1) {
454 		err = activate_fd(irq, fd, type, dev_id, timetravel_handler);
455 		if (err)
456 			goto error;
457 	}
458 
459 	err = request_irq(irq, handler, irqflags, devname, dev_id);
460 	if (err < 0)
461 		goto error;
462 
463 	return irq;
464 error:
465 	clear_bit(irq, irqs_allocated);
466 	return err;
467 }
468 
um_request_irq(int irq,int fd,enum um_irq_type type,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id)469 int um_request_irq(int irq, int fd, enum um_irq_type type,
470 		   irq_handler_t handler, unsigned long irqflags,
471 		   const char *devname, void *dev_id)
472 {
473 	return _um_request_irq(irq, fd, type, handler, irqflags,
474 			       devname, dev_id, NULL);
475 }
476 EXPORT_SYMBOL(um_request_irq);
477 
478 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
um_request_irq_tt(int irq,int fd,enum um_irq_type type,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id,void (* timetravel_handler)(int,int,void *,struct time_travel_event *))479 int um_request_irq_tt(int irq, int fd, enum um_irq_type type,
480 		      irq_handler_t handler, unsigned long irqflags,
481 		      const char *devname, void *dev_id,
482 		      void (*timetravel_handler)(int, int, void *,
483 						 struct time_travel_event *))
484 {
485 	return _um_request_irq(irq, fd, type, handler, irqflags,
486 			       devname, dev_id, timetravel_handler);
487 }
488 EXPORT_SYMBOL(um_request_irq_tt);
489 
sigio_run_timetravel_handlers(void)490 void sigio_run_timetravel_handlers(void)
491 {
492 	_sigio_handler(NULL, true);
493 }
494 #endif
495 
496 #ifdef CONFIG_PM_SLEEP
um_irqs_suspend(void)497 void um_irqs_suspend(void)
498 {
499 	struct irq_entry *entry;
500 	unsigned long flags;
501 
502 	irqs_suspended = true;
503 
504 	spin_lock_irqsave(&irq_lock, flags);
505 	list_for_each_entry(entry, &active_fds, list) {
506 		enum um_irq_type t;
507 		bool clear = true;
508 
509 		for (t = 0; t < NUM_IRQ_TYPES; t++) {
510 			if (!entry->reg[t].events)
511 				continue;
512 
513 			/*
514 			 * For the SIGIO_WRITE_IRQ, which is used to handle the
515 			 * SIGIO workaround thread, we need special handling:
516 			 * enable wake for it itself, but below we tell it about
517 			 * any FDs that should be suspended.
518 			 */
519 			if (entry->reg[t].wakeup ||
520 			    entry->reg[t].irq == SIGIO_WRITE_IRQ
521 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
522 			    || entry->reg[t].timetravel_handler
523 #endif
524 			    ) {
525 				clear = false;
526 				break;
527 			}
528 		}
529 
530 		if (clear) {
531 			entry->suspended = true;
532 			os_clear_fd_async(entry->fd);
533 			entry->sigio_workaround =
534 				!__ignore_sigio_fd(entry->fd);
535 		}
536 	}
537 	spin_unlock_irqrestore(&irq_lock, flags);
538 }
539 
um_irqs_resume(void)540 void um_irqs_resume(void)
541 {
542 	struct irq_entry *entry;
543 	unsigned long flags;
544 
545 
546 	local_irq_save(flags);
547 #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
548 	/*
549 	 * We don't need to lock anything here since we're in resume
550 	 * and nothing else is running, but have disabled IRQs so we
551 	 * don't try anything else with the interrupt list from there.
552 	 */
553 	list_for_each_entry(entry, &active_fds, list) {
554 		enum um_irq_type t;
555 
556 		for (t = 0; t < NUM_IRQ_TYPES; t++) {
557 			struct irq_reg *reg = &entry->reg[t];
558 
559 			if (reg->pending_on_resume) {
560 				irq_enter();
561 				generic_handle_irq(reg->irq);
562 				irq_exit();
563 				reg->pending_on_resume = false;
564 			}
565 		}
566 	}
567 #endif
568 
569 	spin_lock(&irq_lock);
570 	list_for_each_entry(entry, &active_fds, list) {
571 		if (entry->suspended) {
572 			int err = os_set_fd_async(entry->fd);
573 
574 			WARN(err < 0, "os_set_fd_async returned %d\n", err);
575 			entry->suspended = false;
576 
577 			if (entry->sigio_workaround) {
578 				err = __add_sigio_fd(entry->fd);
579 				WARN(err < 0, "add_sigio_returned %d\n", err);
580 			}
581 		}
582 	}
583 	spin_unlock_irqrestore(&irq_lock, flags);
584 
585 	irqs_suspended = false;
586 	send_sigio_to_self();
587 }
588 
normal_irq_set_wake(struct irq_data * d,unsigned int on)589 static int normal_irq_set_wake(struct irq_data *d, unsigned int on)
590 {
591 	struct irq_entry *entry;
592 	unsigned long flags;
593 
594 	spin_lock_irqsave(&irq_lock, flags);
595 	list_for_each_entry(entry, &active_fds, list) {
596 		enum um_irq_type t;
597 
598 		for (t = 0; t < NUM_IRQ_TYPES; t++) {
599 			if (!entry->reg[t].events)
600 				continue;
601 
602 			if (entry->reg[t].irq != d->irq)
603 				continue;
604 			entry->reg[t].wakeup = on;
605 			goto unlock;
606 		}
607 	}
608 unlock:
609 	spin_unlock_irqrestore(&irq_lock, flags);
610 	return 0;
611 }
612 #else
613 #define normal_irq_set_wake NULL
614 #endif
615 
616 /*
617  * irq_chip must define at least enable/disable and ack when
618  * the edge handler is used.
619  */
dummy(struct irq_data * d)620 static void dummy(struct irq_data *d)
621 {
622 }
623 
624 /* This is used for everything other than the timer. */
625 static struct irq_chip normal_irq_type = {
626 	.name = "SIGIO",
627 	.irq_disable = dummy,
628 	.irq_enable = dummy,
629 	.irq_ack = dummy,
630 	.irq_mask = dummy,
631 	.irq_unmask = dummy,
632 	.irq_set_wake = normal_irq_set_wake,
633 };
634 
635 static struct irq_chip alarm_irq_type = {
636 	.name = "SIGALRM",
637 	.irq_disable = dummy,
638 	.irq_enable = dummy,
639 	.irq_ack = dummy,
640 	.irq_mask = dummy,
641 	.irq_unmask = dummy,
642 };
643 
init_IRQ(void)644 void __init init_IRQ(void)
645 {
646 	int i;
647 
648 	irq_set_chip_and_handler(TIMER_IRQ, &alarm_irq_type, handle_edge_irq);
649 
650 	for (i = 1; i < UM_LAST_SIGNAL_IRQ; i++)
651 		irq_set_chip_and_handler(i, &normal_irq_type, handle_edge_irq);
652 	/* Initialize EPOLL Loop */
653 	os_setup_epoll();
654 }
655 
656 /*
657  * IRQ stack entry and exit:
658  *
659  * Unlike i386, UML doesn't receive IRQs on the normal kernel stack
660  * and switch over to the IRQ stack after some preparation.  We use
661  * sigaltstack to receive signals on a separate stack from the start.
662  * These two functions make sure the rest of the kernel won't be too
663  * upset by being on a different stack.  The IRQ stack has a
664  * thread_info structure at the bottom so that current et al continue
665  * to work.
666  *
667  * to_irq_stack copies the current task's thread_info to the IRQ stack
668  * thread_info and sets the tasks's stack to point to the IRQ stack.
669  *
670  * from_irq_stack copies the thread_info struct back (flags may have
671  * been modified) and resets the task's stack pointer.
672  *
673  * Tricky bits -
674  *
675  * What happens when two signals race each other?  UML doesn't block
676  * signals with sigprocmask, SA_DEFER, or sa_mask, so a second signal
677  * could arrive while a previous one is still setting up the
678  * thread_info.
679  *
680  * There are three cases -
681  *     The first interrupt on the stack - sets up the thread_info and
682  * handles the interrupt
683  *     A nested interrupt interrupting the copying of the thread_info -
684  * can't handle the interrupt, as the stack is in an unknown state
685  *     A nested interrupt not interrupting the copying of the
686  * thread_info - doesn't do any setup, just handles the interrupt
687  *
688  * The first job is to figure out whether we interrupted stack setup.
689  * This is done by xchging the signal mask with thread_info->pending.
690  * If the value that comes back is zero, then there is no setup in
691  * progress, and the interrupt can be handled.  If the value is
692  * non-zero, then there is stack setup in progress.  In order to have
693  * the interrupt handled, we leave our signal in the mask, and it will
694  * be handled by the upper handler after it has set up the stack.
695  *
696  * Next is to figure out whether we are the outer handler or a nested
697  * one.  As part of setting up the stack, thread_info->real_thread is
698  * set to non-NULL (and is reset to NULL on exit).  This is the
699  * nesting indicator.  If it is non-NULL, then the stack is already
700  * set up and the handler can run.
701  */
702 
703 static unsigned long pending_mask;
704 
to_irq_stack(unsigned long * mask_out)705 unsigned long to_irq_stack(unsigned long *mask_out)
706 {
707 	struct thread_info *ti;
708 	unsigned long mask, old;
709 	int nested;
710 
711 	mask = xchg(&pending_mask, *mask_out);
712 	if (mask != 0) {
713 		/*
714 		 * If any interrupts come in at this point, we want to
715 		 * make sure that their bits aren't lost by our
716 		 * putting our bit in.  So, this loop accumulates bits
717 		 * until xchg returns the same value that we put in.
718 		 * When that happens, there were no new interrupts,
719 		 * and pending_mask contains a bit for each interrupt
720 		 * that came in.
721 		 */
722 		old = *mask_out;
723 		do {
724 			old |= mask;
725 			mask = xchg(&pending_mask, old);
726 		} while (mask != old);
727 		return 1;
728 	}
729 
730 	ti = current_thread_info();
731 	nested = (ti->real_thread != NULL);
732 	if (!nested) {
733 		struct task_struct *task;
734 		struct thread_info *tti;
735 
736 		task = cpu_tasks[ti->cpu].task;
737 		tti = task_thread_info(task);
738 
739 		*ti = *tti;
740 		ti->real_thread = tti;
741 		task->stack = ti;
742 	}
743 
744 	mask = xchg(&pending_mask, 0);
745 	*mask_out |= mask | nested;
746 	return 0;
747 }
748 
from_irq_stack(int nested)749 unsigned long from_irq_stack(int nested)
750 {
751 	struct thread_info *ti, *to;
752 	unsigned long mask;
753 
754 	ti = current_thread_info();
755 
756 	pending_mask = 1;
757 
758 	to = ti->real_thread;
759 	current->stack = to;
760 	ti->real_thread = NULL;
761 	*to = *ti;
762 
763 	mask = xchg(&pending_mask, 0);
764 	return mask & ~1;
765 }
766 
767