1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2021 Facebook
3  */
4 
5 #ifndef __MMAP_UNLOCK_WORK_H__
6 #define __MMAP_UNLOCK_WORK_H__
7 #include <linux/irq_work.h>
8 
9 /* irq_work to run mmap_read_unlock() in irq_work */
10 struct mmap_unlock_irq_work {
11 	struct irq_work irq_work;
12 	struct mm_struct *mm;
13 };
14 
15 DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
16 
17 /*
18  * We cannot do mmap_read_unlock() when the irq is disabled, because of
19  * risk to deadlock with rq_lock. To look up vma when the irqs are
20  * disabled, we need to run mmap_read_unlock() in irq_work. We use a
21  * percpu variable to do the irq_work. If the irq_work is already used
22  * by another lookup, we fall over.
23  */
bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work ** work_ptr)24 static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr)
25 {
26 	struct mmap_unlock_irq_work *work = NULL;
27 	bool irq_work_busy = false;
28 
29 	if (irqs_disabled()) {
30 		if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
31 			work = this_cpu_ptr(&mmap_unlock_work);
32 			if (irq_work_is_busy(&work->irq_work)) {
33 				/* cannot queue more up_read, fallback */
34 				irq_work_busy = true;
35 			}
36 		} else {
37 			/*
38 			 * PREEMPT_RT does not allow to trylock mmap sem in
39 			 * interrupt disabled context. Force the fallback code.
40 			 */
41 			irq_work_busy = true;
42 		}
43 	}
44 
45 	*work_ptr = work;
46 	return irq_work_busy;
47 }
48 
bpf_mmap_unlock_mm(struct mmap_unlock_irq_work * work,struct mm_struct * mm)49 static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)
50 {
51 	if (!work) {
52 		mmap_read_unlock(mm);
53 	} else {
54 		work->mm = mm;
55 
56 		/* The lock will be released once we're out of interrupt
57 		 * context. Tell lockdep that we've released it now so
58 		 * it doesn't complain that we forgot to release it.
59 		 */
60 		rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
61 		irq_work_queue(&work->irq_work);
62 	}
63 }
64 
65 #endif /* __MMAP_UNLOCK_WORK_H__ */
66