1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) Gao Xiang <xiang@kernel.org>
4  *
5  * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
6  * per-CPU virtual memory (in pages) in advance to store such inplace I/O
7  * data if inplace decompression is failed (due to unmet inplace margin for
8  * example).
9  */
10 #include "internal.h"
11 
12 struct erofs_pcpubuf {
13 	raw_spinlock_t lock;
14 	void *ptr;
15 	struct page **pages;
16 	unsigned int nrpages;
17 };
18 
19 static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
20 
erofs_get_pcpubuf(unsigned int requiredpages)21 void *erofs_get_pcpubuf(unsigned int requiredpages)
22 	__acquires(pcb->lock)
23 {
24 	struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
25 
26 	raw_spin_lock(&pcb->lock);
27 	/* check if the per-CPU buffer is too small */
28 	if (requiredpages > pcb->nrpages) {
29 		raw_spin_unlock(&pcb->lock);
30 		put_cpu_var(erofs_pcb);
31 		/* (for sparse checker) pretend pcb->lock is still taken */
32 		__acquire(pcb->lock);
33 		return NULL;
34 	}
35 	return pcb->ptr;
36 }
37 
erofs_put_pcpubuf(void * ptr)38 void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
39 {
40 	struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
41 
42 	DBG_BUGON(pcb->ptr != ptr);
43 	raw_spin_unlock(&pcb->lock);
44 	put_cpu_var(erofs_pcb);
45 }
46 
47 /* the next step: support per-CPU page buffers hotplug */
erofs_pcpubuf_growsize(unsigned int nrpages)48 int erofs_pcpubuf_growsize(unsigned int nrpages)
49 {
50 	static DEFINE_MUTEX(pcb_resize_mutex);
51 	static unsigned int pcb_nrpages;
52 	struct page *pagepool = NULL;
53 	int delta, cpu, ret, i;
54 
55 	mutex_lock(&pcb_resize_mutex);
56 	delta = nrpages - pcb_nrpages;
57 	ret = 0;
58 	/* avoid shrinking pcpubuf, since no idea how many fses rely on */
59 	if (delta <= 0)
60 		goto out;
61 
62 	for_each_possible_cpu(cpu) {
63 		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
64 		struct page **pages, **oldpages;
65 		void *ptr, *old_ptr;
66 
67 		pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
68 		if (!pages) {
69 			ret = -ENOMEM;
70 			break;
71 		}
72 
73 		for (i = 0; i < nrpages; ++i) {
74 			pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
75 			if (!pages[i]) {
76 				ret = -ENOMEM;
77 				oldpages = pages;
78 				goto free_pagearray;
79 			}
80 		}
81 		ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
82 		if (!ptr) {
83 			ret = -ENOMEM;
84 			oldpages = pages;
85 			goto free_pagearray;
86 		}
87 		raw_spin_lock(&pcb->lock);
88 		old_ptr = pcb->ptr;
89 		pcb->ptr = ptr;
90 		oldpages = pcb->pages;
91 		pcb->pages = pages;
92 		i = pcb->nrpages;
93 		pcb->nrpages = nrpages;
94 		raw_spin_unlock(&pcb->lock);
95 
96 		if (!oldpages) {
97 			DBG_BUGON(old_ptr);
98 			continue;
99 		}
100 
101 		if (old_ptr)
102 			vunmap(old_ptr);
103 free_pagearray:
104 		while (i)
105 			erofs_pagepool_add(&pagepool, oldpages[--i]);
106 		kfree(oldpages);
107 		if (ret)
108 			break;
109 	}
110 	pcb_nrpages = nrpages;
111 	erofs_release_pages(&pagepool);
112 out:
113 	mutex_unlock(&pcb_resize_mutex);
114 	return ret;
115 }
116 
erofs_pcpubuf_init(void)117 void erofs_pcpubuf_init(void)
118 {
119 	int cpu;
120 
121 	for_each_possible_cpu(cpu) {
122 		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
123 
124 		raw_spin_lock_init(&pcb->lock);
125 	}
126 }
127 
erofs_pcpubuf_exit(void)128 void erofs_pcpubuf_exit(void)
129 {
130 	int cpu, i;
131 
132 	for_each_possible_cpu(cpu) {
133 		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
134 
135 		if (pcb->ptr) {
136 			vunmap(pcb->ptr);
137 			pcb->ptr = NULL;
138 		}
139 		if (!pcb->pages)
140 			continue;
141 
142 		for (i = 0; i < pcb->nrpages; ++i)
143 			if (pcb->pages[i])
144 				put_page(pcb->pages[i]);
145 		kfree(pcb->pages);
146 		pcb->pages = NULL;
147 	}
148 }
149