1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/xz.h>
3 #include <linux/module.h>
4 #include "compress.h"
5 
6 struct z_erofs_lzma {
7 	struct z_erofs_lzma *next;
8 	struct xz_dec_microlzma *state;
9 	struct xz_buf buf;
10 	u8 bounce[PAGE_SIZE];
11 };
12 
13 /* considering the LZMA performance, no need to use a lockless list for now */
14 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
15 static unsigned int z_erofs_lzma_max_dictsize;
16 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
17 static struct z_erofs_lzma *z_erofs_lzma_head;
18 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
19 
20 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
21 
z_erofs_lzma_exit(void)22 void z_erofs_lzma_exit(void)
23 {
24 	/* there should be no running fs instance */
25 	while (z_erofs_lzma_avail_strms) {
26 		struct z_erofs_lzma *strm;
27 
28 		spin_lock(&z_erofs_lzma_lock);
29 		strm = z_erofs_lzma_head;
30 		if (!strm) {
31 			spin_unlock(&z_erofs_lzma_lock);
32 			DBG_BUGON(1);
33 			return;
34 		}
35 		z_erofs_lzma_head = NULL;
36 		spin_unlock(&z_erofs_lzma_lock);
37 
38 		while (strm) {
39 			struct z_erofs_lzma *n = strm->next;
40 
41 			if (strm->state)
42 				xz_dec_microlzma_end(strm->state);
43 			kfree(strm);
44 			--z_erofs_lzma_avail_strms;
45 			strm = n;
46 		}
47 	}
48 }
49 
z_erofs_lzma_init(void)50 int __init z_erofs_lzma_init(void)
51 {
52 	unsigned int i;
53 
54 	/* by default, use # of possible CPUs instead */
55 	if (!z_erofs_lzma_nstrms)
56 		z_erofs_lzma_nstrms = num_possible_cpus();
57 
58 	for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
59 		struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
60 
61 		if (!strm) {
62 			z_erofs_lzma_exit();
63 			return -ENOMEM;
64 		}
65 		spin_lock(&z_erofs_lzma_lock);
66 		strm->next = z_erofs_lzma_head;
67 		z_erofs_lzma_head = strm;
68 		spin_unlock(&z_erofs_lzma_lock);
69 		++z_erofs_lzma_avail_strms;
70 	}
71 	return 0;
72 }
73 
z_erofs_load_lzma_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)74 int z_erofs_load_lzma_config(struct super_block *sb,
75 			struct erofs_super_block *dsb, void *data, int size)
76 {
77 	static DEFINE_MUTEX(lzma_resize_mutex);
78 	struct z_erofs_lzma_cfgs *lzma = data;
79 	unsigned int dict_size, i;
80 	struct z_erofs_lzma *strm, *head = NULL;
81 	int err;
82 
83 	if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
84 		erofs_err(sb, "invalid lzma cfgs, size=%u", size);
85 		return -EINVAL;
86 	}
87 	if (lzma->format) {
88 		erofs_err(sb, "unidentified lzma format %x, please check kernel version",
89 			  le16_to_cpu(lzma->format));
90 		return -EINVAL;
91 	}
92 	dict_size = le32_to_cpu(lzma->dict_size);
93 	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
94 		erofs_err(sb, "unsupported lzma dictionary size %u",
95 			  dict_size);
96 		return -EINVAL;
97 	}
98 
99 	erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
100 
101 	/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
102 	mutex_lock(&lzma_resize_mutex);
103 
104 	if (z_erofs_lzma_max_dictsize >= dict_size) {
105 		mutex_unlock(&lzma_resize_mutex);
106 		return 0;
107 	}
108 
109 	/* 1. collect/isolate all streams for the following check */
110 	for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
111 		struct z_erofs_lzma *last;
112 
113 again:
114 		spin_lock(&z_erofs_lzma_lock);
115 		strm = z_erofs_lzma_head;
116 		if (!strm) {
117 			spin_unlock(&z_erofs_lzma_lock);
118 			wait_event(z_erofs_lzma_wq,
119 				   READ_ONCE(z_erofs_lzma_head));
120 			goto again;
121 		}
122 		z_erofs_lzma_head = NULL;
123 		spin_unlock(&z_erofs_lzma_lock);
124 
125 		for (last = strm; last->next; last = last->next)
126 			++i;
127 		last->next = head;
128 		head = strm;
129 	}
130 
131 	err = 0;
132 	/* 2. walk each isolated stream and grow max dict_size if needed */
133 	for (strm = head; strm; strm = strm->next) {
134 		if (strm->state)
135 			xz_dec_microlzma_end(strm->state);
136 		strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
137 		if (!strm->state)
138 			err = -ENOMEM;
139 	}
140 
141 	/* 3. push back all to the global list and update max dict_size */
142 	spin_lock(&z_erofs_lzma_lock);
143 	DBG_BUGON(z_erofs_lzma_head);
144 	z_erofs_lzma_head = head;
145 	spin_unlock(&z_erofs_lzma_lock);
146 	wake_up_all(&z_erofs_lzma_wq);
147 
148 	z_erofs_lzma_max_dictsize = dict_size;
149 	mutex_unlock(&lzma_resize_mutex);
150 	return err;
151 }
152 
z_erofs_lzma_decompress(struct z_erofs_decompress_req * rq,struct page ** pagepool)153 int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
154 			    struct page **pagepool)
155 {
156 	const unsigned int nrpages_out =
157 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
158 	const unsigned int nrpages_in =
159 		PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
160 	unsigned int inlen, outlen, pageofs;
161 	struct z_erofs_lzma *strm;
162 	u8 *kin;
163 	bool bounced = false;
164 	int no, ni, j, err = 0;
165 
166 	/* 1. get the exact LZMA compressed size */
167 	kin = kmap(*rq->in);
168 	err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
169 			min_t(unsigned int, rq->inputsize,
170 			      rq->sb->s_blocksize - rq->pageofs_in));
171 	if (err) {
172 		kunmap(*rq->in);
173 		return err;
174 	}
175 
176 	/* 2. get an available lzma context */
177 again:
178 	spin_lock(&z_erofs_lzma_lock);
179 	strm = z_erofs_lzma_head;
180 	if (!strm) {
181 		spin_unlock(&z_erofs_lzma_lock);
182 		wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
183 		goto again;
184 	}
185 	z_erofs_lzma_head = strm->next;
186 	spin_unlock(&z_erofs_lzma_lock);
187 
188 	/* 3. multi-call decompress */
189 	inlen = rq->inputsize;
190 	outlen = rq->outputsize;
191 	xz_dec_microlzma_reset(strm->state, inlen, outlen,
192 			       !rq->partial_decoding);
193 	pageofs = rq->pageofs_out;
194 	strm->buf.in = kin + rq->pageofs_in;
195 	strm->buf.in_pos = 0;
196 	strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
197 	inlen -= strm->buf.in_size;
198 	strm->buf.out = NULL;
199 	strm->buf.out_pos = 0;
200 	strm->buf.out_size = 0;
201 
202 	for (ni = 0, no = -1;;) {
203 		enum xz_ret xz_err;
204 
205 		if (strm->buf.out_pos == strm->buf.out_size) {
206 			if (strm->buf.out) {
207 				kunmap(rq->out[no]);
208 				strm->buf.out = NULL;
209 			}
210 
211 			if (++no >= nrpages_out || !outlen) {
212 				erofs_err(rq->sb, "decompressed buf out of bound");
213 				err = -EFSCORRUPTED;
214 				break;
215 			}
216 			strm->buf.out_pos = 0;
217 			strm->buf.out_size = min_t(u32, outlen,
218 						   PAGE_SIZE - pageofs);
219 			outlen -= strm->buf.out_size;
220 			if (!rq->out[no] && rq->fillgaps) {	/* deduped */
221 				rq->out[no] = erofs_allocpage(pagepool,
222 						GFP_KERNEL | __GFP_NOFAIL);
223 				set_page_private(rq->out[no],
224 						 Z_EROFS_SHORTLIVED_PAGE);
225 			}
226 			if (rq->out[no])
227 				strm->buf.out = kmap(rq->out[no]) + pageofs;
228 			pageofs = 0;
229 		} else if (strm->buf.in_pos == strm->buf.in_size) {
230 			kunmap(rq->in[ni]);
231 
232 			if (++ni >= nrpages_in || !inlen) {
233 				erofs_err(rq->sb, "compressed buf out of bound");
234 				err = -EFSCORRUPTED;
235 				break;
236 			}
237 			strm->buf.in_pos = 0;
238 			strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
239 			inlen -= strm->buf.in_size;
240 			kin = kmap(rq->in[ni]);
241 			strm->buf.in = kin;
242 			bounced = false;
243 		}
244 
245 		/*
246 		 * Handle overlapping: Use bounced buffer if the compressed
247 		 * data is under processing; Otherwise, Use short-lived pages
248 		 * from the on-stack pagepool where pages share with the same
249 		 * request.
250 		 */
251 		if (!bounced && rq->out[no] == rq->in[ni]) {
252 			memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
253 			strm->buf.in = strm->bounce;
254 			bounced = true;
255 		}
256 		for (j = ni + 1; j < nrpages_in; ++j) {
257 			struct page *tmppage;
258 
259 			if (rq->out[no] != rq->in[j])
260 				continue;
261 
262 			DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
263 							rq->in[j]));
264 			tmppage = erofs_allocpage(pagepool,
265 						  GFP_KERNEL | __GFP_NOFAIL);
266 			set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
267 			copy_highpage(tmppage, rq->in[j]);
268 			rq->in[j] = tmppage;
269 		}
270 		xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
271 		DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
272 		DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
273 
274 		if (xz_err != XZ_OK) {
275 			if (xz_err == XZ_STREAM_END && !outlen)
276 				break;
277 			erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
278 				  xz_err, rq->inputsize, rq->outputsize);
279 			err = -EFSCORRUPTED;
280 			break;
281 		}
282 	}
283 	if (no < nrpages_out && strm->buf.out)
284 		kunmap(rq->out[no]);
285 	if (ni < nrpages_in)
286 		kunmap(rq->in[ni]);
287 	/* 4. push back LZMA stream context to the global list */
288 	spin_lock(&z_erofs_lzma_lock);
289 	strm->next = z_erofs_lzma_head;
290 	z_erofs_lzma_head = strm;
291 	spin_unlock(&z_erofs_lzma_lock);
292 	wake_up(&z_erofs_lzma_wq);
293 	return err;
294 }
295