1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Google, Inc.
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/device.h>
9 #include <linux/err.h>
10 #include <linux/errno.h>
11 #include <linux/init.h>
12 #include <linux/io.h>
13 #include <linux/kernel.h>
14 #include <linux/list.h>
15 #include <linux/memblock.h>
16 #include <linux/pstore_ram.h>
17 #include <linux/rslib.h>
18 #include <linux/slab.h>
19 #include <linux/uaccess.h>
20 #include <linux/vmalloc.h>
21 #include <asm/page.h>
22 
23 /**
24  * struct persistent_ram_buffer - persistent circular RAM buffer
25  *
26  * @sig:
27  *	signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value)
28  * @start:
29  *	offset into @data where the beginning of the stored bytes begin
30  * @size:
31  *	number of valid bytes stored in @data
32  */
33 struct persistent_ram_buffer {
34 	uint32_t    sig;
35 	atomic_t    start;
36 	atomic_t    size;
37 	uint8_t     data[];
38 };
39 
40 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
41 
buffer_size(struct persistent_ram_zone * prz)42 static inline size_t buffer_size(struct persistent_ram_zone *prz)
43 {
44 	return atomic_read(&prz->buffer->size);
45 }
46 
buffer_start(struct persistent_ram_zone * prz)47 static inline size_t buffer_start(struct persistent_ram_zone *prz)
48 {
49 	return atomic_read(&prz->buffer->start);
50 }
51 
52 /* increase and wrap the start pointer, returning the old value */
buffer_start_add(struct persistent_ram_zone * prz,size_t a)53 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)
54 {
55 	int old;
56 	int new;
57 	unsigned long flags = 0;
58 
59 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
60 		raw_spin_lock_irqsave(&prz->buffer_lock, flags);
61 
62 	old = atomic_read(&prz->buffer->start);
63 	new = old + a;
64 	while (unlikely(new >= prz->buffer_size))
65 		new -= prz->buffer_size;
66 	atomic_set(&prz->buffer->start, new);
67 
68 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
69 		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);
70 
71 	return old;
72 }
73 
74 /* increase the size counter until it hits the max size */
buffer_size_add(struct persistent_ram_zone * prz,size_t a)75 static void buffer_size_add(struct persistent_ram_zone *prz, size_t a)
76 {
77 	size_t old;
78 	size_t new;
79 	unsigned long flags = 0;
80 
81 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
82 		raw_spin_lock_irqsave(&prz->buffer_lock, flags);
83 
84 	old = atomic_read(&prz->buffer->size);
85 	if (old == prz->buffer_size)
86 		goto exit;
87 
88 	new = old + a;
89 	if (new > prz->buffer_size)
90 		new = prz->buffer_size;
91 	atomic_set(&prz->buffer->size, new);
92 
93 exit:
94 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
95 		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);
96 }
97 
persistent_ram_encode_rs8(struct persistent_ram_zone * prz,uint8_t * data,size_t len,uint8_t * ecc)98 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,
99 	uint8_t *data, size_t len, uint8_t *ecc)
100 {
101 	int i;
102 
103 	/* Initialize the parity buffer */
104 	memset(prz->ecc_info.par, 0,
105 	       prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0]));
106 	encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0);
107 	for (i = 0; i < prz->ecc_info.ecc_size; i++)
108 		ecc[i] = prz->ecc_info.par[i];
109 }
110 
persistent_ram_decode_rs8(struct persistent_ram_zone * prz,void * data,size_t len,uint8_t * ecc)111 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,
112 	void *data, size_t len, uint8_t *ecc)
113 {
114 	int i;
115 
116 	for (i = 0; i < prz->ecc_info.ecc_size; i++)
117 		prz->ecc_info.par[i] = ecc[i];
118 	return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len,
119 				NULL, 0, NULL, 0, NULL);
120 }
121 
persistent_ram_update_ecc(struct persistent_ram_zone * prz,unsigned int start,unsigned int count)122 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,
123 	unsigned int start, unsigned int count)
124 {
125 	struct persistent_ram_buffer *buffer = prz->buffer;
126 	uint8_t *buffer_end = buffer->data + prz->buffer_size;
127 	uint8_t *block;
128 	uint8_t *par;
129 	int ecc_block_size = prz->ecc_info.block_size;
130 	int ecc_size = prz->ecc_info.ecc_size;
131 	int size = ecc_block_size;
132 
133 	if (!ecc_size)
134 		return;
135 
136 	block = buffer->data + (start & ~(ecc_block_size - 1));
137 	par = prz->par_buffer + (start / ecc_block_size) * ecc_size;
138 
139 	do {
140 		if (block + ecc_block_size > buffer_end)
141 			size = buffer_end - block;
142 		persistent_ram_encode_rs8(prz, block, size, par);
143 		block += ecc_block_size;
144 		par += ecc_size;
145 	} while (block < buffer->data + start + count);
146 }
147 
persistent_ram_update_header_ecc(struct persistent_ram_zone * prz)148 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)
149 {
150 	struct persistent_ram_buffer *buffer = prz->buffer;
151 
152 	if (!prz->ecc_info.ecc_size)
153 		return;
154 
155 	persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),
156 				  prz->par_header);
157 }
158 
persistent_ram_ecc_old(struct persistent_ram_zone * prz)159 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)
160 {
161 	struct persistent_ram_buffer *buffer = prz->buffer;
162 	uint8_t *block;
163 	uint8_t *par;
164 
165 	if (!prz->ecc_info.ecc_size)
166 		return;
167 
168 	block = buffer->data;
169 	par = prz->par_buffer;
170 	while (block < buffer->data + buffer_size(prz)) {
171 		int numerr;
172 		int size = prz->ecc_info.block_size;
173 		if (block + size > buffer->data + prz->buffer_size)
174 			size = buffer->data + prz->buffer_size - block;
175 		numerr = persistent_ram_decode_rs8(prz, block, size, par);
176 		if (numerr > 0) {
177 			pr_devel("error in block %p, %d\n", block, numerr);
178 			prz->corrected_bytes += numerr;
179 		} else if (numerr < 0) {
180 			pr_devel("uncorrectable error in block %p\n", block);
181 			prz->bad_blocks++;
182 		}
183 		block += prz->ecc_info.block_size;
184 		par += prz->ecc_info.ecc_size;
185 	}
186 }
187 
persistent_ram_init_ecc(struct persistent_ram_zone * prz,struct persistent_ram_ecc_info * ecc_info)188 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,
189 				   struct persistent_ram_ecc_info *ecc_info)
190 {
191 	int numerr;
192 	struct persistent_ram_buffer *buffer = prz->buffer;
193 	int ecc_blocks;
194 	size_t ecc_total;
195 
196 	if (!ecc_info || !ecc_info->ecc_size)
197 		return 0;
198 
199 	prz->ecc_info.block_size = ecc_info->block_size ?: 128;
200 	prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16;
201 	prz->ecc_info.symsize = ecc_info->symsize ?: 8;
202 	prz->ecc_info.poly = ecc_info->poly ?: 0x11d;
203 
204 	ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size,
205 				  prz->ecc_info.block_size +
206 				  prz->ecc_info.ecc_size);
207 	ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size;
208 	if (ecc_total >= prz->buffer_size) {
209 		pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n",
210 		       __func__, prz->ecc_info.ecc_size,
211 		       ecc_total, prz->buffer_size);
212 		return -EINVAL;
213 	}
214 
215 	prz->buffer_size -= ecc_total;
216 	prz->par_buffer = buffer->data + prz->buffer_size;
217 	prz->par_header = prz->par_buffer +
218 			  ecc_blocks * prz->ecc_info.ecc_size;
219 
220 	/*
221 	 * first consecutive root is 0
222 	 * primitive element to generate roots = 1
223 	 */
224 	prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly,
225 				  0, 1, prz->ecc_info.ecc_size);
226 	if (prz->rs_decoder == NULL) {
227 		pr_info("init_rs failed\n");
228 		return -EINVAL;
229 	}
230 
231 	/* allocate workspace instead of using stack VLA */
232 	prz->ecc_info.par = kmalloc_array(prz->ecc_info.ecc_size,
233 					  sizeof(*prz->ecc_info.par),
234 					  GFP_KERNEL);
235 	if (!prz->ecc_info.par) {
236 		pr_err("cannot allocate ECC parity workspace\n");
237 		return -ENOMEM;
238 	}
239 
240 	prz->corrected_bytes = 0;
241 	prz->bad_blocks = 0;
242 
243 	numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),
244 					   prz->par_header);
245 	if (numerr > 0) {
246 		pr_info("error in header, %d\n", numerr);
247 		prz->corrected_bytes += numerr;
248 	} else if (numerr < 0) {
249 		pr_info_ratelimited("uncorrectable error in header\n");
250 		prz->bad_blocks++;
251 	}
252 
253 	return 0;
254 }
255 
persistent_ram_ecc_string(struct persistent_ram_zone * prz,char * str,size_t len)256 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,
257 	char *str, size_t len)
258 {
259 	ssize_t ret;
260 
261 	if (!prz->ecc_info.ecc_size)
262 		return 0;
263 
264 	if (prz->corrected_bytes || prz->bad_blocks)
265 		ret = snprintf(str, len, ""
266 			"\nECC: %d Corrected bytes, %d unrecoverable blocks\n",
267 			prz->corrected_bytes, prz->bad_blocks);
268 	else
269 		ret = snprintf(str, len, "\nECC: No errors detected\n");
270 
271 	return ret;
272 }
273 
persistent_ram_update(struct persistent_ram_zone * prz,const void * s,unsigned int start,unsigned int count)274 static void notrace persistent_ram_update(struct persistent_ram_zone *prz,
275 	const void *s, unsigned int start, unsigned int count)
276 {
277 	struct persistent_ram_buffer *buffer = prz->buffer;
278 	memcpy_toio(buffer->data + start, s, count);
279 	persistent_ram_update_ecc(prz, start, count);
280 }
281 
persistent_ram_update_user(struct persistent_ram_zone * prz,const void __user * s,unsigned int start,unsigned int count)282 static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz,
283 	const void __user *s, unsigned int start, unsigned int count)
284 {
285 	struct persistent_ram_buffer *buffer = prz->buffer;
286 	int ret = unlikely(copy_from_user(buffer->data + start, s, count)) ?
287 		-EFAULT : 0;
288 	persistent_ram_update_ecc(prz, start, count);
289 	return ret;
290 }
291 
persistent_ram_save_old(struct persistent_ram_zone * prz)292 void persistent_ram_save_old(struct persistent_ram_zone *prz)
293 {
294 	struct persistent_ram_buffer *buffer = prz->buffer;
295 	size_t size = buffer_size(prz);
296 	size_t start = buffer_start(prz);
297 
298 	if (!size)
299 		return;
300 
301 	if (!prz->old_log) {
302 		persistent_ram_ecc_old(prz);
303 		prz->old_log = kmalloc(size, GFP_KERNEL);
304 	}
305 	if (!prz->old_log) {
306 		pr_err("failed to allocate buffer\n");
307 		return;
308 	}
309 
310 	prz->old_log_size = size;
311 	memcpy_fromio(prz->old_log, &buffer->data[start], size - start);
312 	memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start);
313 }
314 
persistent_ram_write(struct persistent_ram_zone * prz,const void * s,unsigned int count)315 int notrace persistent_ram_write(struct persistent_ram_zone *prz,
316 	const void *s, unsigned int count)
317 {
318 	int rem;
319 	int c = count;
320 	size_t start;
321 
322 	if (unlikely(c > prz->buffer_size)) {
323 		s += c - prz->buffer_size;
324 		c = prz->buffer_size;
325 	}
326 
327 	buffer_size_add(prz, c);
328 
329 	start = buffer_start_add(prz, c);
330 
331 	rem = prz->buffer_size - start;
332 	if (unlikely(rem < c)) {
333 		persistent_ram_update(prz, s, start, rem);
334 		s += rem;
335 		c -= rem;
336 		start = 0;
337 	}
338 	persistent_ram_update(prz, s, start, c);
339 
340 	persistent_ram_update_header_ecc(prz);
341 
342 	return count;
343 }
344 
persistent_ram_write_user(struct persistent_ram_zone * prz,const void __user * s,unsigned int count)345 int notrace persistent_ram_write_user(struct persistent_ram_zone *prz,
346 	const void __user *s, unsigned int count)
347 {
348 	int rem, ret = 0, c = count;
349 	size_t start;
350 
351 	if (unlikely(c > prz->buffer_size)) {
352 		s += c - prz->buffer_size;
353 		c = prz->buffer_size;
354 	}
355 
356 	buffer_size_add(prz, c);
357 
358 	start = buffer_start_add(prz, c);
359 
360 	rem = prz->buffer_size - start;
361 	if (unlikely(rem < c)) {
362 		ret = persistent_ram_update_user(prz, s, start, rem);
363 		s += rem;
364 		c -= rem;
365 		start = 0;
366 	}
367 	if (likely(!ret))
368 		ret = persistent_ram_update_user(prz, s, start, c);
369 
370 	persistent_ram_update_header_ecc(prz);
371 
372 	return unlikely(ret) ? ret : count;
373 }
374 
persistent_ram_old_size(struct persistent_ram_zone * prz)375 size_t persistent_ram_old_size(struct persistent_ram_zone *prz)
376 {
377 	return prz->old_log_size;
378 }
379 
persistent_ram_old(struct persistent_ram_zone * prz)380 void *persistent_ram_old(struct persistent_ram_zone *prz)
381 {
382 	return prz->old_log;
383 }
384 
persistent_ram_free_old(struct persistent_ram_zone * prz)385 void persistent_ram_free_old(struct persistent_ram_zone *prz)
386 {
387 	kfree(prz->old_log);
388 	prz->old_log = NULL;
389 	prz->old_log_size = 0;
390 }
391 
persistent_ram_zap(struct persistent_ram_zone * prz)392 void persistent_ram_zap(struct persistent_ram_zone *prz)
393 {
394 	atomic_set(&prz->buffer->start, 0);
395 	atomic_set(&prz->buffer->size, 0);
396 	persistent_ram_update_header_ecc(prz);
397 }
398 
399 #define MEM_TYPE_WCOMBINE	0
400 #define MEM_TYPE_NONCACHED	1
401 #define MEM_TYPE_NORMAL		2
402 
persistent_ram_vmap(phys_addr_t start,size_t size,unsigned int memtype)403 static void *persistent_ram_vmap(phys_addr_t start, size_t size,
404 		unsigned int memtype)
405 {
406 	struct page **pages;
407 	phys_addr_t page_start;
408 	unsigned int page_count;
409 	pgprot_t prot;
410 	unsigned int i;
411 	void *vaddr;
412 
413 	page_start = start - offset_in_page(start);
414 	page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE);
415 
416 	switch (memtype) {
417 	case MEM_TYPE_NORMAL:
418 		prot = PAGE_KERNEL;
419 		break;
420 	case MEM_TYPE_NONCACHED:
421 		prot = pgprot_noncached(PAGE_KERNEL);
422 		break;
423 	case MEM_TYPE_WCOMBINE:
424 		prot = pgprot_writecombine(PAGE_KERNEL);
425 		break;
426 	default:
427 		pr_err("invalid mem_type=%d\n", memtype);
428 		return NULL;
429 	}
430 
431 	pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
432 	if (!pages) {
433 		pr_err("%s: Failed to allocate array for %u pages\n",
434 		       __func__, page_count);
435 		return NULL;
436 	}
437 
438 	for (i = 0; i < page_count; i++) {
439 		phys_addr_t addr = page_start + i * PAGE_SIZE;
440 		pages[i] = pfn_to_page(addr >> PAGE_SHIFT);
441 	}
442 	vaddr = vmap(pages, page_count, VM_MAP, prot);
443 	kfree(pages);
444 
445 	/*
446 	 * Since vmap() uses page granularity, we must add the offset
447 	 * into the page here, to get the byte granularity address
448 	 * into the mapping to represent the actual "start" location.
449 	 */
450 	return vaddr + offset_in_page(start);
451 }
452 
persistent_ram_iomap(phys_addr_t start,size_t size,unsigned int memtype,char * label)453 static void *persistent_ram_iomap(phys_addr_t start, size_t size,
454 		unsigned int memtype, char *label)
455 {
456 	void *va;
457 
458 	if (!request_mem_region(start, size, label ?: "ramoops")) {
459 		pr_err("request mem region (%s 0x%llx@0x%llx) failed\n",
460 			label ?: "ramoops",
461 			(unsigned long long)size, (unsigned long long)start);
462 		return NULL;
463 	}
464 
465 	if (memtype)
466 		va = ioremap(start, size);
467 	else
468 		va = ioremap_wc(start, size);
469 
470 	/*
471 	 * Since request_mem_region() and ioremap() are byte-granularity
472 	 * there is no need handle anything special like we do when the
473 	 * vmap() case in persistent_ram_vmap() above.
474 	 */
475 	return va;
476 }
477 
persistent_ram_buffer_map(phys_addr_t start,phys_addr_t size,struct persistent_ram_zone * prz,int memtype)478 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size,
479 		struct persistent_ram_zone *prz, int memtype)
480 {
481 	prz->paddr = start;
482 	prz->size = size;
483 
484 	if (pfn_valid(start >> PAGE_SHIFT))
485 		prz->vaddr = persistent_ram_vmap(start, size, memtype);
486 	else
487 		prz->vaddr = persistent_ram_iomap(start, size, memtype,
488 						  prz->label);
489 
490 	if (!prz->vaddr) {
491 		pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__,
492 			(unsigned long long)size, (unsigned long long)start);
493 		return -ENOMEM;
494 	}
495 
496 	prz->buffer = prz->vaddr;
497 	prz->buffer_size = size - sizeof(struct persistent_ram_buffer);
498 
499 	return 0;
500 }
501 
persistent_ram_post_init(struct persistent_ram_zone * prz,u32 sig,struct persistent_ram_ecc_info * ecc_info)502 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,
503 				    struct persistent_ram_ecc_info *ecc_info)
504 {
505 	int ret;
506 	bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD);
507 
508 	ret = persistent_ram_init_ecc(prz, ecc_info);
509 	if (ret) {
510 		pr_warn("ECC failed %s\n", prz->label);
511 		return ret;
512 	}
513 
514 	sig ^= PERSISTENT_RAM_SIG;
515 
516 	if (prz->buffer->sig == sig) {
517 		if (buffer_size(prz) == 0) {
518 			pr_debug("found existing empty buffer\n");
519 			return 0;
520 		}
521 
522 		if (buffer_size(prz) > prz->buffer_size ||
523 		    buffer_start(prz) > buffer_size(prz)) {
524 			pr_info("found existing invalid buffer, size %zu, start %zu\n",
525 				buffer_size(prz), buffer_start(prz));
526 			zap = true;
527 		} else {
528 			pr_debug("found existing buffer, size %zu, start %zu\n",
529 				 buffer_size(prz), buffer_start(prz));
530 			persistent_ram_save_old(prz);
531 		}
532 	} else {
533 		pr_debug("no valid data in buffer (sig = 0x%08x)\n",
534 			 prz->buffer->sig);
535 		prz->buffer->sig = sig;
536 		zap = true;
537 	}
538 
539 	/* Reset missing, invalid, or single-use memory area. */
540 	if (zap)
541 		persistent_ram_zap(prz);
542 
543 	return 0;
544 }
545 
persistent_ram_free(struct persistent_ram_zone * prz)546 void persistent_ram_free(struct persistent_ram_zone *prz)
547 {
548 	if (!prz)
549 		return;
550 
551 	if (prz->vaddr) {
552 		if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {
553 			/* We must vunmap() at page-granularity. */
554 			vunmap(prz->vaddr - offset_in_page(prz->paddr));
555 		} else {
556 			iounmap(prz->vaddr);
557 			release_mem_region(prz->paddr, prz->size);
558 		}
559 		prz->vaddr = NULL;
560 	}
561 	if (prz->rs_decoder) {
562 		free_rs(prz->rs_decoder);
563 		prz->rs_decoder = NULL;
564 	}
565 	kfree(prz->ecc_info.par);
566 	prz->ecc_info.par = NULL;
567 
568 	persistent_ram_free_old(prz);
569 	kfree(prz->label);
570 	kfree(prz);
571 }
572 
persistent_ram_new(phys_addr_t start,size_t size,u32 sig,struct persistent_ram_ecc_info * ecc_info,unsigned int memtype,u32 flags,char * label)573 struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size,
574 			u32 sig, struct persistent_ram_ecc_info *ecc_info,
575 			unsigned int memtype, u32 flags, char *label)
576 {
577 	struct persistent_ram_zone *prz;
578 	int ret = -ENOMEM;
579 
580 	prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);
581 	if (!prz) {
582 		pr_err("failed to allocate persistent ram zone\n");
583 		goto err;
584 	}
585 
586 	/* Initialize general buffer state. */
587 	raw_spin_lock_init(&prz->buffer_lock);
588 	prz->flags = flags;
589 	prz->label = kstrdup(label, GFP_KERNEL);
590 
591 	ret = persistent_ram_buffer_map(start, size, prz, memtype);
592 	if (ret)
593 		goto err;
594 
595 	ret = persistent_ram_post_init(prz, sig, ecc_info);
596 	if (ret)
597 		goto err;
598 
599 	pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n",
600 		prz->label, prz->size, (unsigned long long)prz->paddr,
601 		sizeof(*prz->buffer), prz->buffer_size,
602 		prz->size - sizeof(*prz->buffer) - prz->buffer_size,
603 		prz->ecc_info.ecc_size, prz->ecc_info.block_size);
604 
605 	return prz;
606 err:
607 	persistent_ram_free(prz);
608 	return ERR_PTR(ret);
609 }
610