1 // SPDX-License-Identifier: GPL-2.0-only
2 /* arch/sparc64/kernel/traps.c
3  *
4  * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net)
5  * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
6  */
7 
8 /*
9  * I like traps on v9, :))))
10  */
11 
12 #include <linux/extable.h>
13 #include <linux/sched/mm.h>
14 #include <linux/sched/debug.h>
15 #include <linux/linkage.h>
16 #include <linux/kernel.h>
17 #include <linux/signal.h>
18 #include <linux/smp.h>
19 #include <linux/mm.h>
20 #include <linux/init.h>
21 #include <linux/kallsyms.h>
22 #include <linux/kdebug.h>
23 #include <linux/ftrace.h>
24 #include <linux/reboot.h>
25 #include <linux/gfp.h>
26 #include <linux/context_tracking.h>
27 
28 #include <asm/smp.h>
29 #include <asm/delay.h>
30 #include <asm/ptrace.h>
31 #include <asm/oplib.h>
32 #include <asm/page.h>
33 #include <asm/unistd.h>
34 #include <linux/uaccess.h>
35 #include <asm/fpumacro.h>
36 #include <asm/lsu.h>
37 #include <asm/dcu.h>
38 #include <asm/estate.h>
39 #include <asm/chafsr.h>
40 #include <asm/sfafsr.h>
41 #include <asm/psrcompat.h>
42 #include <asm/processor.h>
43 #include <asm/timer.h>
44 #include <asm/head.h>
45 #include <asm/prom.h>
46 #include <asm/memctrl.h>
47 #include <asm/cacheflush.h>
48 #include <asm/setup.h>
49 
50 #include "entry.h"
51 #include "kernel.h"
52 #include "kstack.h"
53 
54 /* When an irrecoverable trap occurs at tl > 0, the trap entry
55  * code logs the trap state registers at every level in the trap
56  * stack.  It is found at (pt_regs + sizeof(pt_regs)) and the layout
57  * is as follows:
58  */
59 struct tl1_traplog {
60 	struct {
61 		unsigned long tstate;
62 		unsigned long tpc;
63 		unsigned long tnpc;
64 		unsigned long tt;
65 	} trapstack[4];
66 	unsigned long tl;
67 };
68 
dump_tl1_traplog(struct tl1_traplog * p)69 static void dump_tl1_traplog(struct tl1_traplog *p)
70 {
71 	int i, limit;
72 
73 	printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
74 	       "dumping track stack.\n", p->tl);
75 
76 	limit = (tlb_type == hypervisor) ? 2 : 4;
77 	for (i = 0; i < limit; i++) {
78 		printk(KERN_EMERG
79 		       "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
80 		       "TNPC[%016lx] TT[%lx]\n",
81 		       i + 1,
82 		       p->trapstack[i].tstate, p->trapstack[i].tpc,
83 		       p->trapstack[i].tnpc, p->trapstack[i].tt);
84 		printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
85 	}
86 }
87 
bad_trap(struct pt_regs * regs,long lvl)88 void bad_trap(struct pt_regs *regs, long lvl)
89 {
90 	char buffer[36];
91 
92 	if (notify_die(DIE_TRAP, "bad trap", regs,
93 		       0, lvl, SIGTRAP) == NOTIFY_STOP)
94 		return;
95 
96 	if (lvl < 0x100) {
97 		sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
98 		die_if_kernel(buffer, regs);
99 	}
100 
101 	lvl -= 0x100;
102 	if (regs->tstate & TSTATE_PRIV) {
103 		sprintf(buffer, "Kernel bad sw trap %lx", lvl);
104 		die_if_kernel(buffer, regs);
105 	}
106 	if (test_thread_flag(TIF_32BIT)) {
107 		regs->tpc &= 0xffffffff;
108 		regs->tnpc &= 0xffffffff;
109 	}
110 	force_sig_fault_trapno(SIGILL, ILL_ILLTRP,
111 			       (void __user *)regs->tpc, lvl);
112 }
113 
bad_trap_tl1(struct pt_regs * regs,long lvl)114 void bad_trap_tl1(struct pt_regs *regs, long lvl)
115 {
116 	char buffer[36];
117 
118 	if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
119 		       0, lvl, SIGTRAP) == NOTIFY_STOP)
120 		return;
121 
122 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
123 
124 	sprintf (buffer, "Bad trap %lx at tl>0", lvl);
125 	die_if_kernel (buffer, regs);
126 }
127 
128 #ifdef CONFIG_DEBUG_BUGVERBOSE
do_BUG(const char * file,int line)129 void do_BUG(const char *file, int line)
130 {
131 	bust_spinlocks(1);
132 	printk("kernel BUG at %s:%d!\n", file, line);
133 }
134 EXPORT_SYMBOL(do_BUG);
135 #endif
136 
137 static DEFINE_SPINLOCK(dimm_handler_lock);
138 static dimm_printer_t dimm_handler;
139 
sprintf_dimm(int synd_code,unsigned long paddr,char * buf,int buflen)140 static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen)
141 {
142 	unsigned long flags;
143 	int ret = -ENODEV;
144 
145 	spin_lock_irqsave(&dimm_handler_lock, flags);
146 	if (dimm_handler) {
147 		ret = dimm_handler(synd_code, paddr, buf, buflen);
148 	} else if (tlb_type == spitfire) {
149 		if (prom_getunumber(synd_code, paddr, buf, buflen) == -1)
150 			ret = -EINVAL;
151 		else
152 			ret = 0;
153 	} else
154 		ret = -ENODEV;
155 	spin_unlock_irqrestore(&dimm_handler_lock, flags);
156 
157 	return ret;
158 }
159 
register_dimm_printer(dimm_printer_t func)160 int register_dimm_printer(dimm_printer_t func)
161 {
162 	unsigned long flags;
163 	int ret = 0;
164 
165 	spin_lock_irqsave(&dimm_handler_lock, flags);
166 	if (!dimm_handler)
167 		dimm_handler = func;
168 	else
169 		ret = -EEXIST;
170 	spin_unlock_irqrestore(&dimm_handler_lock, flags);
171 
172 	return ret;
173 }
174 EXPORT_SYMBOL_GPL(register_dimm_printer);
175 
unregister_dimm_printer(dimm_printer_t func)176 void unregister_dimm_printer(dimm_printer_t func)
177 {
178 	unsigned long flags;
179 
180 	spin_lock_irqsave(&dimm_handler_lock, flags);
181 	if (dimm_handler == func)
182 		dimm_handler = NULL;
183 	spin_unlock_irqrestore(&dimm_handler_lock, flags);
184 }
185 EXPORT_SYMBOL_GPL(unregister_dimm_printer);
186 
spitfire_insn_access_exception(struct pt_regs * regs,unsigned long sfsr,unsigned long sfar)187 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
188 {
189 	enum ctx_state prev_state = exception_enter();
190 
191 	if (notify_die(DIE_TRAP, "instruction access exception", regs,
192 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
193 		goto out;
194 
195 	if (regs->tstate & TSTATE_PRIV) {
196 		printk("spitfire_insn_access_exception: SFSR[%016lx] "
197 		       "SFAR[%016lx], going.\n", sfsr, sfar);
198 		die_if_kernel("Iax", regs);
199 	}
200 	if (test_thread_flag(TIF_32BIT)) {
201 		regs->tpc &= 0xffffffff;
202 		regs->tnpc &= 0xffffffff;
203 	}
204 	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)regs->tpc);
205 out:
206 	exception_exit(prev_state);
207 }
208 
spitfire_insn_access_exception_tl1(struct pt_regs * regs,unsigned long sfsr,unsigned long sfar)209 void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
210 {
211 	if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
212 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
213 		return;
214 
215 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
216 	spitfire_insn_access_exception(regs, sfsr, sfar);
217 }
218 
sun4v_insn_access_exception(struct pt_regs * regs,unsigned long addr,unsigned long type_ctx)219 void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
220 {
221 	unsigned short type = (type_ctx >> 16);
222 	unsigned short ctx  = (type_ctx & 0xffff);
223 
224 	if (notify_die(DIE_TRAP, "instruction access exception", regs,
225 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
226 		return;
227 
228 	if (regs->tstate & TSTATE_PRIV) {
229 		printk("sun4v_insn_access_exception: ADDR[%016lx] "
230 		       "CTX[%04x] TYPE[%04x], going.\n",
231 		       addr, ctx, type);
232 		die_if_kernel("Iax", regs);
233 	}
234 
235 	if (test_thread_flag(TIF_32BIT)) {
236 		regs->tpc &= 0xffffffff;
237 		regs->tnpc &= 0xffffffff;
238 	}
239 	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *) addr);
240 }
241 
sun4v_insn_access_exception_tl1(struct pt_regs * regs,unsigned long addr,unsigned long type_ctx)242 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
243 {
244 	if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
245 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
246 		return;
247 
248 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
249 	sun4v_insn_access_exception(regs, addr, type_ctx);
250 }
251 
is_no_fault_exception(struct pt_regs * regs)252 bool is_no_fault_exception(struct pt_regs *regs)
253 {
254 	unsigned char asi;
255 	u32 insn;
256 
257 	if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT)
258 		return false;
259 
260 	/*
261 	 * Must do a little instruction decoding here in order to
262 	 * decide on a course of action. The bits of interest are:
263 	 *  insn[31:30] = op, where 3 indicates the load/store group
264 	 *  insn[24:19] = op3, which identifies individual opcodes
265 	 *  insn[13] indicates an immediate offset
266 	 *  op3[4]=1 identifies alternate space instructions
267 	 *  op3[5:4]=3 identifies floating point instructions
268 	 *  op3[2]=1 identifies stores
269 	 * See "Opcode Maps" in the appendix of any Sparc V9
270 	 * architecture spec for full details.
271 	 */
272 	if ((insn & 0xc0800000) == 0xc0800000) {    /* op=3, op3[4]=1   */
273 		if (insn & 0x2000)		    /* immediate offset */
274 			asi = (regs->tstate >> 24); /* saved %asi       */
275 		else
276 			asi = (insn >> 5);	    /* immediate asi    */
277 		if ((asi & 0xf6) == ASI_PNF) {
278 			if (insn & 0x200000)        /* op3[2], stores   */
279 				return false;
280 			if (insn & 0x1000000)       /* op3[5:4]=3 (fp)  */
281 				handle_ldf_stq(insn, regs);
282 			else
283 				handle_ld_nf(insn, regs);
284 			return true;
285 		}
286 	}
287 	return false;
288 }
289 
spitfire_data_access_exception(struct pt_regs * regs,unsigned long sfsr,unsigned long sfar)290 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
291 {
292 	enum ctx_state prev_state = exception_enter();
293 
294 	if (notify_die(DIE_TRAP, "data access exception", regs,
295 		       0, 0x30, SIGTRAP) == NOTIFY_STOP)
296 		goto out;
297 
298 	if (regs->tstate & TSTATE_PRIV) {
299 		/* Test if this comes from uaccess places. */
300 		const struct exception_table_entry *entry;
301 
302 		entry = search_exception_tables(regs->tpc);
303 		if (entry) {
304 			/* Ouch, somebody is trying VM hole tricks on us... */
305 #ifdef DEBUG_EXCEPTIONS
306 			printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
307 			printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
308 			       regs->tpc, entry->fixup);
309 #endif
310 			regs->tpc = entry->fixup;
311 			regs->tnpc = regs->tpc + 4;
312 			goto out;
313 		}
314 		/* Shit... */
315 		printk("spitfire_data_access_exception: SFSR[%016lx] "
316 		       "SFAR[%016lx], going.\n", sfsr, sfar);
317 		die_if_kernel("Dax", regs);
318 	}
319 
320 	if (is_no_fault_exception(regs))
321 		return;
322 
323 	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)sfar);
324 out:
325 	exception_exit(prev_state);
326 }
327 
spitfire_data_access_exception_tl1(struct pt_regs * regs,unsigned long sfsr,unsigned long sfar)328 void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
329 {
330 	if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
331 		       0, 0x30, SIGTRAP) == NOTIFY_STOP)
332 		return;
333 
334 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
335 	spitfire_data_access_exception(regs, sfsr, sfar);
336 }
337 
sun4v_data_access_exception(struct pt_regs * regs,unsigned long addr,unsigned long type_ctx)338 void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
339 {
340 	unsigned short type = (type_ctx >> 16);
341 	unsigned short ctx  = (type_ctx & 0xffff);
342 
343 	if (notify_die(DIE_TRAP, "data access exception", regs,
344 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
345 		return;
346 
347 	if (regs->tstate & TSTATE_PRIV) {
348 		/* Test if this comes from uaccess places. */
349 		const struct exception_table_entry *entry;
350 
351 		entry = search_exception_tables(regs->tpc);
352 		if (entry) {
353 			/* Ouch, somebody is trying VM hole tricks on us... */
354 #ifdef DEBUG_EXCEPTIONS
355 			printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
356 			printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
357 			       regs->tpc, entry->fixup);
358 #endif
359 			regs->tpc = entry->fixup;
360 			regs->tnpc = regs->tpc + 4;
361 			return;
362 		}
363 		printk("sun4v_data_access_exception: ADDR[%016lx] "
364 		       "CTX[%04x] TYPE[%04x], going.\n",
365 		       addr, ctx, type);
366 		die_if_kernel("Dax", regs);
367 	}
368 
369 	if (test_thread_flag(TIF_32BIT)) {
370 		regs->tpc &= 0xffffffff;
371 		regs->tnpc &= 0xffffffff;
372 	}
373 	if (is_no_fault_exception(regs))
374 		return;
375 
376 	/* MCD (Memory Corruption Detection) disabled trap (TT=0x19) in HV
377 	 * is vectored thorugh data access exception trap with fault type
378 	 * set to HV_FAULT_TYPE_MCD_DIS. Check for MCD disabled trap.
379 	 * Accessing an address with invalid ASI for the address, for
380 	 * example setting an ADI tag on an address with ASI_MCD_PRIMARY
381 	 * when TTE.mcd is not set for the VA, is also vectored into
382 	 * kerbel by HV as data access exception with fault type set to
383 	 * HV_FAULT_TYPE_INV_ASI.
384 	 */
385 	switch (type) {
386 	case HV_FAULT_TYPE_INV_ASI:
387 		force_sig_fault(SIGILL, ILL_ILLADR, (void __user *)addr);
388 		break;
389 	case HV_FAULT_TYPE_MCD_DIS:
390 		force_sig_fault(SIGSEGV, SEGV_ACCADI, (void __user *)addr);
391 		break;
392 	default:
393 		force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)addr);
394 		break;
395 	}
396 }
397 
sun4v_data_access_exception_tl1(struct pt_regs * regs,unsigned long addr,unsigned long type_ctx)398 void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
399 {
400 	if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
401 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
402 		return;
403 
404 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
405 	sun4v_data_access_exception(regs, addr, type_ctx);
406 }
407 
408 #ifdef CONFIG_PCI
409 #include "pci_impl.h"
410 #endif
411 
412 /* When access exceptions happen, we must do this. */
spitfire_clean_and_reenable_l1_caches(void)413 static void spitfire_clean_and_reenable_l1_caches(void)
414 {
415 	unsigned long va;
416 
417 	if (tlb_type != spitfire)
418 		BUG();
419 
420 	/* Clean 'em. */
421 	for (va =  0; va < (PAGE_SIZE << 1); va += 32) {
422 		spitfire_put_icache_tag(va, 0x0);
423 		spitfire_put_dcache_tag(va, 0x0);
424 	}
425 
426 	/* Re-enable in LSU. */
427 	__asm__ __volatile__("flush %%g6\n\t"
428 			     "membar #Sync\n\t"
429 			     "stxa %0, [%%g0] %1\n\t"
430 			     "membar #Sync"
431 			     : /* no outputs */
432 			     : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
433 				    LSU_CONTROL_IM | LSU_CONTROL_DM),
434 			     "i" (ASI_LSU_CONTROL)
435 			     : "memory");
436 }
437 
spitfire_enable_estate_errors(void)438 static void spitfire_enable_estate_errors(void)
439 {
440 	__asm__ __volatile__("stxa	%0, [%%g0] %1\n\t"
441 			     "membar	#Sync"
442 			     : /* no outputs */
443 			     : "r" (ESTATE_ERR_ALL),
444 			       "i" (ASI_ESTATE_ERROR_EN));
445 }
446 
447 static char ecc_syndrome_table[] = {
448 	0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
449 	0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
450 	0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
451 	0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
452 	0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
453 	0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
454 	0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
455 	0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
456 	0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
457 	0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
458 	0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
459 	0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
460 	0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
461 	0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
462 	0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
463 	0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
464 	0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
465 	0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
466 	0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
467 	0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
468 	0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
469 	0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
470 	0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
471 	0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
472 	0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
473 	0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
474 	0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
475 	0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
476 	0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
477 	0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
478 	0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
479 	0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
480 };
481 
482 static char *syndrome_unknown = "<Unknown>";
483 
spitfire_log_udb_syndrome(unsigned long afar,unsigned long udbh,unsigned long udbl,unsigned long bit)484 static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
485 {
486 	unsigned short scode;
487 	char memmod_str[64], *p;
488 
489 	if (udbl & bit) {
490 		scode = ecc_syndrome_table[udbl & 0xff];
491 		if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
492 			p = syndrome_unknown;
493 		else
494 			p = memmod_str;
495 		printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
496 		       "Memory Module \"%s\"\n",
497 		       smp_processor_id(), scode, p);
498 	}
499 
500 	if (udbh & bit) {
501 		scode = ecc_syndrome_table[udbh & 0xff];
502 		if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
503 			p = syndrome_unknown;
504 		else
505 			p = memmod_str;
506 		printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
507 		       "Memory Module \"%s\"\n",
508 		       smp_processor_id(), scode, p);
509 	}
510 
511 }
512 
spitfire_cee_log(unsigned long afsr,unsigned long afar,unsigned long udbh,unsigned long udbl,int tl1,struct pt_regs * regs)513 static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
514 {
515 
516 	printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
517 	       "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
518 	       smp_processor_id(), afsr, afar, udbl, udbh, tl1);
519 
520 	spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
521 
522 	/* We always log it, even if someone is listening for this
523 	 * trap.
524 	 */
525 	notify_die(DIE_TRAP, "Correctable ECC Error", regs,
526 		   0, TRAP_TYPE_CEE, SIGTRAP);
527 
528 	/* The Correctable ECC Error trap does not disable I/D caches.  So
529 	 * we only have to restore the ESTATE Error Enable register.
530 	 */
531 	spitfire_enable_estate_errors();
532 }
533 
spitfire_ue_log(unsigned long afsr,unsigned long afar,unsigned long udbh,unsigned long udbl,unsigned long tt,int tl1,struct pt_regs * regs)534 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
535 {
536 	printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
537 	       "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
538 	       smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
539 
540 	/* XXX add more human friendly logging of the error status
541 	 * XXX as is implemented for cheetah
542 	 */
543 
544 	spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
545 
546 	/* We always log it, even if someone is listening for this
547 	 * trap.
548 	 */
549 	notify_die(DIE_TRAP, "Uncorrectable Error", regs,
550 		   0, tt, SIGTRAP);
551 
552 	if (regs->tstate & TSTATE_PRIV) {
553 		if (tl1)
554 			dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
555 		die_if_kernel("UE", regs);
556 	}
557 
558 	/* XXX need more intelligent processing here, such as is implemented
559 	 * XXX for cheetah errors, in fact if the E-cache still holds the
560 	 * XXX line with bad parity this will loop
561 	 */
562 
563 	spitfire_clean_and_reenable_l1_caches();
564 	spitfire_enable_estate_errors();
565 
566 	if (test_thread_flag(TIF_32BIT)) {
567 		regs->tpc &= 0xffffffff;
568 		regs->tnpc &= 0xffffffff;
569 	}
570 	force_sig_fault(SIGBUS, BUS_OBJERR, (void *)0);
571 }
572 
spitfire_access_error(struct pt_regs * regs,unsigned long status_encoded,unsigned long afar)573 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
574 {
575 	unsigned long afsr, tt, udbh, udbl;
576 	int tl1;
577 
578 	afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
579 	tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
580 	tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
581 	udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
582 	udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
583 
584 #ifdef CONFIG_PCI
585 	if (tt == TRAP_TYPE_DAE &&
586 	    pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
587 		spitfire_clean_and_reenable_l1_caches();
588 		spitfire_enable_estate_errors();
589 
590 		pci_poke_faulted = 1;
591 		regs->tnpc = regs->tpc + 4;
592 		return;
593 	}
594 #endif
595 
596 	if (afsr & SFAFSR_UE)
597 		spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
598 
599 	if (tt == TRAP_TYPE_CEE) {
600 		/* Handle the case where we took a CEE trap, but ACK'd
601 		 * only the UE state in the UDB error registers.
602 		 */
603 		if (afsr & SFAFSR_UE) {
604 			if (udbh & UDBE_CE) {
605 				__asm__ __volatile__(
606 					"stxa	%0, [%1] %2\n\t"
607 					"membar	#Sync"
608 					: /* no outputs */
609 					: "r" (udbh & UDBE_CE),
610 					  "r" (0x0), "i" (ASI_UDB_ERROR_W));
611 			}
612 			if (udbl & UDBE_CE) {
613 				__asm__ __volatile__(
614 					"stxa	%0, [%1] %2\n\t"
615 					"membar	#Sync"
616 					: /* no outputs */
617 					: "r" (udbl & UDBE_CE),
618 					  "r" (0x18), "i" (ASI_UDB_ERROR_W));
619 			}
620 		}
621 
622 		spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
623 	}
624 }
625 
626 int cheetah_pcache_forced_on;
627 
cheetah_enable_pcache(void)628 void cheetah_enable_pcache(void)
629 {
630 	unsigned long dcr;
631 
632 	printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
633 	       smp_processor_id());
634 
635 	__asm__ __volatile__("ldxa [%%g0] %1, %0"
636 			     : "=r" (dcr)
637 			     : "i" (ASI_DCU_CONTROL_REG));
638 	dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
639 	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
640 			     "membar #Sync"
641 			     : /* no outputs */
642 			     : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
643 }
644 
645 /* Cheetah error trap handling. */
646 static unsigned long ecache_flush_physbase;
647 static unsigned long ecache_flush_linesize;
648 static unsigned long ecache_flush_size;
649 
650 /* This table is ordered in priority of errors and matches the
651  * AFAR overwrite policy as well.
652  */
653 
654 struct afsr_error_table {
655 	unsigned long mask;
656 	const char *name;
657 };
658 
659 static const char CHAFSR_PERR_msg[] =
660 	"System interface protocol error";
661 static const char CHAFSR_IERR_msg[] =
662 	"Internal processor error";
663 static const char CHAFSR_ISAP_msg[] =
664 	"System request parity error on incoming address";
665 static const char CHAFSR_UCU_msg[] =
666 	"Uncorrectable E-cache ECC error for ifetch/data";
667 static const char CHAFSR_UCC_msg[] =
668 	"SW Correctable E-cache ECC error for ifetch/data";
669 static const char CHAFSR_UE_msg[] =
670 	"Uncorrectable system bus data ECC error for read";
671 static const char CHAFSR_EDU_msg[] =
672 	"Uncorrectable E-cache ECC error for stmerge/blkld";
673 static const char CHAFSR_EMU_msg[] =
674 	"Uncorrectable system bus MTAG error";
675 static const char CHAFSR_WDU_msg[] =
676 	"Uncorrectable E-cache ECC error for writeback";
677 static const char CHAFSR_CPU_msg[] =
678 	"Uncorrectable ECC error for copyout";
679 static const char CHAFSR_CE_msg[] =
680 	"HW corrected system bus data ECC error for read";
681 static const char CHAFSR_EDC_msg[] =
682 	"HW corrected E-cache ECC error for stmerge/blkld";
683 static const char CHAFSR_EMC_msg[] =
684 	"HW corrected system bus MTAG ECC error";
685 static const char CHAFSR_WDC_msg[] =
686 	"HW corrected E-cache ECC error for writeback";
687 static const char CHAFSR_CPC_msg[] =
688 	"HW corrected ECC error for copyout";
689 static const char CHAFSR_TO_msg[] =
690 	"Unmapped error from system bus";
691 static const char CHAFSR_BERR_msg[] =
692 	"Bus error response from system bus";
693 static const char CHAFSR_IVC_msg[] =
694 	"HW corrected system bus data ECC error for ivec read";
695 static const char CHAFSR_IVU_msg[] =
696 	"Uncorrectable system bus data ECC error for ivec read";
697 static struct afsr_error_table __cheetah_error_table[] = {
698 	{	CHAFSR_PERR,	CHAFSR_PERR_msg		},
699 	{	CHAFSR_IERR,	CHAFSR_IERR_msg		},
700 	{	CHAFSR_ISAP,	CHAFSR_ISAP_msg		},
701 	{	CHAFSR_UCU,	CHAFSR_UCU_msg		},
702 	{	CHAFSR_UCC,	CHAFSR_UCC_msg		},
703 	{	CHAFSR_UE,	CHAFSR_UE_msg		},
704 	{	CHAFSR_EDU,	CHAFSR_EDU_msg		},
705 	{	CHAFSR_EMU,	CHAFSR_EMU_msg		},
706 	{	CHAFSR_WDU,	CHAFSR_WDU_msg		},
707 	{	CHAFSR_CPU,	CHAFSR_CPU_msg		},
708 	{	CHAFSR_CE,	CHAFSR_CE_msg		},
709 	{	CHAFSR_EDC,	CHAFSR_EDC_msg		},
710 	{	CHAFSR_EMC,	CHAFSR_EMC_msg		},
711 	{	CHAFSR_WDC,	CHAFSR_WDC_msg		},
712 	{	CHAFSR_CPC,	CHAFSR_CPC_msg		},
713 	{	CHAFSR_TO,	CHAFSR_TO_msg		},
714 	{	CHAFSR_BERR,	CHAFSR_BERR_msg		},
715 	/* These two do not update the AFAR. */
716 	{	CHAFSR_IVC,	CHAFSR_IVC_msg		},
717 	{	CHAFSR_IVU,	CHAFSR_IVU_msg		},
718 	{	0,		NULL			},
719 };
720 static const char CHPAFSR_DTO_msg[] =
721 	"System bus unmapped error for prefetch/storequeue-read";
722 static const char CHPAFSR_DBERR_msg[] =
723 	"System bus error for prefetch/storequeue-read";
724 static const char CHPAFSR_THCE_msg[] =
725 	"Hardware corrected E-cache Tag ECC error";
726 static const char CHPAFSR_TSCE_msg[] =
727 	"SW handled correctable E-cache Tag ECC error";
728 static const char CHPAFSR_TUE_msg[] =
729 	"Uncorrectable E-cache Tag ECC error";
730 static const char CHPAFSR_DUE_msg[] =
731 	"System bus uncorrectable data ECC error due to prefetch/store-fill";
732 static struct afsr_error_table __cheetah_plus_error_table[] = {
733 	{	CHAFSR_PERR,	CHAFSR_PERR_msg		},
734 	{	CHAFSR_IERR,	CHAFSR_IERR_msg		},
735 	{	CHAFSR_ISAP,	CHAFSR_ISAP_msg		},
736 	{	CHAFSR_UCU,	CHAFSR_UCU_msg		},
737 	{	CHAFSR_UCC,	CHAFSR_UCC_msg		},
738 	{	CHAFSR_UE,	CHAFSR_UE_msg		},
739 	{	CHAFSR_EDU,	CHAFSR_EDU_msg		},
740 	{	CHAFSR_EMU,	CHAFSR_EMU_msg		},
741 	{	CHAFSR_WDU,	CHAFSR_WDU_msg		},
742 	{	CHAFSR_CPU,	CHAFSR_CPU_msg		},
743 	{	CHAFSR_CE,	CHAFSR_CE_msg		},
744 	{	CHAFSR_EDC,	CHAFSR_EDC_msg		},
745 	{	CHAFSR_EMC,	CHAFSR_EMC_msg		},
746 	{	CHAFSR_WDC,	CHAFSR_WDC_msg		},
747 	{	CHAFSR_CPC,	CHAFSR_CPC_msg		},
748 	{	CHAFSR_TO,	CHAFSR_TO_msg		},
749 	{	CHAFSR_BERR,	CHAFSR_BERR_msg		},
750 	{	CHPAFSR_DTO,	CHPAFSR_DTO_msg		},
751 	{	CHPAFSR_DBERR,	CHPAFSR_DBERR_msg	},
752 	{	CHPAFSR_THCE,	CHPAFSR_THCE_msg	},
753 	{	CHPAFSR_TSCE,	CHPAFSR_TSCE_msg	},
754 	{	CHPAFSR_TUE,	CHPAFSR_TUE_msg		},
755 	{	CHPAFSR_DUE,	CHPAFSR_DUE_msg		},
756 	/* These two do not update the AFAR. */
757 	{	CHAFSR_IVC,	CHAFSR_IVC_msg		},
758 	{	CHAFSR_IVU,	CHAFSR_IVU_msg		},
759 	{	0,		NULL			},
760 };
761 static const char JPAFSR_JETO_msg[] =
762 	"System interface protocol error, hw timeout caused";
763 static const char JPAFSR_SCE_msg[] =
764 	"Parity error on system snoop results";
765 static const char JPAFSR_JEIC_msg[] =
766 	"System interface protocol error, illegal command detected";
767 static const char JPAFSR_JEIT_msg[] =
768 	"System interface protocol error, illegal ADTYPE detected";
769 static const char JPAFSR_OM_msg[] =
770 	"Out of range memory error has occurred";
771 static const char JPAFSR_ETP_msg[] =
772 	"Parity error on L2 cache tag SRAM";
773 static const char JPAFSR_UMS_msg[] =
774 	"Error due to unsupported store";
775 static const char JPAFSR_RUE_msg[] =
776 	"Uncorrectable ECC error from remote cache/memory";
777 static const char JPAFSR_RCE_msg[] =
778 	"Correctable ECC error from remote cache/memory";
779 static const char JPAFSR_BP_msg[] =
780 	"JBUS parity error on returned read data";
781 static const char JPAFSR_WBP_msg[] =
782 	"JBUS parity error on data for writeback or block store";
783 static const char JPAFSR_FRC_msg[] =
784 	"Foreign read to DRAM incurring correctable ECC error";
785 static const char JPAFSR_FRU_msg[] =
786 	"Foreign read to DRAM incurring uncorrectable ECC error";
787 static struct afsr_error_table __jalapeno_error_table[] = {
788 	{	JPAFSR_JETO,	JPAFSR_JETO_msg		},
789 	{	JPAFSR_SCE,	JPAFSR_SCE_msg		},
790 	{	JPAFSR_JEIC,	JPAFSR_JEIC_msg		},
791 	{	JPAFSR_JEIT,	JPAFSR_JEIT_msg		},
792 	{	CHAFSR_PERR,	CHAFSR_PERR_msg		},
793 	{	CHAFSR_IERR,	CHAFSR_IERR_msg		},
794 	{	CHAFSR_ISAP,	CHAFSR_ISAP_msg		},
795 	{	CHAFSR_UCU,	CHAFSR_UCU_msg		},
796 	{	CHAFSR_UCC,	CHAFSR_UCC_msg		},
797 	{	CHAFSR_UE,	CHAFSR_UE_msg		},
798 	{	CHAFSR_EDU,	CHAFSR_EDU_msg		},
799 	{	JPAFSR_OM,	JPAFSR_OM_msg		},
800 	{	CHAFSR_WDU,	CHAFSR_WDU_msg		},
801 	{	CHAFSR_CPU,	CHAFSR_CPU_msg		},
802 	{	CHAFSR_CE,	CHAFSR_CE_msg		},
803 	{	CHAFSR_EDC,	CHAFSR_EDC_msg		},
804 	{	JPAFSR_ETP,	JPAFSR_ETP_msg		},
805 	{	CHAFSR_WDC,	CHAFSR_WDC_msg		},
806 	{	CHAFSR_CPC,	CHAFSR_CPC_msg		},
807 	{	CHAFSR_TO,	CHAFSR_TO_msg		},
808 	{	CHAFSR_BERR,	CHAFSR_BERR_msg		},
809 	{	JPAFSR_UMS,	JPAFSR_UMS_msg		},
810 	{	JPAFSR_RUE,	JPAFSR_RUE_msg		},
811 	{	JPAFSR_RCE,	JPAFSR_RCE_msg		},
812 	{	JPAFSR_BP,	JPAFSR_BP_msg		},
813 	{	JPAFSR_WBP,	JPAFSR_WBP_msg		},
814 	{	JPAFSR_FRC,	JPAFSR_FRC_msg		},
815 	{	JPAFSR_FRU,	JPAFSR_FRU_msg		},
816 	/* These two do not update the AFAR. */
817 	{	CHAFSR_IVU,	CHAFSR_IVU_msg		},
818 	{	0,		NULL			},
819 };
820 static struct afsr_error_table *cheetah_error_table;
821 static unsigned long cheetah_afsr_errors;
822 
823 struct cheetah_err_info *cheetah_error_log;
824 
cheetah_get_error_log(unsigned long afsr)825 static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
826 {
827 	struct cheetah_err_info *p;
828 	int cpu = smp_processor_id();
829 
830 	if (!cheetah_error_log)
831 		return NULL;
832 
833 	p = cheetah_error_log + (cpu * 2);
834 	if ((afsr & CHAFSR_TL1) != 0UL)
835 		p++;
836 
837 	return p;
838 }
839 
840 extern unsigned int tl0_icpe[], tl1_icpe[];
841 extern unsigned int tl0_dcpe[], tl1_dcpe[];
842 extern unsigned int tl0_fecc[], tl1_fecc[];
843 extern unsigned int tl0_cee[], tl1_cee[];
844 extern unsigned int tl0_iae[], tl1_iae[];
845 extern unsigned int tl0_dae[], tl1_dae[];
846 extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
847 extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
848 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
849 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
850 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
851 
cheetah_ecache_flush_init(void)852 void __init cheetah_ecache_flush_init(void)
853 {
854 	unsigned long largest_size, smallest_linesize, order, ver;
855 	int i, sz;
856 
857 	/* Scan all cpu device tree nodes, note two values:
858 	 * 1) largest E-cache size
859 	 * 2) smallest E-cache line size
860 	 */
861 	largest_size = 0UL;
862 	smallest_linesize = ~0UL;
863 
864 	for (i = 0; i < NR_CPUS; i++) {
865 		unsigned long val;
866 
867 		val = cpu_data(i).ecache_size;
868 		if (!val)
869 			continue;
870 
871 		if (val > largest_size)
872 			largest_size = val;
873 
874 		val = cpu_data(i).ecache_line_size;
875 		if (val < smallest_linesize)
876 			smallest_linesize = val;
877 
878 	}
879 
880 	if (largest_size == 0UL || smallest_linesize == ~0UL) {
881 		prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
882 			    "parameters.\n");
883 		prom_halt();
884 	}
885 
886 	ecache_flush_size = (2 * largest_size);
887 	ecache_flush_linesize = smallest_linesize;
888 
889 	ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
890 
891 	if (ecache_flush_physbase == ~0UL) {
892 		prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
893 			    "contiguous physical memory.\n",
894 			    ecache_flush_size);
895 		prom_halt();
896 	}
897 
898 	/* Now allocate error trap reporting scoreboard. */
899 	sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
900 	for (order = 0; order <= MAX_ORDER; order++) {
901 		if ((PAGE_SIZE << order) >= sz)
902 			break;
903 	}
904 	cheetah_error_log = (struct cheetah_err_info *)
905 		__get_free_pages(GFP_KERNEL, order);
906 	if (!cheetah_error_log) {
907 		prom_printf("cheetah_ecache_flush_init: Failed to allocate "
908 			    "error logging scoreboard (%d bytes).\n", sz);
909 		prom_halt();
910 	}
911 	memset(cheetah_error_log, 0, PAGE_SIZE << order);
912 
913 	/* Mark all AFSRs as invalid so that the trap handler will
914 	 * log new new information there.
915 	 */
916 	for (i = 0; i < 2 * NR_CPUS; i++)
917 		cheetah_error_log[i].afsr = CHAFSR_INVALID;
918 
919 	__asm__ ("rdpr %%ver, %0" : "=r" (ver));
920 	if ((ver >> 32) == __JALAPENO_ID ||
921 	    (ver >> 32) == __SERRANO_ID) {
922 		cheetah_error_table = &__jalapeno_error_table[0];
923 		cheetah_afsr_errors = JPAFSR_ERRORS;
924 	} else if ((ver >> 32) == 0x003e0015) {
925 		cheetah_error_table = &__cheetah_plus_error_table[0];
926 		cheetah_afsr_errors = CHPAFSR_ERRORS;
927 	} else {
928 		cheetah_error_table = &__cheetah_error_table[0];
929 		cheetah_afsr_errors = CHAFSR_ERRORS;
930 	}
931 
932 	/* Now patch trap tables. */
933 	memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
934 	memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
935 	memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
936 	memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
937 	memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
938 	memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
939 	memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
940 	memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
941 	if (tlb_type == cheetah_plus) {
942 		memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
943 		memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
944 		memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
945 		memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
946 	}
947 	flushi(PAGE_OFFSET);
948 }
949 
cheetah_flush_ecache(void)950 static void cheetah_flush_ecache(void)
951 {
952 	unsigned long flush_base = ecache_flush_physbase;
953 	unsigned long flush_linesize = ecache_flush_linesize;
954 	unsigned long flush_size = ecache_flush_size;
955 
956 	__asm__ __volatile__("1: subcc	%0, %4, %0\n\t"
957 			     "   bne,pt	%%xcc, 1b\n\t"
958 			     "    ldxa	[%2 + %0] %3, %%g0\n\t"
959 			     : "=&r" (flush_size)
960 			     : "0" (flush_size), "r" (flush_base),
961 			       "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
962 }
963 
cheetah_flush_ecache_line(unsigned long physaddr)964 static void cheetah_flush_ecache_line(unsigned long physaddr)
965 {
966 	unsigned long alias;
967 
968 	physaddr &= ~(8UL - 1UL);
969 	physaddr = (ecache_flush_physbase +
970 		    (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
971 	alias = physaddr + (ecache_flush_size >> 1UL);
972 	__asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
973 			     "ldxa [%1] %2, %%g0\n\t"
974 			     "membar #Sync"
975 			     : /* no outputs */
976 			     : "r" (physaddr), "r" (alias),
977 			       "i" (ASI_PHYS_USE_EC));
978 }
979 
980 /* Unfortunately, the diagnostic access to the I-cache tags we need to
981  * use to clear the thing interferes with I-cache coherency transactions.
982  *
983  * So we must only flush the I-cache when it is disabled.
984  */
__cheetah_flush_icache(void)985 static void __cheetah_flush_icache(void)
986 {
987 	unsigned int icache_size, icache_line_size;
988 	unsigned long addr;
989 
990 	icache_size = local_cpu_data().icache_size;
991 	icache_line_size = local_cpu_data().icache_line_size;
992 
993 	/* Clear the valid bits in all the tags. */
994 	for (addr = 0; addr < icache_size; addr += icache_line_size) {
995 		__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
996 				     "membar #Sync"
997 				     : /* no outputs */
998 				     : "r" (addr | (2 << 3)),
999 				       "i" (ASI_IC_TAG));
1000 	}
1001 }
1002 
cheetah_flush_icache(void)1003 static void cheetah_flush_icache(void)
1004 {
1005 	unsigned long dcu_save;
1006 
1007 	/* Save current DCU, disable I-cache. */
1008 	__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1009 			     "or %0, %2, %%g1\n\t"
1010 			     "stxa %%g1, [%%g0] %1\n\t"
1011 			     "membar #Sync"
1012 			     : "=r" (dcu_save)
1013 			     : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
1014 			     : "g1");
1015 
1016 	__cheetah_flush_icache();
1017 
1018 	/* Restore DCU register */
1019 	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1020 			     "membar #Sync"
1021 			     : /* no outputs */
1022 			     : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
1023 }
1024 
cheetah_flush_dcache(void)1025 static void cheetah_flush_dcache(void)
1026 {
1027 	unsigned int dcache_size, dcache_line_size;
1028 	unsigned long addr;
1029 
1030 	dcache_size = local_cpu_data().dcache_size;
1031 	dcache_line_size = local_cpu_data().dcache_line_size;
1032 
1033 	for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1034 		__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1035 				     "membar #Sync"
1036 				     : /* no outputs */
1037 				     : "r" (addr), "i" (ASI_DCACHE_TAG));
1038 	}
1039 }
1040 
1041 /* In order to make the even parity correct we must do two things.
1042  * First, we clear DC_data_parity and set DC_utag to an appropriate value.
1043  * Next, we clear out all 32-bytes of data for that line.  Data of
1044  * all-zero + tag parity value of zero == correct parity.
1045  */
cheetah_plus_zap_dcache_parity(void)1046 static void cheetah_plus_zap_dcache_parity(void)
1047 {
1048 	unsigned int dcache_size, dcache_line_size;
1049 	unsigned long addr;
1050 
1051 	dcache_size = local_cpu_data().dcache_size;
1052 	dcache_line_size = local_cpu_data().dcache_line_size;
1053 
1054 	for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1055 		unsigned long tag = (addr >> 14);
1056 		unsigned long line;
1057 
1058 		__asm__ __volatile__("membar	#Sync\n\t"
1059 				     "stxa	%0, [%1] %2\n\t"
1060 				     "membar	#Sync"
1061 				     : /* no outputs */
1062 				     : "r" (tag), "r" (addr),
1063 				       "i" (ASI_DCACHE_UTAG));
1064 		for (line = addr; line < addr + dcache_line_size; line += 8)
1065 			__asm__ __volatile__("membar	#Sync\n\t"
1066 					     "stxa	%%g0, [%0] %1\n\t"
1067 					     "membar	#Sync"
1068 					     : /* no outputs */
1069 					     : "r" (line),
1070 					       "i" (ASI_DCACHE_DATA));
1071 	}
1072 }
1073 
1074 /* Conversion tables used to frob Cheetah AFSR syndrome values into
1075  * something palatable to the memory controller driver get_unumber
1076  * routine.
1077  */
1078 #define MT0	137
1079 #define MT1	138
1080 #define MT2	139
1081 #define NONE	254
1082 #define MTC0	140
1083 #define MTC1	141
1084 #define MTC2	142
1085 #define MTC3	143
1086 #define C0	128
1087 #define C1	129
1088 #define C2	130
1089 #define C3	131
1090 #define C4	132
1091 #define C5	133
1092 #define C6	134
1093 #define C7	135
1094 #define C8	136
1095 #define M2	144
1096 #define M3	145
1097 #define M4	146
1098 #define M	147
1099 static unsigned char cheetah_ecc_syntab[] = {
1100 /*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1101 /*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1102 /*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1103 /*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1104 /*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1105 /*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1106 /*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1107 /*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1108 /*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1109 /*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1110 /*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1111 /*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1112 /*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1113 /*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1114 /*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1115 /*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1116 /*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1117 /*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1118 /*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1119 /*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1120 /*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1121 /*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1122 /*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1123 /*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1124 /*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1125 /*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1126 /*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1127 /*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1128 /*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1129 /*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1130 /*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1131 /*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1132 };
1133 static unsigned char cheetah_mtag_syntab[] = {
1134        NONE, MTC0,
1135        MTC1, NONE,
1136        MTC2, NONE,
1137        NONE, MT0,
1138        MTC3, NONE,
1139        NONE, MT1,
1140        NONE, MT2,
1141        NONE, NONE
1142 };
1143 
1144 /* Return the highest priority error conditon mentioned. */
cheetah_get_hipri(unsigned long afsr)1145 static inline unsigned long cheetah_get_hipri(unsigned long afsr)
1146 {
1147 	unsigned long tmp = 0;
1148 	int i;
1149 
1150 	for (i = 0; cheetah_error_table[i].mask; i++) {
1151 		if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1152 			return tmp;
1153 	}
1154 	return tmp;
1155 }
1156 
cheetah_get_string(unsigned long bit)1157 static const char *cheetah_get_string(unsigned long bit)
1158 {
1159 	int i;
1160 
1161 	for (i = 0; cheetah_error_table[i].mask; i++) {
1162 		if ((bit & cheetah_error_table[i].mask) != 0UL)
1163 			return cheetah_error_table[i].name;
1164 	}
1165 	return "???";
1166 }
1167 
cheetah_log_errors(struct pt_regs * regs,struct cheetah_err_info * info,unsigned long afsr,unsigned long afar,int recoverable)1168 static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1169 			       unsigned long afsr, unsigned long afar, int recoverable)
1170 {
1171 	unsigned long hipri;
1172 	char unum[256];
1173 
1174 	printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1175 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1176 	       afsr, afar,
1177 	       (afsr & CHAFSR_TL1) ? 1 : 0);
1178 	printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1179 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1180 	       regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1181 	printk("%s" "ERROR(%d): ",
1182 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
1183 	printk("TPC<%pS>\n", (void *) regs->tpc);
1184 	printk("%s" "ERROR(%d): M_SYND(%lx),  E_SYND(%lx)%s%s\n",
1185 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1186 	       (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1187 	       (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1188 	       (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1189 	       (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1190 	hipri = cheetah_get_hipri(afsr);
1191 	printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1192 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1193 	       hipri, cheetah_get_string(hipri));
1194 
1195 	/* Try to get unumber if relevant. */
1196 #define ESYND_ERRORS	(CHAFSR_IVC | CHAFSR_IVU | \
1197 			 CHAFSR_CPC | CHAFSR_CPU | \
1198 			 CHAFSR_UE  | CHAFSR_CE  | \
1199 			 CHAFSR_EDC | CHAFSR_EDU  | \
1200 			 CHAFSR_UCC | CHAFSR_UCU  | \
1201 			 CHAFSR_WDU | CHAFSR_WDC)
1202 #define MSYND_ERRORS	(CHAFSR_EMC | CHAFSR_EMU)
1203 	if (afsr & ESYND_ERRORS) {
1204 		int syndrome;
1205 		int ret;
1206 
1207 		syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1208 		syndrome = cheetah_ecc_syntab[syndrome];
1209 		ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1210 		if (ret != -1)
1211 			printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1212 			       (recoverable ? KERN_WARNING : KERN_CRIT),
1213 			       smp_processor_id(), unum);
1214 	} else if (afsr & MSYND_ERRORS) {
1215 		int syndrome;
1216 		int ret;
1217 
1218 		syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1219 		syndrome = cheetah_mtag_syntab[syndrome];
1220 		ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1221 		if (ret != -1)
1222 			printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1223 			       (recoverable ? KERN_WARNING : KERN_CRIT),
1224 			       smp_processor_id(), unum);
1225 	}
1226 
1227 	/* Now dump the cache snapshots. */
1228 	printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n",
1229 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1230 	       (int) info->dcache_index,
1231 	       info->dcache_tag,
1232 	       info->dcache_utag,
1233 	       info->dcache_stag);
1234 	printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1235 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1236 	       info->dcache_data[0],
1237 	       info->dcache_data[1],
1238 	       info->dcache_data[2],
1239 	       info->dcache_data[3]);
1240 	printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] "
1241 	       "u[%016llx] l[%016llx]\n",
1242 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1243 	       (int) info->icache_index,
1244 	       info->icache_tag,
1245 	       info->icache_utag,
1246 	       info->icache_stag,
1247 	       info->icache_upper,
1248 	       info->icache_lower);
1249 	printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n",
1250 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1251 	       info->icache_data[0],
1252 	       info->icache_data[1],
1253 	       info->icache_data[2],
1254 	       info->icache_data[3]);
1255 	printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n",
1256 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1257 	       info->icache_data[4],
1258 	       info->icache_data[5],
1259 	       info->icache_data[6],
1260 	       info->icache_data[7]);
1261 	printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n",
1262 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1263 	       (int) info->ecache_index, info->ecache_tag);
1264 	printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1265 	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1266 	       info->ecache_data[0],
1267 	       info->ecache_data[1],
1268 	       info->ecache_data[2],
1269 	       info->ecache_data[3]);
1270 
1271 	afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1272 	while (afsr != 0UL) {
1273 		unsigned long bit = cheetah_get_hipri(afsr);
1274 
1275 		printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1276 		       (recoverable ? KERN_WARNING : KERN_CRIT),
1277 		       bit, cheetah_get_string(bit));
1278 
1279 		afsr &= ~bit;
1280 	}
1281 
1282 	if (!recoverable)
1283 		printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1284 }
1285 
cheetah_recheck_errors(struct cheetah_err_info * logp)1286 static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1287 {
1288 	unsigned long afsr, afar;
1289 	int ret = 0;
1290 
1291 	__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1292 			     : "=r" (afsr)
1293 			     : "i" (ASI_AFSR));
1294 	if ((afsr & cheetah_afsr_errors) != 0) {
1295 		if (logp != NULL) {
1296 			__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1297 					     : "=r" (afar)
1298 					     : "i" (ASI_AFAR));
1299 			logp->afsr = afsr;
1300 			logp->afar = afar;
1301 		}
1302 		ret = 1;
1303 	}
1304 	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1305 			     "membar #Sync\n\t"
1306 			     : : "r" (afsr), "i" (ASI_AFSR));
1307 
1308 	return ret;
1309 }
1310 
cheetah_fecc_handler(struct pt_regs * regs,unsigned long afsr,unsigned long afar)1311 void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1312 {
1313 	struct cheetah_err_info local_snapshot, *p;
1314 	int recoverable;
1315 
1316 	/* Flush E-cache */
1317 	cheetah_flush_ecache();
1318 
1319 	p = cheetah_get_error_log(afsr);
1320 	if (!p) {
1321 		prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1322 			    afsr, afar);
1323 		prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1324 			    smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1325 		prom_halt();
1326 	}
1327 
1328 	/* Grab snapshot of logged error. */
1329 	memcpy(&local_snapshot, p, sizeof(local_snapshot));
1330 
1331 	/* If the current trap snapshot does not match what the
1332 	 * trap handler passed along into our args, big trouble.
1333 	 * In such a case, mark the local copy as invalid.
1334 	 *
1335 	 * Else, it matches and we mark the afsr in the non-local
1336 	 * copy as invalid so we may log new error traps there.
1337 	 */
1338 	if (p->afsr != afsr || p->afar != afar)
1339 		local_snapshot.afsr = CHAFSR_INVALID;
1340 	else
1341 		p->afsr = CHAFSR_INVALID;
1342 
1343 	cheetah_flush_icache();
1344 	cheetah_flush_dcache();
1345 
1346 	/* Re-enable I-cache/D-cache */
1347 	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1348 			     "or %%g1, %1, %%g1\n\t"
1349 			     "stxa %%g1, [%%g0] %0\n\t"
1350 			     "membar #Sync"
1351 			     : /* no outputs */
1352 			     : "i" (ASI_DCU_CONTROL_REG),
1353 			       "i" (DCU_DC | DCU_IC)
1354 			     : "g1");
1355 
1356 	/* Re-enable error reporting */
1357 	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1358 			     "or %%g1, %1, %%g1\n\t"
1359 			     "stxa %%g1, [%%g0] %0\n\t"
1360 			     "membar #Sync"
1361 			     : /* no outputs */
1362 			     : "i" (ASI_ESTATE_ERROR_EN),
1363 			       "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1364 			     : "g1");
1365 
1366 	/* Decide if we can continue after handling this trap and
1367 	 * logging the error.
1368 	 */
1369 	recoverable = 1;
1370 	if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1371 		recoverable = 0;
1372 
1373 	/* Re-check AFSR/AFAR.  What we are looking for here is whether a new
1374 	 * error was logged while we had error reporting traps disabled.
1375 	 */
1376 	if (cheetah_recheck_errors(&local_snapshot)) {
1377 		unsigned long new_afsr = local_snapshot.afsr;
1378 
1379 		/* If we got a new asynchronous error, die... */
1380 		if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1381 				CHAFSR_WDU | CHAFSR_CPU |
1382 				CHAFSR_IVU | CHAFSR_UE |
1383 				CHAFSR_BERR | CHAFSR_TO))
1384 			recoverable = 0;
1385 	}
1386 
1387 	/* Log errors. */
1388 	cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1389 
1390 	if (!recoverable)
1391 		panic("Irrecoverable Fast-ECC error trap.\n");
1392 
1393 	/* Flush E-cache to kick the error trap handlers out. */
1394 	cheetah_flush_ecache();
1395 }
1396 
1397 /* Try to fix a correctable error by pushing the line out from
1398  * the E-cache.  Recheck error reporting registers to see if the
1399  * problem is intermittent.
1400  */
cheetah_fix_ce(unsigned long physaddr)1401 static int cheetah_fix_ce(unsigned long physaddr)
1402 {
1403 	unsigned long orig_estate;
1404 	unsigned long alias1, alias2;
1405 	int ret;
1406 
1407 	/* Make sure correctable error traps are disabled. */
1408 	__asm__ __volatile__("ldxa	[%%g0] %2, %0\n\t"
1409 			     "andn	%0, %1, %%g1\n\t"
1410 			     "stxa	%%g1, [%%g0] %2\n\t"
1411 			     "membar	#Sync"
1412 			     : "=&r" (orig_estate)
1413 			     : "i" (ESTATE_ERROR_CEEN),
1414 			       "i" (ASI_ESTATE_ERROR_EN)
1415 			     : "g1");
1416 
1417 	/* We calculate alias addresses that will force the
1418 	 * cache line in question out of the E-cache.  Then
1419 	 * we bring it back in with an atomic instruction so
1420 	 * that we get it in some modified/exclusive state,
1421 	 * then we displace it again to try and get proper ECC
1422 	 * pushed back into the system.
1423 	 */
1424 	physaddr &= ~(8UL - 1UL);
1425 	alias1 = (ecache_flush_physbase +
1426 		  (physaddr & ((ecache_flush_size >> 1) - 1)));
1427 	alias2 = alias1 + (ecache_flush_size >> 1);
1428 	__asm__ __volatile__("ldxa	[%0] %3, %%g0\n\t"
1429 			     "ldxa	[%1] %3, %%g0\n\t"
1430 			     "casxa	[%2] %3, %%g0, %%g0\n\t"
1431 			     "ldxa	[%0] %3, %%g0\n\t"
1432 			     "ldxa	[%1] %3, %%g0\n\t"
1433 			     "membar	#Sync"
1434 			     : /* no outputs */
1435 			     : "r" (alias1), "r" (alias2),
1436 			       "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1437 
1438 	/* Did that trigger another error? */
1439 	if (cheetah_recheck_errors(NULL)) {
1440 		/* Try one more time. */
1441 		__asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1442 				     "membar #Sync"
1443 				     : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1444 		if (cheetah_recheck_errors(NULL))
1445 			ret = 2;
1446 		else
1447 			ret = 1;
1448 	} else {
1449 		/* No new error, intermittent problem. */
1450 		ret = 0;
1451 	}
1452 
1453 	/* Restore error enables. */
1454 	__asm__ __volatile__("stxa	%0, [%%g0] %1\n\t"
1455 			     "membar	#Sync"
1456 			     : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1457 
1458 	return ret;
1459 }
1460 
1461 /* Return non-zero if PADDR is a valid physical memory address. */
cheetah_check_main_memory(unsigned long paddr)1462 static int cheetah_check_main_memory(unsigned long paddr)
1463 {
1464 	unsigned long vaddr = PAGE_OFFSET + paddr;
1465 
1466 	if (vaddr > (unsigned long) high_memory)
1467 		return 0;
1468 
1469 	return kern_addr_valid(vaddr);
1470 }
1471 
cheetah_cee_handler(struct pt_regs * regs,unsigned long afsr,unsigned long afar)1472 void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1473 {
1474 	struct cheetah_err_info local_snapshot, *p;
1475 	int recoverable, is_memory;
1476 
1477 	p = cheetah_get_error_log(afsr);
1478 	if (!p) {
1479 		prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1480 			    afsr, afar);
1481 		prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1482 			    smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1483 		prom_halt();
1484 	}
1485 
1486 	/* Grab snapshot of logged error. */
1487 	memcpy(&local_snapshot, p, sizeof(local_snapshot));
1488 
1489 	/* If the current trap snapshot does not match what the
1490 	 * trap handler passed along into our args, big trouble.
1491 	 * In such a case, mark the local copy as invalid.
1492 	 *
1493 	 * Else, it matches and we mark the afsr in the non-local
1494 	 * copy as invalid so we may log new error traps there.
1495 	 */
1496 	if (p->afsr != afsr || p->afar != afar)
1497 		local_snapshot.afsr = CHAFSR_INVALID;
1498 	else
1499 		p->afsr = CHAFSR_INVALID;
1500 
1501 	is_memory = cheetah_check_main_memory(afar);
1502 
1503 	if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1504 		/* XXX Might want to log the results of this operation
1505 		 * XXX somewhere... -DaveM
1506 		 */
1507 		cheetah_fix_ce(afar);
1508 	}
1509 
1510 	{
1511 		int flush_all, flush_line;
1512 
1513 		flush_all = flush_line = 0;
1514 		if ((afsr & CHAFSR_EDC) != 0UL) {
1515 			if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1516 				flush_line = 1;
1517 			else
1518 				flush_all = 1;
1519 		} else if ((afsr & CHAFSR_CPC) != 0UL) {
1520 			if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1521 				flush_line = 1;
1522 			else
1523 				flush_all = 1;
1524 		}
1525 
1526 		/* Trap handler only disabled I-cache, flush it. */
1527 		cheetah_flush_icache();
1528 
1529 		/* Re-enable I-cache */
1530 		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1531 				     "or %%g1, %1, %%g1\n\t"
1532 				     "stxa %%g1, [%%g0] %0\n\t"
1533 				     "membar #Sync"
1534 				     : /* no outputs */
1535 				     : "i" (ASI_DCU_CONTROL_REG),
1536 				     "i" (DCU_IC)
1537 				     : "g1");
1538 
1539 		if (flush_all)
1540 			cheetah_flush_ecache();
1541 		else if (flush_line)
1542 			cheetah_flush_ecache_line(afar);
1543 	}
1544 
1545 	/* Re-enable error reporting */
1546 	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1547 			     "or %%g1, %1, %%g1\n\t"
1548 			     "stxa %%g1, [%%g0] %0\n\t"
1549 			     "membar #Sync"
1550 			     : /* no outputs */
1551 			     : "i" (ASI_ESTATE_ERROR_EN),
1552 			       "i" (ESTATE_ERROR_CEEN)
1553 			     : "g1");
1554 
1555 	/* Decide if we can continue after handling this trap and
1556 	 * logging the error.
1557 	 */
1558 	recoverable = 1;
1559 	if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1560 		recoverable = 0;
1561 
1562 	/* Re-check AFSR/AFAR */
1563 	(void) cheetah_recheck_errors(&local_snapshot);
1564 
1565 	/* Log errors. */
1566 	cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1567 
1568 	if (!recoverable)
1569 		panic("Irrecoverable Correctable-ECC error trap.\n");
1570 }
1571 
cheetah_deferred_handler(struct pt_regs * regs,unsigned long afsr,unsigned long afar)1572 void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1573 {
1574 	struct cheetah_err_info local_snapshot, *p;
1575 	int recoverable, is_memory;
1576 
1577 #ifdef CONFIG_PCI
1578 	/* Check for the special PCI poke sequence. */
1579 	if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1580 		cheetah_flush_icache();
1581 		cheetah_flush_dcache();
1582 
1583 		/* Re-enable I-cache/D-cache */
1584 		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1585 				     "or %%g1, %1, %%g1\n\t"
1586 				     "stxa %%g1, [%%g0] %0\n\t"
1587 				     "membar #Sync"
1588 				     : /* no outputs */
1589 				     : "i" (ASI_DCU_CONTROL_REG),
1590 				       "i" (DCU_DC | DCU_IC)
1591 				     : "g1");
1592 
1593 		/* Re-enable error reporting */
1594 		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1595 				     "or %%g1, %1, %%g1\n\t"
1596 				     "stxa %%g1, [%%g0] %0\n\t"
1597 				     "membar #Sync"
1598 				     : /* no outputs */
1599 				     : "i" (ASI_ESTATE_ERROR_EN),
1600 				       "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1601 				     : "g1");
1602 
1603 		(void) cheetah_recheck_errors(NULL);
1604 
1605 		pci_poke_faulted = 1;
1606 		regs->tpc += 4;
1607 		regs->tnpc = regs->tpc + 4;
1608 		return;
1609 	}
1610 #endif
1611 
1612 	p = cheetah_get_error_log(afsr);
1613 	if (!p) {
1614 		prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1615 			    afsr, afar);
1616 		prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1617 			    smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1618 		prom_halt();
1619 	}
1620 
1621 	/* Grab snapshot of logged error. */
1622 	memcpy(&local_snapshot, p, sizeof(local_snapshot));
1623 
1624 	/* If the current trap snapshot does not match what the
1625 	 * trap handler passed along into our args, big trouble.
1626 	 * In such a case, mark the local copy as invalid.
1627 	 *
1628 	 * Else, it matches and we mark the afsr in the non-local
1629 	 * copy as invalid so we may log new error traps there.
1630 	 */
1631 	if (p->afsr != afsr || p->afar != afar)
1632 		local_snapshot.afsr = CHAFSR_INVALID;
1633 	else
1634 		p->afsr = CHAFSR_INVALID;
1635 
1636 	is_memory = cheetah_check_main_memory(afar);
1637 
1638 	{
1639 		int flush_all, flush_line;
1640 
1641 		flush_all = flush_line = 0;
1642 		if ((afsr & CHAFSR_EDU) != 0UL) {
1643 			if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1644 				flush_line = 1;
1645 			else
1646 				flush_all = 1;
1647 		} else if ((afsr & CHAFSR_BERR) != 0UL) {
1648 			if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1649 				flush_line = 1;
1650 			else
1651 				flush_all = 1;
1652 		}
1653 
1654 		cheetah_flush_icache();
1655 		cheetah_flush_dcache();
1656 
1657 		/* Re-enable I/D caches */
1658 		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1659 				     "or %%g1, %1, %%g1\n\t"
1660 				     "stxa %%g1, [%%g0] %0\n\t"
1661 				     "membar #Sync"
1662 				     : /* no outputs */
1663 				     : "i" (ASI_DCU_CONTROL_REG),
1664 				     "i" (DCU_IC | DCU_DC)
1665 				     : "g1");
1666 
1667 		if (flush_all)
1668 			cheetah_flush_ecache();
1669 		else if (flush_line)
1670 			cheetah_flush_ecache_line(afar);
1671 	}
1672 
1673 	/* Re-enable error reporting */
1674 	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1675 			     "or %%g1, %1, %%g1\n\t"
1676 			     "stxa %%g1, [%%g0] %0\n\t"
1677 			     "membar #Sync"
1678 			     : /* no outputs */
1679 			     : "i" (ASI_ESTATE_ERROR_EN),
1680 			     "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1681 			     : "g1");
1682 
1683 	/* Decide if we can continue after handling this trap and
1684 	 * logging the error.
1685 	 */
1686 	recoverable = 1;
1687 	if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1688 		recoverable = 0;
1689 
1690 	/* Re-check AFSR/AFAR.  What we are looking for here is whether a new
1691 	 * error was logged while we had error reporting traps disabled.
1692 	 */
1693 	if (cheetah_recheck_errors(&local_snapshot)) {
1694 		unsigned long new_afsr = local_snapshot.afsr;
1695 
1696 		/* If we got a new asynchronous error, die... */
1697 		if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1698 				CHAFSR_WDU | CHAFSR_CPU |
1699 				CHAFSR_IVU | CHAFSR_UE |
1700 				CHAFSR_BERR | CHAFSR_TO))
1701 			recoverable = 0;
1702 	}
1703 
1704 	/* Log errors. */
1705 	cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1706 
1707 	/* "Recoverable" here means we try to yank the page from ever
1708 	 * being newly used again.  This depends upon a few things:
1709 	 * 1) Must be main memory, and AFAR must be valid.
1710 	 * 2) If we trapped from user, OK.
1711 	 * 3) Else, if we trapped from kernel we must find exception
1712 	 *    table entry (ie. we have to have been accessing user
1713 	 *    space).
1714 	 *
1715 	 * If AFAR is not in main memory, or we trapped from kernel
1716 	 * and cannot find an exception table entry, it is unacceptable
1717 	 * to try and continue.
1718 	 */
1719 	if (recoverable && is_memory) {
1720 		if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1721 			/* OK, usermode access. */
1722 			recoverable = 1;
1723 		} else {
1724 			const struct exception_table_entry *entry;
1725 
1726 			entry = search_exception_tables(regs->tpc);
1727 			if (entry) {
1728 				/* OK, kernel access to userspace. */
1729 				recoverable = 1;
1730 
1731 			} else {
1732 				/* BAD, privileged state is corrupted. */
1733 				recoverable = 0;
1734 			}
1735 
1736 			if (recoverable) {
1737 				if (pfn_valid(afar >> PAGE_SHIFT))
1738 					get_page(pfn_to_page(afar >> PAGE_SHIFT));
1739 				else
1740 					recoverable = 0;
1741 
1742 				/* Only perform fixup if we still have a
1743 				 * recoverable condition.
1744 				 */
1745 				if (recoverable) {
1746 					regs->tpc = entry->fixup;
1747 					regs->tnpc = regs->tpc + 4;
1748 				}
1749 			}
1750 		}
1751 	} else {
1752 		recoverable = 0;
1753 	}
1754 
1755 	if (!recoverable)
1756 		panic("Irrecoverable deferred error trap.\n");
1757 }
1758 
1759 /* Handle a D/I cache parity error trap.  TYPE is encoded as:
1760  *
1761  * Bit0:	0=dcache,1=icache
1762  * Bit1:	0=recoverable,1=unrecoverable
1763  *
1764  * The hardware has disabled both the I-cache and D-cache in
1765  * the %dcr register.
1766  */
cheetah_plus_parity_error(int type,struct pt_regs * regs)1767 void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1768 {
1769 	if (type & 0x1)
1770 		__cheetah_flush_icache();
1771 	else
1772 		cheetah_plus_zap_dcache_parity();
1773 	cheetah_flush_dcache();
1774 
1775 	/* Re-enable I-cache/D-cache */
1776 	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1777 			     "or %%g1, %1, %%g1\n\t"
1778 			     "stxa %%g1, [%%g0] %0\n\t"
1779 			     "membar #Sync"
1780 			     : /* no outputs */
1781 			     : "i" (ASI_DCU_CONTROL_REG),
1782 			       "i" (DCU_DC | DCU_IC)
1783 			     : "g1");
1784 
1785 	if (type & 0x2) {
1786 		printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1787 		       smp_processor_id(),
1788 		       (type & 0x1) ? 'I' : 'D',
1789 		       regs->tpc);
1790 		printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
1791 		panic("Irrecoverable Cheetah+ parity error.");
1792 	}
1793 
1794 	printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1795 	       smp_processor_id(),
1796 	       (type & 0x1) ? 'I' : 'D',
1797 	       regs->tpc);
1798 	printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
1799 }
1800 
1801 struct sun4v_error_entry {
1802 	/* Unique error handle */
1803 /*0x00*/u64		err_handle;
1804 
1805 	/* %stick value at the time of the error */
1806 /*0x08*/u64		err_stick;
1807 
1808 /*0x10*/u8		reserved_1[3];
1809 
1810 	/* Error type */
1811 /*0x13*/u8		err_type;
1812 #define SUN4V_ERR_TYPE_UNDEFINED	0
1813 #define SUN4V_ERR_TYPE_UNCORRECTED_RES	1
1814 #define SUN4V_ERR_TYPE_PRECISE_NONRES	2
1815 #define SUN4V_ERR_TYPE_DEFERRED_NONRES	3
1816 #define SUN4V_ERR_TYPE_SHUTDOWN_RQST	4
1817 #define SUN4V_ERR_TYPE_DUMP_CORE	5
1818 #define SUN4V_ERR_TYPE_SP_STATE_CHANGE	6
1819 #define SUN4V_ERR_TYPE_NUM		7
1820 
1821 	/* Error attributes */
1822 /*0x14*/u32		err_attrs;
1823 #define SUN4V_ERR_ATTRS_PROCESSOR	0x00000001
1824 #define SUN4V_ERR_ATTRS_MEMORY		0x00000002
1825 #define SUN4V_ERR_ATTRS_PIO		0x00000004
1826 #define SUN4V_ERR_ATTRS_INT_REGISTERS	0x00000008
1827 #define SUN4V_ERR_ATTRS_FPU_REGISTERS	0x00000010
1828 #define SUN4V_ERR_ATTRS_SHUTDOWN_RQST	0x00000020
1829 #define SUN4V_ERR_ATTRS_ASR		0x00000040
1830 #define SUN4V_ERR_ATTRS_ASI		0x00000080
1831 #define SUN4V_ERR_ATTRS_PRIV_REG	0x00000100
1832 #define SUN4V_ERR_ATTRS_SPSTATE_MSK	0x00000600
1833 #define SUN4V_ERR_ATTRS_MCD		0x00000800
1834 #define SUN4V_ERR_ATTRS_SPSTATE_SHFT	9
1835 #define SUN4V_ERR_ATTRS_MODE_MSK	0x03000000
1836 #define SUN4V_ERR_ATTRS_MODE_SHFT	24
1837 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL	0x80000000
1838 
1839 #define SUN4V_ERR_SPSTATE_FAULTED	0
1840 #define SUN4V_ERR_SPSTATE_AVAILABLE	1
1841 #define SUN4V_ERR_SPSTATE_NOT_PRESENT	2
1842 
1843 #define SUN4V_ERR_MODE_USER		1
1844 #define SUN4V_ERR_MODE_PRIV		2
1845 
1846 	/* Real address of the memory region or PIO transaction */
1847 /*0x18*/u64		err_raddr;
1848 
1849 	/* Size of the operation triggering the error, in bytes */
1850 /*0x20*/u32		err_size;
1851 
1852 	/* ID of the CPU */
1853 /*0x24*/u16		err_cpu;
1854 
1855 	/* Grace periof for shutdown, in seconds */
1856 /*0x26*/u16		err_secs;
1857 
1858 	/* Value of the %asi register */
1859 /*0x28*/u8		err_asi;
1860 
1861 /*0x29*/u8		reserved_2;
1862 
1863 	/* Value of the ASR register number */
1864 /*0x2a*/u16		err_asr;
1865 #define SUN4V_ERR_ASR_VALID		0x8000
1866 
1867 /*0x2c*/u32		reserved_3;
1868 /*0x30*/u64		reserved_4;
1869 /*0x38*/u64		reserved_5;
1870 };
1871 
1872 static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1873 static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1874 
sun4v_err_type_to_str(u8 type)1875 static const char *sun4v_err_type_to_str(u8 type)
1876 {
1877 	static const char *types[SUN4V_ERR_TYPE_NUM] = {
1878 		"undefined",
1879 		"uncorrected resumable",
1880 		"precise nonresumable",
1881 		"deferred nonresumable",
1882 		"shutdown request",
1883 		"dump core",
1884 		"SP state change",
1885 	};
1886 
1887 	if (type < SUN4V_ERR_TYPE_NUM)
1888 		return types[type];
1889 
1890 	return "unknown";
1891 }
1892 
sun4v_emit_err_attr_strings(u32 attrs)1893 static void sun4v_emit_err_attr_strings(u32 attrs)
1894 {
1895 	static const char *attr_names[] = {
1896 		"processor",
1897 		"memory",
1898 		"PIO",
1899 		"int-registers",
1900 		"fpu-registers",
1901 		"shutdown-request",
1902 		"ASR",
1903 		"ASI",
1904 		"priv-reg",
1905 	};
1906 	static const char *sp_states[] = {
1907 		"sp-faulted",
1908 		"sp-available",
1909 		"sp-not-present",
1910 		"sp-state-reserved",
1911 	};
1912 	static const char *modes[] = {
1913 		"mode-reserved0",
1914 		"user",
1915 		"priv",
1916 		"mode-reserved1",
1917 	};
1918 	u32 sp_state, mode;
1919 	int i;
1920 
1921 	for (i = 0; i < ARRAY_SIZE(attr_names); i++) {
1922 		if (attrs & (1U << i)) {
1923 			const char *s = attr_names[i];
1924 
1925 			pr_cont("%s ", s);
1926 		}
1927 	}
1928 
1929 	sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >>
1930 		    SUN4V_ERR_ATTRS_SPSTATE_SHFT);
1931 	pr_cont("%s ", sp_states[sp_state]);
1932 
1933 	mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >>
1934 		SUN4V_ERR_ATTRS_MODE_SHFT);
1935 	pr_cont("%s ", modes[mode]);
1936 
1937 	if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL)
1938 		pr_cont("res-queue-full ");
1939 }
1940 
1941 /* When the report contains a real-address of "-1" it means that the
1942  * hardware did not provide the address.  So we compute the effective
1943  * address of the load or store instruction at regs->tpc and report
1944  * that.  Usually when this happens it's a PIO and in such a case we
1945  * are using physical addresses with bypass ASIs anyways, so what we
1946  * report here is exactly what we want.
1947  */
sun4v_report_real_raddr(const char * pfx,struct pt_regs * regs)1948 static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs)
1949 {
1950 	unsigned int insn;
1951 	u64 addr;
1952 
1953 	if (!(regs->tstate & TSTATE_PRIV))
1954 		return;
1955 
1956 	insn = *(unsigned int *) regs->tpc;
1957 
1958 	addr = compute_effective_address(regs, insn, 0);
1959 
1960 	printk("%s: insn effective address [0x%016llx]\n",
1961 	       pfx, addr);
1962 }
1963 
sun4v_log_error(struct pt_regs * regs,struct sun4v_error_entry * ent,int cpu,const char * pfx,atomic_t * ocnt)1964 static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
1965 			    int cpu, const char *pfx, atomic_t *ocnt)
1966 {
1967 	u64 *raw_ptr = (u64 *) ent;
1968 	u32 attrs;
1969 	int cnt;
1970 
1971 	printk("%s: Reporting on cpu %d\n", pfx, cpu);
1972 	printk("%s: TPC [0x%016lx] <%pS>\n",
1973 	       pfx, regs->tpc, (void *) regs->tpc);
1974 
1975 	printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n",
1976 	       pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]);
1977 	printk("%s:      %016llx:%016llx:%016llx:%016llx]\n",
1978 	       pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]);
1979 
1980 	printk("%s: handle [0x%016llx] stick [0x%016llx]\n",
1981 	       pfx, ent->err_handle, ent->err_stick);
1982 
1983 	printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type));
1984 
1985 	attrs = ent->err_attrs;
1986 	printk("%s: attrs [0x%08x] < ", pfx, attrs);
1987 	sun4v_emit_err_attr_strings(attrs);
1988 	pr_cont(">\n");
1989 
1990 	/* Various fields in the error report are only valid if
1991 	 * certain attribute bits are set.
1992 	 */
1993 	if (attrs & (SUN4V_ERR_ATTRS_MEMORY |
1994 		     SUN4V_ERR_ATTRS_PIO |
1995 		     SUN4V_ERR_ATTRS_ASI)) {
1996 		printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr);
1997 
1998 		if (ent->err_raddr == ~(u64)0)
1999 			sun4v_report_real_raddr(pfx, regs);
2000 	}
2001 
2002 	if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI))
2003 		printk("%s: size [0x%x]\n", pfx, ent->err_size);
2004 
2005 	if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR |
2006 		     SUN4V_ERR_ATTRS_INT_REGISTERS |
2007 		     SUN4V_ERR_ATTRS_FPU_REGISTERS |
2008 		     SUN4V_ERR_ATTRS_PRIV_REG))
2009 		printk("%s: cpu[%u]\n", pfx, ent->err_cpu);
2010 
2011 	if (attrs & SUN4V_ERR_ATTRS_ASI)
2012 		printk("%s: asi [0x%02x]\n", pfx, ent->err_asi);
2013 
2014 	if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS |
2015 		      SUN4V_ERR_ATTRS_FPU_REGISTERS |
2016 		      SUN4V_ERR_ATTRS_PRIV_REG)) &&
2017 	    (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0)
2018 		printk("%s: reg [0x%04x]\n",
2019 		       pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID);
2020 
2021 	show_regs(regs);
2022 
2023 	if ((cnt = atomic_read(ocnt)) != 0) {
2024 		atomic_set(ocnt, 0);
2025 		wmb();
2026 		printk("%s: Queue overflowed %d times.\n",
2027 		       pfx, cnt);
2028 	}
2029 }
2030 
2031 /* Handle memory corruption detected error which is vectored in
2032  * through resumable error trap.
2033  */
do_mcd_err(struct pt_regs * regs,struct sun4v_error_entry ent)2034 void do_mcd_err(struct pt_regs *regs, struct sun4v_error_entry ent)
2035 {
2036 	if (notify_die(DIE_TRAP, "MCD error", regs, 0, 0x34,
2037 		       SIGSEGV) == NOTIFY_STOP)
2038 		return;
2039 
2040 	if (regs->tstate & TSTATE_PRIV) {
2041 		/* MCD exception could happen because the task was
2042 		 * running a system call with MCD enabled and passed a
2043 		 * non-versioned pointer or pointer with bad version
2044 		 * tag to the system call. In such cases, hypervisor
2045 		 * places the address of offending instruction in the
2046 		 * resumable error report. This is a deferred error,
2047 		 * so the read/write that caused the trap was potentially
2048 		 * retired long time back and we may have no choice
2049 		 * but to send SIGSEGV to the process.
2050 		 */
2051 		const struct exception_table_entry *entry;
2052 
2053 		entry = search_exception_tables(regs->tpc);
2054 		if (entry) {
2055 			/* Looks like a bad syscall parameter */
2056 #ifdef DEBUG_EXCEPTIONS
2057 			pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n",
2058 				 regs->tpc);
2059 			pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
2060 				 ent.err_raddr, entry->fixup);
2061 #endif
2062 			regs->tpc = entry->fixup;
2063 			regs->tnpc = regs->tpc + 4;
2064 			return;
2065 		}
2066 	}
2067 
2068 	/* Send SIGSEGV to the userspace process with the right signal
2069 	 * code
2070 	 */
2071 	force_sig_fault(SIGSEGV, SEGV_ADIDERR, (void __user *)ent.err_raddr);
2072 }
2073 
2074 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2075  * Log the event and clear the first word of the entry.
2076  */
sun4v_resum_error(struct pt_regs * regs,unsigned long offset)2077 void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
2078 {
2079 	enum ctx_state prev_state = exception_enter();
2080 	struct sun4v_error_entry *ent, local_copy;
2081 	struct trap_per_cpu *tb;
2082 	unsigned long paddr;
2083 	int cpu;
2084 
2085 	cpu = get_cpu();
2086 
2087 	tb = &trap_block[cpu];
2088 	paddr = tb->resum_kernel_buf_pa + offset;
2089 	ent = __va(paddr);
2090 
2091 	memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2092 
2093 	/* We have a local copy now, so release the entry.  */
2094 	ent->err_handle = 0;
2095 	wmb();
2096 
2097 	put_cpu();
2098 
2099 	if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) {
2100 		/* We should really take the seconds field of
2101 		 * the error report and use it for the shutdown
2102 		 * invocation, but for now do the same thing we
2103 		 * do for a DS shutdown request.
2104 		 */
2105 		pr_info("Shutdown request, %u seconds...\n",
2106 			local_copy.err_secs);
2107 		orderly_poweroff(true);
2108 		goto out;
2109 	}
2110 
2111 	/* If this is a memory corruption detected error vectored in
2112 	 * by HV through resumable error trap, call the handler
2113 	 */
2114 	if (local_copy.err_attrs & SUN4V_ERR_ATTRS_MCD) {
2115 		do_mcd_err(regs, local_copy);
2116 		return;
2117 	}
2118 
2119 	sun4v_log_error(regs, &local_copy, cpu,
2120 			KERN_ERR "RESUMABLE ERROR",
2121 			&sun4v_resum_oflow_cnt);
2122 out:
2123 	exception_exit(prev_state);
2124 }
2125 
2126 /* If we try to printk() we'll probably make matters worse, by trying
2127  * to retake locks this cpu already holds or causing more errors. So
2128  * just bump a counter, and we'll report these counter bumps above.
2129  */
sun4v_resum_overflow(struct pt_regs * regs)2130 void sun4v_resum_overflow(struct pt_regs *regs)
2131 {
2132 	atomic_inc(&sun4v_resum_oflow_cnt);
2133 }
2134 
2135 /* Given a set of registers, get the virtual addressi that was being accessed
2136  * by the faulting instructions at tpc.
2137  */
sun4v_get_vaddr(struct pt_regs * regs)2138 static unsigned long sun4v_get_vaddr(struct pt_regs *regs)
2139 {
2140 	unsigned int insn;
2141 
2142 	if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) {
2143 		return compute_effective_address(regs, insn,
2144 						 (insn >> 25) & 0x1f);
2145 	}
2146 	return 0;
2147 }
2148 
2149 /* Attempt to handle non-resumable errors generated from userspace.
2150  * Returns true if the signal was handled, false otherwise.
2151  */
sun4v_nonresum_error_user_handled(struct pt_regs * regs,struct sun4v_error_entry * ent)2152 bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
2153 				  struct sun4v_error_entry *ent) {
2154 
2155 	unsigned int attrs = ent->err_attrs;
2156 
2157 	if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
2158 		unsigned long addr = ent->err_raddr;
2159 
2160 		if (addr == ~(u64)0) {
2161 			/* This seems highly unlikely to ever occur */
2162 			pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n");
2163 		} else {
2164 			unsigned long page_cnt = DIV_ROUND_UP(ent->err_size,
2165 							      PAGE_SIZE);
2166 
2167 			/* Break the unfortunate news. */
2168 			pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n",
2169 				 addr);
2170 			pr_emerg("SUN4V NON-RECOVERABLE ERROR:   Claiming %lu ages.\n",
2171 				 page_cnt);
2172 
2173 			while (page_cnt-- > 0) {
2174 				if (pfn_valid(addr >> PAGE_SHIFT))
2175 					get_page(pfn_to_page(addr >> PAGE_SHIFT));
2176 				addr += PAGE_SIZE;
2177 			}
2178 		}
2179 		force_sig(SIGKILL);
2180 
2181 		return true;
2182 	}
2183 	if (attrs & SUN4V_ERR_ATTRS_PIO) {
2184 		force_sig_fault(SIGBUS, BUS_ADRERR,
2185 				(void __user *)sun4v_get_vaddr(regs));
2186 		return true;
2187 	}
2188 
2189 	/* Default to doing nothing */
2190 	return false;
2191 }
2192 
2193 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2194  * Log the event, clear the first word of the entry, and die.
2195  */
sun4v_nonresum_error(struct pt_regs * regs,unsigned long offset)2196 void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
2197 {
2198 	struct sun4v_error_entry *ent, local_copy;
2199 	struct trap_per_cpu *tb;
2200 	unsigned long paddr;
2201 	int cpu;
2202 
2203 	cpu = get_cpu();
2204 
2205 	tb = &trap_block[cpu];
2206 	paddr = tb->nonresum_kernel_buf_pa + offset;
2207 	ent = __va(paddr);
2208 
2209 	memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2210 
2211 	/* We have a local copy now, so release the entry.  */
2212 	ent->err_handle = 0;
2213 	wmb();
2214 
2215 	put_cpu();
2216 
2217 	if (!(regs->tstate & TSTATE_PRIV) &&
2218 	    sun4v_nonresum_error_user_handled(regs, &local_copy)) {
2219 		/* DON'T PANIC: This userspace error was handled. */
2220 		return;
2221 	}
2222 
2223 #ifdef CONFIG_PCI
2224 	/* Check for the special PCI poke sequence. */
2225 	if (pci_poke_in_progress && pci_poke_cpu == cpu) {
2226 		pci_poke_faulted = 1;
2227 		regs->tpc += 4;
2228 		regs->tnpc = regs->tpc + 4;
2229 		return;
2230 	}
2231 #endif
2232 
2233 	sun4v_log_error(regs, &local_copy, cpu,
2234 			KERN_EMERG "NON-RESUMABLE ERROR",
2235 			&sun4v_nonresum_oflow_cnt);
2236 
2237 	panic("Non-resumable error.");
2238 }
2239 
2240 /* If we try to printk() we'll probably make matters worse, by trying
2241  * to retake locks this cpu already holds or causing more errors. So
2242  * just bump a counter, and we'll report these counter bumps above.
2243  */
sun4v_nonresum_overflow(struct pt_regs * regs)2244 void sun4v_nonresum_overflow(struct pt_regs *regs)
2245 {
2246 	/* XXX Actually even this can make not that much sense.  Perhaps
2247 	 * XXX we should just pull the plug and panic directly from here?
2248 	 */
2249 	atomic_inc(&sun4v_nonresum_oflow_cnt);
2250 }
2251 
sun4v_tlb_error(struct pt_regs * regs)2252 static void sun4v_tlb_error(struct pt_regs *regs)
2253 {
2254 	die_if_kernel("TLB/TSB error", regs);
2255 }
2256 
2257 unsigned long sun4v_err_itlb_vaddr;
2258 unsigned long sun4v_err_itlb_ctx;
2259 unsigned long sun4v_err_itlb_pte;
2260 unsigned long sun4v_err_itlb_error;
2261 
sun4v_itlb_error_report(struct pt_regs * regs,int tl)2262 void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2263 {
2264 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2265 
2266 	printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2267 	       regs->tpc, tl);
2268 	printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
2269 	printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2270 	printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
2271 	       (void *) regs->u_regs[UREG_I7]);
2272 	printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
2273 	       "pte[%lx] error[%lx]\n",
2274 	       sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2275 	       sun4v_err_itlb_pte, sun4v_err_itlb_error);
2276 
2277 	sun4v_tlb_error(regs);
2278 }
2279 
2280 unsigned long sun4v_err_dtlb_vaddr;
2281 unsigned long sun4v_err_dtlb_ctx;
2282 unsigned long sun4v_err_dtlb_pte;
2283 unsigned long sun4v_err_dtlb_error;
2284 
sun4v_dtlb_error_report(struct pt_regs * regs,int tl)2285 void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2286 {
2287 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2288 
2289 	printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2290 	       regs->tpc, tl);
2291 	printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
2292 	printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2293 	printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
2294 	       (void *) regs->u_regs[UREG_I7]);
2295 	printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
2296 	       "pte[%lx] error[%lx]\n",
2297 	       sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2298 	       sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2299 
2300 	sun4v_tlb_error(regs);
2301 }
2302 
hypervisor_tlbop_error(unsigned long err,unsigned long op)2303 void hypervisor_tlbop_error(unsigned long err, unsigned long op)
2304 {
2305 	printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
2306 	       err, op);
2307 }
2308 
hypervisor_tlbop_error_xcall(unsigned long err,unsigned long op)2309 void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
2310 {
2311 	printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
2312 	       err, op);
2313 }
2314 
do_fpe_common(struct pt_regs * regs)2315 static void do_fpe_common(struct pt_regs *regs)
2316 {
2317 	if (regs->tstate & TSTATE_PRIV) {
2318 		regs->tpc = regs->tnpc;
2319 		regs->tnpc += 4;
2320 	} else {
2321 		unsigned long fsr = current_thread_info()->xfsr[0];
2322 		int code;
2323 
2324 		if (test_thread_flag(TIF_32BIT)) {
2325 			regs->tpc &= 0xffffffff;
2326 			regs->tnpc &= 0xffffffff;
2327 		}
2328 		code = FPE_FLTUNK;
2329 		if ((fsr & 0x1c000) == (1 << 14)) {
2330 			if (fsr & 0x10)
2331 				code = FPE_FLTINV;
2332 			else if (fsr & 0x08)
2333 				code = FPE_FLTOVF;
2334 			else if (fsr & 0x04)
2335 				code = FPE_FLTUND;
2336 			else if (fsr & 0x02)
2337 				code = FPE_FLTDIV;
2338 			else if (fsr & 0x01)
2339 				code = FPE_FLTRES;
2340 		}
2341 		force_sig_fault(SIGFPE, code, (void __user *)regs->tpc);
2342 	}
2343 }
2344 
do_fpieee(struct pt_regs * regs)2345 void do_fpieee(struct pt_regs *regs)
2346 {
2347 	enum ctx_state prev_state = exception_enter();
2348 
2349 	if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2350 		       0, 0x24, SIGFPE) == NOTIFY_STOP)
2351 		goto out;
2352 
2353 	do_fpe_common(regs);
2354 out:
2355 	exception_exit(prev_state);
2356 }
2357 
do_fpother(struct pt_regs * regs)2358 void do_fpother(struct pt_regs *regs)
2359 {
2360 	enum ctx_state prev_state = exception_enter();
2361 	struct fpustate *f = FPUSTATE;
2362 	int ret = 0;
2363 
2364 	if (notify_die(DIE_TRAP, "fpu exception other", regs,
2365 		       0, 0x25, SIGFPE) == NOTIFY_STOP)
2366 		goto out;
2367 
2368 	switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2369 	case (2 << 14): /* unfinished_FPop */
2370 	case (3 << 14): /* unimplemented_FPop */
2371 		ret = do_mathemu(regs, f, false);
2372 		break;
2373 	}
2374 	if (ret)
2375 		goto out;
2376 	do_fpe_common(regs);
2377 out:
2378 	exception_exit(prev_state);
2379 }
2380 
do_tof(struct pt_regs * regs)2381 void do_tof(struct pt_regs *regs)
2382 {
2383 	enum ctx_state prev_state = exception_enter();
2384 
2385 	if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2386 		       0, 0x26, SIGEMT) == NOTIFY_STOP)
2387 		goto out;
2388 
2389 	if (regs->tstate & TSTATE_PRIV)
2390 		die_if_kernel("Penguin overflow trap from kernel mode", regs);
2391 	if (test_thread_flag(TIF_32BIT)) {
2392 		regs->tpc &= 0xffffffff;
2393 		regs->tnpc &= 0xffffffff;
2394 	}
2395 	force_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)regs->tpc);
2396 out:
2397 	exception_exit(prev_state);
2398 }
2399 
do_div0(struct pt_regs * regs)2400 void do_div0(struct pt_regs *regs)
2401 {
2402 	enum ctx_state prev_state = exception_enter();
2403 
2404 	if (notify_die(DIE_TRAP, "integer division by zero", regs,
2405 		       0, 0x28, SIGFPE) == NOTIFY_STOP)
2406 		goto out;
2407 
2408 	if (regs->tstate & TSTATE_PRIV)
2409 		die_if_kernel("TL0: Kernel divide by zero.", regs);
2410 	if (test_thread_flag(TIF_32BIT)) {
2411 		regs->tpc &= 0xffffffff;
2412 		regs->tnpc &= 0xffffffff;
2413 	}
2414 	force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->tpc);
2415 out:
2416 	exception_exit(prev_state);
2417 }
2418 
instruction_dump(unsigned int * pc)2419 static void instruction_dump(unsigned int *pc)
2420 {
2421 	int i;
2422 
2423 	if ((((unsigned long) pc) & 3))
2424 		return;
2425 
2426 	printk("Instruction DUMP:");
2427 	for (i = -3; i < 6; i++)
2428 		printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2429 	printk("\n");
2430 }
2431 
user_instruction_dump(unsigned int __user * pc)2432 static void user_instruction_dump(unsigned int __user *pc)
2433 {
2434 	int i;
2435 	unsigned int buf[9];
2436 
2437 	if ((((unsigned long) pc) & 3))
2438 		return;
2439 
2440 	if (copy_from_user(buf, pc - 3, sizeof(buf)))
2441 		return;
2442 
2443 	printk("Instruction DUMP:");
2444 	for (i = 0; i < 9; i++)
2445 		printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2446 	printk("\n");
2447 }
2448 
show_stack(struct task_struct * tsk,unsigned long * _ksp,const char * loglvl)2449 void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl)
2450 {
2451 	unsigned long fp, ksp;
2452 	struct thread_info *tp;
2453 	int count = 0;
2454 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2455 	int graph = 0;
2456 #endif
2457 
2458 	ksp = (unsigned long) _ksp;
2459 	if (!tsk)
2460 		tsk = current;
2461 	tp = task_thread_info(tsk);
2462 	if (ksp == 0UL) {
2463 		if (tsk == current)
2464 			asm("mov %%fp, %0" : "=r" (ksp));
2465 		else
2466 			ksp = tp->ksp;
2467 	}
2468 	if (tp == current_thread_info())
2469 		flushw_all();
2470 
2471 	fp = ksp + STACK_BIAS;
2472 
2473 	printk("%sCall Trace:\n", loglvl);
2474 	do {
2475 		struct sparc_stackf *sf;
2476 		struct pt_regs *regs;
2477 		unsigned long pc;
2478 
2479 		if (!kstack_valid(tp, fp))
2480 			break;
2481 		sf = (struct sparc_stackf *) fp;
2482 		regs = (struct pt_regs *) (sf + 1);
2483 
2484 		if (kstack_is_trap_frame(tp, regs)) {
2485 			if (!(regs->tstate & TSTATE_PRIV))
2486 				break;
2487 			pc = regs->tpc;
2488 			fp = regs->u_regs[UREG_I6] + STACK_BIAS;
2489 		} else {
2490 			pc = sf->callers_pc;
2491 			fp = (unsigned long)sf->fp + STACK_BIAS;
2492 		}
2493 
2494 		print_ip_sym(loglvl, pc);
2495 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2496 		if ((pc + 8UL) == (unsigned long) &return_to_handler) {
2497 			struct ftrace_ret_stack *ret_stack;
2498 			ret_stack = ftrace_graph_get_ret_stack(tsk, graph);
2499 			if (ret_stack) {
2500 				pc = ret_stack->ret;
2501 				print_ip_sym(loglvl, pc);
2502 				graph++;
2503 			}
2504 		}
2505 #endif
2506 	} while (++count < 16);
2507 }
2508 
kernel_stack_up(struct reg_window * rw)2509 static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2510 {
2511 	unsigned long fp = rw->ins[6];
2512 
2513 	if (!fp)
2514 		return NULL;
2515 
2516 	return (struct reg_window *) (fp + STACK_BIAS);
2517 }
2518 
die_if_kernel(char * str,struct pt_regs * regs)2519 void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
2520 {
2521 	static int die_counter;
2522 	int count = 0;
2523 
2524 	/* Amuse the user. */
2525 	printk(
2526 "              \\|/ ____ \\|/\n"
2527 "              \"@'/ .. \\`@\"\n"
2528 "              /_| \\__/ |_\\\n"
2529 "                 \\__U_/\n");
2530 
2531 	printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter);
2532 	notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2533 	__asm__ __volatile__("flushw");
2534 	show_regs(regs);
2535 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
2536 	if (regs->tstate & TSTATE_PRIV) {
2537 		struct thread_info *tp = current_thread_info();
2538 		struct reg_window *rw = (struct reg_window *)
2539 			(regs->u_regs[UREG_FP] + STACK_BIAS);
2540 
2541 		/* Stop the back trace when we hit userland or we
2542 		 * find some badly aligned kernel stack.
2543 		 */
2544 		while (rw &&
2545 		       count++ < 30 &&
2546 		       kstack_valid(tp, (unsigned long) rw)) {
2547 			printk("Caller[%016lx]: %pS\n", rw->ins[7],
2548 			       (void *) rw->ins[7]);
2549 
2550 			rw = kernel_stack_up(rw);
2551 		}
2552 		instruction_dump ((unsigned int *) regs->tpc);
2553 	} else {
2554 		if (test_thread_flag(TIF_32BIT)) {
2555 			regs->tpc &= 0xffffffff;
2556 			regs->tnpc &= 0xffffffff;
2557 		}
2558 		user_instruction_dump ((unsigned int __user *) regs->tpc);
2559 	}
2560 	if (panic_on_oops)
2561 		panic("Fatal exception");
2562 	make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV);
2563 }
2564 EXPORT_SYMBOL(die_if_kernel);
2565 
2566 #define VIS_OPCODE_MASK	((0x3 << 30) | (0x3f << 19))
2567 #define VIS_OPCODE_VAL	((0x2 << 30) | (0x36 << 19))
2568 
do_illegal_instruction(struct pt_regs * regs)2569 void do_illegal_instruction(struct pt_regs *regs)
2570 {
2571 	enum ctx_state prev_state = exception_enter();
2572 	unsigned long pc = regs->tpc;
2573 	unsigned long tstate = regs->tstate;
2574 	u32 insn;
2575 
2576 	if (notify_die(DIE_TRAP, "illegal instruction", regs,
2577 		       0, 0x10, SIGILL) == NOTIFY_STOP)
2578 		goto out;
2579 
2580 	if (tstate & TSTATE_PRIV)
2581 		die_if_kernel("Kernel illegal instruction", regs);
2582 	if (test_thread_flag(TIF_32BIT))
2583 		pc = (u32)pc;
2584 	if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2585 		if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2586 			if (handle_popc(insn, regs))
2587 				goto out;
2588 		} else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2589 			if (handle_ldf_stq(insn, regs))
2590 				goto out;
2591 		} else if (tlb_type == hypervisor) {
2592 			if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
2593 				if (!vis_emul(regs, insn))
2594 					goto out;
2595 			} else {
2596 				struct fpustate *f = FPUSTATE;
2597 
2598 				/* On UltraSPARC T2 and later, FPU insns which
2599 				 * are not implemented in HW signal an illegal
2600 				 * instruction trap and do not set the FP Trap
2601 				 * Trap in the %fsr to unimplemented_FPop.
2602 				 */
2603 				if (do_mathemu(regs, f, true))
2604 					goto out;
2605 			}
2606 		}
2607 	}
2608 	force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc);
2609 out:
2610 	exception_exit(prev_state);
2611 }
2612 
mem_address_unaligned(struct pt_regs * regs,unsigned long sfar,unsigned long sfsr)2613 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2614 {
2615 	enum ctx_state prev_state = exception_enter();
2616 
2617 	if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2618 		       0, 0x34, SIGSEGV) == NOTIFY_STOP)
2619 		goto out;
2620 
2621 	if (regs->tstate & TSTATE_PRIV) {
2622 		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2623 		goto out;
2624 	}
2625 	if (is_no_fault_exception(regs))
2626 		return;
2627 
2628 	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)sfar);
2629 out:
2630 	exception_exit(prev_state);
2631 }
2632 
sun4v_do_mna(struct pt_regs * regs,unsigned long addr,unsigned long type_ctx)2633 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
2634 {
2635 	if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2636 		       0, 0x34, SIGSEGV) == NOTIFY_STOP)
2637 		return;
2638 
2639 	if (regs->tstate & TSTATE_PRIV) {
2640 		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2641 		return;
2642 	}
2643 	if (is_no_fault_exception(regs))
2644 		return;
2645 
2646 	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) addr);
2647 }
2648 
2649 /* sun4v_mem_corrupt_detect_precise() - Handle precise exception on an ADI
2650  * tag mismatch.
2651  *
2652  * ADI version tag mismatch on a load from memory always results in a
2653  * precise exception. Tag mismatch on a store to memory will result in
2654  * precise exception if MCDPER or PMCDPER is set to 1.
2655  */
sun4v_mem_corrupt_detect_precise(struct pt_regs * regs,unsigned long addr,unsigned long context)2656 void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, unsigned long addr,
2657 				      unsigned long context)
2658 {
2659 	if (notify_die(DIE_TRAP, "memory corruption precise exception", regs,
2660 		       0, 0x8, SIGSEGV) == NOTIFY_STOP)
2661 		return;
2662 
2663 	if (regs->tstate & TSTATE_PRIV) {
2664 		/* MCD exception could happen because the task was running
2665 		 * a system call with MCD enabled and passed a non-versioned
2666 		 * pointer or pointer with bad version tag to  the system
2667 		 * call.
2668 		 */
2669 		const struct exception_table_entry *entry;
2670 
2671 		entry = search_exception_tables(regs->tpc);
2672 		if (entry) {
2673 			/* Looks like a bad syscall parameter */
2674 #ifdef DEBUG_EXCEPTIONS
2675 			pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n",
2676 				 regs->tpc);
2677 			pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
2678 				 regs->tpc, entry->fixup);
2679 #endif
2680 			regs->tpc = entry->fixup;
2681 			regs->tnpc = regs->tpc + 4;
2682 			return;
2683 		}
2684 		pr_emerg("%s: ADDR[%016lx] CTX[%lx], going.\n",
2685 			 __func__, addr, context);
2686 		die_if_kernel("MCD precise", regs);
2687 	}
2688 
2689 	if (test_thread_flag(TIF_32BIT)) {
2690 		regs->tpc &= 0xffffffff;
2691 		regs->tnpc &= 0xffffffff;
2692 	}
2693 	force_sig_fault(SIGSEGV, SEGV_ADIPERR, (void __user *)addr);
2694 }
2695 
do_privop(struct pt_regs * regs)2696 void do_privop(struct pt_regs *regs)
2697 {
2698 	enum ctx_state prev_state = exception_enter();
2699 
2700 	if (notify_die(DIE_TRAP, "privileged operation", regs,
2701 		       0, 0x11, SIGILL) == NOTIFY_STOP)
2702 		goto out;
2703 
2704 	if (test_thread_flag(TIF_32BIT)) {
2705 		regs->tpc &= 0xffffffff;
2706 		regs->tnpc &= 0xffffffff;
2707 	}
2708 	force_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)regs->tpc);
2709 out:
2710 	exception_exit(prev_state);
2711 }
2712 
do_privact(struct pt_regs * regs)2713 void do_privact(struct pt_regs *regs)
2714 {
2715 	do_privop(regs);
2716 }
2717 
2718 /* Trap level 1 stuff or other traps we should never see... */
do_cee(struct pt_regs * regs)2719 void do_cee(struct pt_regs *regs)
2720 {
2721 	exception_enter();
2722 	die_if_kernel("TL0: Cache Error Exception", regs);
2723 }
2724 
do_div0_tl1(struct pt_regs * regs)2725 void do_div0_tl1(struct pt_regs *regs)
2726 {
2727 	exception_enter();
2728 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2729 	die_if_kernel("TL1: DIV0 Exception", regs);
2730 }
2731 
do_fpieee_tl1(struct pt_regs * regs)2732 void do_fpieee_tl1(struct pt_regs *regs)
2733 {
2734 	exception_enter();
2735 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2736 	die_if_kernel("TL1: FPU IEEE Exception", regs);
2737 }
2738 
do_fpother_tl1(struct pt_regs * regs)2739 void do_fpother_tl1(struct pt_regs *regs)
2740 {
2741 	exception_enter();
2742 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2743 	die_if_kernel("TL1: FPU Other Exception", regs);
2744 }
2745 
do_ill_tl1(struct pt_regs * regs)2746 void do_ill_tl1(struct pt_regs *regs)
2747 {
2748 	exception_enter();
2749 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2750 	die_if_kernel("TL1: Illegal Instruction Exception", regs);
2751 }
2752 
do_irq_tl1(struct pt_regs * regs)2753 void do_irq_tl1(struct pt_regs *regs)
2754 {
2755 	exception_enter();
2756 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2757 	die_if_kernel("TL1: IRQ Exception", regs);
2758 }
2759 
do_lddfmna_tl1(struct pt_regs * regs)2760 void do_lddfmna_tl1(struct pt_regs *regs)
2761 {
2762 	exception_enter();
2763 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2764 	die_if_kernel("TL1: LDDF Exception", regs);
2765 }
2766 
do_stdfmna_tl1(struct pt_regs * regs)2767 void do_stdfmna_tl1(struct pt_regs *regs)
2768 {
2769 	exception_enter();
2770 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2771 	die_if_kernel("TL1: STDF Exception", regs);
2772 }
2773 
do_paw(struct pt_regs * regs)2774 void do_paw(struct pt_regs *regs)
2775 {
2776 	exception_enter();
2777 	die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2778 }
2779 
do_paw_tl1(struct pt_regs * regs)2780 void do_paw_tl1(struct pt_regs *regs)
2781 {
2782 	exception_enter();
2783 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2784 	die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2785 }
2786 
do_vaw(struct pt_regs * regs)2787 void do_vaw(struct pt_regs *regs)
2788 {
2789 	exception_enter();
2790 	die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2791 }
2792 
do_vaw_tl1(struct pt_regs * regs)2793 void do_vaw_tl1(struct pt_regs *regs)
2794 {
2795 	exception_enter();
2796 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2797 	die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2798 }
2799 
do_tof_tl1(struct pt_regs * regs)2800 void do_tof_tl1(struct pt_regs *regs)
2801 {
2802 	exception_enter();
2803 	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2804 	die_if_kernel("TL1: Tag Overflow Exception", regs);
2805 }
2806 
do_getpsr(struct pt_regs * regs)2807 void do_getpsr(struct pt_regs *regs)
2808 {
2809 	regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2810 	regs->tpc   = regs->tnpc;
2811 	regs->tnpc += 4;
2812 	if (test_thread_flag(TIF_32BIT)) {
2813 		regs->tpc &= 0xffffffff;
2814 		regs->tnpc &= 0xffffffff;
2815 	}
2816 }
2817 
2818 u64 cpu_mondo_counter[NR_CPUS] = {0};
2819 struct trap_per_cpu trap_block[NR_CPUS];
2820 EXPORT_SYMBOL(trap_block);
2821 
2822 /* This can get invoked before sched_init() so play it super safe
2823  * and use hard_smp_processor_id().
2824  */
init_cur_cpu_trap(struct thread_info * t)2825 void notrace init_cur_cpu_trap(struct thread_info *t)
2826 {
2827 	int cpu = hard_smp_processor_id();
2828 	struct trap_per_cpu *p = &trap_block[cpu];
2829 
2830 	p->thread = t;
2831 	p->pgd_paddr = 0;
2832 }
2833 
2834 extern void thread_info_offsets_are_bolixed_dave(void);
2835 extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2836 extern void tsb_config_offsets_are_bolixed_dave(void);
2837 
2838 /* Only invoked on boot processor. */
trap_init(void)2839 void __init trap_init(void)
2840 {
2841 	/* Compile time sanity check. */
2842 	BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) ||
2843 		     TI_FLAGS != offsetof(struct thread_info, flags) ||
2844 		     TI_CPU != offsetof(struct thread_info, cpu) ||
2845 		     TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2846 		     TI_KSP != offsetof(struct thread_info, ksp) ||
2847 		     TI_FAULT_ADDR != offsetof(struct thread_info,
2848 					       fault_address) ||
2849 		     TI_KREGS != offsetof(struct thread_info, kregs) ||
2850 		     TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2851 		     TI_REG_WINDOW != offsetof(struct thread_info,
2852 					       reg_window) ||
2853 		     TI_RWIN_SPTRS != offsetof(struct thread_info,
2854 					       rwbuf_stkptrs) ||
2855 		     TI_GSR != offsetof(struct thread_info, gsr) ||
2856 		     TI_XFSR != offsetof(struct thread_info, xfsr) ||
2857 		     TI_PRE_COUNT != offsetof(struct thread_info,
2858 					      preempt_count) ||
2859 		     TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2860 		     TI_KUNA_REGS != offsetof(struct thread_info,
2861 					      kern_una_regs) ||
2862 		     TI_KUNA_INSN != offsetof(struct thread_info,
2863 					      kern_una_insn) ||
2864 		     TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2865 		     (TI_FPREGS & (64 - 1)));
2866 
2867 	BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu,
2868 						     thread) ||
2869 		     (TRAP_PER_CPU_PGD_PADDR !=
2870 		      offsetof(struct trap_per_cpu, pgd_paddr)) ||
2871 		     (TRAP_PER_CPU_CPU_MONDO_PA !=
2872 		      offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2873 		     (TRAP_PER_CPU_DEV_MONDO_PA !=
2874 		      offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2875 		     (TRAP_PER_CPU_RESUM_MONDO_PA !=
2876 		      offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
2877 		     (TRAP_PER_CPU_RESUM_KBUF_PA !=
2878 		      offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
2879 		     (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2880 		      offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
2881 		     (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2882 		      offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
2883 		     (TRAP_PER_CPU_FAULT_INFO !=
2884 		      offsetof(struct trap_per_cpu, fault_info)) ||
2885 		     (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2886 		      offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2887 		     (TRAP_PER_CPU_CPU_LIST_PA !=
2888 		      offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2889 		     (TRAP_PER_CPU_TSB_HUGE !=
2890 		      offsetof(struct trap_per_cpu, tsb_huge)) ||
2891 		     (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2892 		      offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2893 		     (TRAP_PER_CPU_IRQ_WORKLIST_PA !=
2894 		      offsetof(struct trap_per_cpu, irq_worklist_pa)) ||
2895 		     (TRAP_PER_CPU_CPU_MONDO_QMASK !=
2896 		      offsetof(struct trap_per_cpu, cpu_mondo_qmask)) ||
2897 		     (TRAP_PER_CPU_DEV_MONDO_QMASK !=
2898 		      offsetof(struct trap_per_cpu, dev_mondo_qmask)) ||
2899 		     (TRAP_PER_CPU_RESUM_QMASK !=
2900 		      offsetof(struct trap_per_cpu, resum_qmask)) ||
2901 		     (TRAP_PER_CPU_NONRESUM_QMASK !=
2902 		      offsetof(struct trap_per_cpu, nonresum_qmask)) ||
2903 		     (TRAP_PER_CPU_PER_CPU_BASE !=
2904 		      offsetof(struct trap_per_cpu, __per_cpu_base)));
2905 
2906 	BUILD_BUG_ON((TSB_CONFIG_TSB !=
2907 		      offsetof(struct tsb_config, tsb)) ||
2908 		     (TSB_CONFIG_RSS_LIMIT !=
2909 		      offsetof(struct tsb_config, tsb_rss_limit)) ||
2910 		     (TSB_CONFIG_NENTRIES !=
2911 		      offsetof(struct tsb_config, tsb_nentries)) ||
2912 		     (TSB_CONFIG_REG_VAL !=
2913 		      offsetof(struct tsb_config, tsb_reg_val)) ||
2914 		     (TSB_CONFIG_MAP_VADDR !=
2915 		      offsetof(struct tsb_config, tsb_map_vaddr)) ||
2916 		     (TSB_CONFIG_MAP_PTE !=
2917 		      offsetof(struct tsb_config, tsb_map_pte)));
2918 
2919 	/* Attach to the address space of init_task.  On SMP we
2920 	 * do this in smp.c:smp_callin for other cpus.
2921 	 */
2922 	mmgrab(&init_mm);
2923 	current->active_mm = &init_mm;
2924 }
2925