54 extern int do_lq(unsigned long ea, unsigned long *regs);
55 extern int do_stq(unsigned long ea, unsigned long val0, unsigned long val1);
56 extern int do_lqarx(unsigned long ea, unsigned long *regs);
57 extern int do_stqcx(unsigned long ea, unsigned long val0, unsigned long val1,
106 				       unsigned long ea, int nb)  in address_ok()  argument
110 	if (access_ok((void __user *)ea, nb))  in address_ok()
112 	if (access_ok((void __user *)ea, 1))  in address_ok()
116 		regs->dar = ea;  in address_ok()
127 	unsigned long ea;  in dform_ea()  local
130 	ea = (signed short) instr;		/* sign-extend */  in dform_ea()
132 		ea += regs->gpr[ra];  in dform_ea()
134 	return ea;  in dform_ea()
145 	unsigned long ea;  in dsform_ea()  local
148 	ea = (signed short) (instr & ~3);	/* sign-extend */  in dsform_ea()
150 		ea += regs->gpr[ra];  in dsform_ea()
152 	return ea;  in dsform_ea()
162 	unsigned long ea;  in dqform_ea()  local
165 	ea = (signed short) (instr & ~0xf);	/* sign-extend */  in dqform_ea()
167 		ea += regs->gpr[ra];  in dqform_ea()
169 	return ea;  in dqform_ea()
180 	unsigned long ea;  in xform_ea()  local
184 	ea = regs->gpr[rb];  in xform_ea()
186 		ea += regs->gpr[ra];  in xform_ea()
188 	return ea;  in xform_ea()
201 	unsigned long ea, d0, d1, d;  in mlsd_8lsd_ea()  local
214 	ea = (signed int)dd;  in mlsd_8lsd_ea()
215 	ea = (ea << 2) | (d & 0x3);  in mlsd_8lsd_ea()
218 		ea += regs->gpr[ra];  in mlsd_8lsd_ea()
222 		ea += regs->nip;  in mlsd_8lsd_ea()
229 	return ea;  in mlsd_8lsd_ea()
301 __read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs)  in __read_mem_aligned()  argument
307 		unsafe_get_user(x, (unsigned char __user *)ea, Efault);  in __read_mem_aligned()
310 		unsafe_get_user(x, (unsigned short __user *)ea, Efault);  in __read_mem_aligned()
313 		unsafe_get_user(x, (unsigned int __user *)ea, Efault);  in __read_mem_aligned()
317 		unsafe_get_user(x, (unsigned long __user *)ea, Efault);  in __read_mem_aligned()
325 	regs->dar = ea;  in __read_mem_aligned()
330 read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs)  in read_mem_aligned()  argument
334 	if (is_kernel_addr(ea))  in read_mem_aligned()
335 		return __read_mem_aligned(dest, ea, nb, regs);  in read_mem_aligned()
337 	if (user_read_access_begin((void __user *)ea, nb)) {  in read_mem_aligned()
338 		err = __read_mem_aligned(dest, ea, nb, regs);  in read_mem_aligned()
342 		regs->dar = ea;  in read_mem_aligned()
352 static __always_inline int __copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)  in __copy_mem_in()  argument
357 		c = max_align(ea);  in __copy_mem_in()
362 			unsafe_get_user(*dest, (u8 __user *)ea, Efault);  in __copy_mem_in()
365 			unsafe_get_user(*(u16 *)dest, (u16 __user *)ea, Efault);  in __copy_mem_in()
368 			unsafe_get_user(*(u32 *)dest, (u32 __user *)ea, Efault);  in __copy_mem_in()
372 			unsafe_get_user(*(u64 *)dest, (u64 __user *)ea, Efault);  in __copy_mem_in()
377 		ea += c;  in __copy_mem_in()
382 	regs->dar = ea;  in __copy_mem_in()
386 static nokprobe_inline int copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)  in copy_mem_in()  argument
390 	if (is_kernel_addr(ea))  in copy_mem_in()
391 		return __copy_mem_in(dest, ea, nb, regs);  in copy_mem_in()
393 	if (user_read_access_begin((void __user *)ea, nb)) {  in copy_mem_in()
394 		err = __copy_mem_in(dest, ea, nb, regs);  in copy_mem_in()
398 		regs->dar = ea;  in copy_mem_in()
405 					      unsigned long ea, int nb,  in read_mem_unaligned()  argument
417 	err = copy_mem_in(&u.b[i], ea, nb, regs);  in read_mem_unaligned()
428 static int read_mem(unsigned long *dest, unsigned long ea, int nb,  in read_mem()  argument
431 	if (!address_ok(regs, ea, nb))  in read_mem()
433 	if ((ea & (nb - 1)) == 0)  in read_mem()
434 		return read_mem_aligned(dest, ea, nb, regs);  in read_mem()
435 	return read_mem_unaligned(dest, ea, nb, regs);  in read_mem()
440 __write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs)  in __write_mem_aligned()  argument
444 		unsafe_put_user(val, (unsigned char __user *)ea, Efault);  in __write_mem_aligned()
447 		unsafe_put_user(val, (unsigned short __user *)ea, Efault);  in __write_mem_aligned()
450 		unsafe_put_user(val, (unsigned int __user *)ea, Efault);  in __write_mem_aligned()
454 		unsafe_put_user(val, (unsigned long __user *)ea, Efault);  in __write_mem_aligned()
461 	regs->dar = ea;  in __write_mem_aligned()
466 write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs)  in write_mem_aligned()  argument
470 	if (is_kernel_addr(ea))  in write_mem_aligned()
471 		return __write_mem_aligned(val, ea, nb, regs);  in write_mem_aligned()
473 	if (user_write_access_begin((void __user *)ea, nb)) {  in write_mem_aligned()
474 		err = __write_mem_aligned(val, ea, nb, regs);  in write_mem_aligned()
478 		regs->dar = ea;  in write_mem_aligned()
488 static nokprobe_inline int __copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)  in __copy_mem_out()  argument
493 		c = max_align(ea);  in __copy_mem_out()
498 			unsafe_put_user(*dest, (u8 __user *)ea, Efault);  in __copy_mem_out()
501 			unsafe_put_user(*(u16 *)dest, (u16 __user *)ea, Efault);  in __copy_mem_out()
504 			unsafe_put_user(*(u32 *)dest, (u32 __user *)ea, Efault);  in __copy_mem_out()
508 			unsafe_put_user(*(u64 *)dest, (u64 __user *)ea, Efault);  in __copy_mem_out()
513 		ea += c;  in __copy_mem_out()
518 	regs->dar = ea;  in __copy_mem_out()
522 static nokprobe_inline int copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)  in copy_mem_out()  argument
526 	if (is_kernel_addr(ea))  in copy_mem_out()
527 		return __copy_mem_out(dest, ea, nb, regs);  in copy_mem_out()
529 	if (user_write_access_begin((void __user *)ea, nb)) {  in copy_mem_out()
530 		err = __copy_mem_out(dest, ea, nb, regs);  in copy_mem_out()
534 		regs->dar = ea;  in copy_mem_out()
541 					       unsigned long ea, int nb,  in write_mem_unaligned()  argument
552 	return copy_mem_out(&u.b[i], ea, nb, regs);  in write_mem_unaligned()
559 static int write_mem(unsigned long val, unsigned long ea, int nb,  in write_mem()  argument
562 	if (!address_ok(regs, ea, nb))  in write_mem()
564 	if ((ea & (nb - 1)) == 0)  in write_mem()
565 		return write_mem_aligned(val, ea, nb, regs);  in write_mem()
566 	return write_mem_unaligned(val, ea, nb, regs);  in write_mem()
575 static int do_fp_load(struct instruction_op *op, unsigned long ea,  in do_fp_load()  argument
589 	if (!address_ok(regs, ea, nb))  in do_fp_load()
592 	err = copy_mem_in(u.b, ea, nb, regs);  in do_fp_load()
626 static int do_fp_store(struct instruction_op *op, unsigned long ea,  in do_fp_store()  argument
639 	if (!address_ok(regs, ea, nb))  in do_fp_store()
666 	return copy_mem_out(u.b, ea, nb, regs);  in do_fp_store()
673 static nokprobe_inline int do_vec_load(int rn, unsigned long ea,  in do_vec_load()  argument
683 	if (!address_ok(regs, ea & ~0xfUL, 16))  in do_vec_load()
686 	ea &= ~(size - 1);  in do_vec_load()
687 	err = copy_mem_in(&u.b[ea & 0xf], ea, size, regs);  in do_vec_load()
691 		do_byte_reverse(&u.b[ea & 0xf], size);  in do_vec_load()
701 static nokprobe_inline int do_vec_store(int rn, unsigned long ea,  in do_vec_store()  argument
710 	if (!address_ok(regs, ea & ~0xfUL, 16))  in do_vec_store()
713 	ea &= ~(size - 1);  in do_vec_store()
722 		do_byte_reverse(&u.b[ea & 0xf], size);  in do_vec_store()
723 	return copy_mem_out(&u.b[ea & 0xf], ea, size, regs);  in do_vec_store()
728 static nokprobe_inline int emulate_lq(struct pt_regs *regs, unsigned long ea,  in emulate_lq()  argument
733 	if (!address_ok(regs, ea, 16))  in emulate_lq()
736 	if ((ea & 0xf) == 0) {  in emulate_lq()
737 		err = do_lq(ea, &regs->gpr[reg]);  in emulate_lq()
739 		err = read_mem(&regs->gpr[reg + IS_LE], ea, 8, regs);  in emulate_lq()
741 			err = read_mem(&regs->gpr[reg + IS_BE], ea + 8, 8, regs);  in emulate_lq()
748 static nokprobe_inline int emulate_stq(struct pt_regs *regs, unsigned long ea,  in emulate_stq()  argument
754 	if (!address_ok(regs, ea, 16))  in emulate_stq()
762 	if ((ea & 0xf) == 0)  in emulate_stq()
763 		return do_stq(ea, vals[0], vals[1]);  in emulate_stq()
765 	err = write_mem(vals[IS_LE], ea, 8, regs);  in emulate_stq()
767 		err = write_mem(vals[IS_BE], ea + 8, 8, regs);  in emulate_stq()
952 				       unsigned long ea, struct pt_regs *regs,  in do_vsx_load()  argument
961 	if (!address_ok(regs, ea, size) || copy_mem_in(mem, ea, size, regs))  in do_vsx_load()
999 					unsigned long ea, struct pt_regs *regs,  in do_vsx_store()  argument
1008 	if (!address_ok(regs, ea, size))  in do_vsx_store()
1042 	return  copy_mem_out(mem, ea, size, regs);  in do_vsx_store()
1046 static int __emulate_dcbz(unsigned long ea)  in __emulate_dcbz()  argument
1052 		unsafe_put_user(0, (unsigned long __user *)(ea + i), Efault);  in __emulate_dcbz()
1060 int emulate_dcbz(unsigned long ea, struct pt_regs *regs)  in emulate_dcbz()  argument
1065 	ea = truncate_if_32bit(regs->msr, ea);  in emulate_dcbz()
1066 	ea &= ~(size - 1);  in emulate_dcbz()
1067 	if (!address_ok(regs, ea, size))  in emulate_dcbz()
1070 	if (is_kernel_addr(ea)) {  in emulate_dcbz()
1071 		err = __emulate_dcbz(ea);  in emulate_dcbz()
1072 	} else if (user_write_access_begin((void __user *)ea, size)) {  in emulate_dcbz()
1073 		err = __emulate_dcbz(ea);  in emulate_dcbz()
1080 		regs->dar = ea;  in emulate_dcbz()
2231 			op->ea = xform_ea(word, regs);  in analyse_instr()
2236 			op->ea = xform_ea(word, regs);  in analyse_instr()
2241 			op->ea = xform_ea(word, regs);  in analyse_instr()
2247 			op->ea = xform_ea(word, regs);  in analyse_instr()
2253 			op->ea = xform_ea(word, regs);  in analyse_instr()
2258 			op->ea = xform_ea(word, regs);  in analyse_instr()
2277 		op->ea = xform_ea(word, regs);  in analyse_instr()
2442 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2509 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2555 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2604 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2750 		op->ea = dform_ea(word, regs);  in analyse_instr()
2756 		op->ea = dform_ea(word, regs);  in analyse_instr()
2762 		op->ea = dform_ea(word, regs);  in analyse_instr()
2768 		op->ea = dform_ea(word, regs);  in analyse_instr()
2774 		op->ea = dform_ea(word, regs);  in analyse_instr()
2780 		op->ea = dform_ea(word, regs);  in analyse_instr()
2786 		op->ea = dform_ea(word, regs);  in analyse_instr()
2793 		op->ea = dform_ea(word, regs);  in analyse_instr()
2798 		op->ea = dform_ea(word, regs);  in analyse_instr()
2805 		op->ea = dform_ea(word, regs);  in analyse_instr()
2811 		op->ea = dform_ea(word, regs);  in analyse_instr()
2817 		op->ea = dform_ea(word, regs);  in analyse_instr()
2823 		op->ea = dform_ea(word, regs);  in analyse_instr()
2831 		op->ea = dqform_ea(word, regs);  in analyse_instr()
2837 		op->ea = dsform_ea(word, regs);  in analyse_instr()
2866 		op->ea = dsform_ea(word, regs);  in analyse_instr()
2885 		op->ea = dqform_ea(word, regs);  in analyse_instr()
2902 			op->ea = dsform_ea(word, regs);  in analyse_instr()
2909 			op->ea = dqform_ea(word, regs);  in analyse_instr()
2921 			op->ea = dsform_ea(word, regs);  in analyse_instr()
2932 			op->ea = dsform_ea(word, regs);  in analyse_instr()
2942 			op->ea = dqform_ea(word, regs);  in analyse_instr()
2955 		op->ea = dsform_ea(word, regs);  in analyse_instr()
2986 			op->ea = mlsd_8lsd_ea(word, suffix, regs);  in analyse_instr()
3066 			op->ea = mlsd_8lsd_ea(word, suffix, regs);  in analyse_instr()
3175 static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs)  in handle_stack_update()  argument
3319 	unsigned long ea;  in emulate_loadstore()  local
3326 	ea = truncate_if_32bit(regs->msr, op->ea);  in emulate_loadstore()
3330 		if (ea & (size - 1))  in emulate_loadstore()
3332 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
3339 			__get_user_asmx(val, ea, err, "lbarx");  in emulate_loadstore()
3342 			__get_user_asmx(val, ea, err, "lharx");  in emulate_loadstore()
3346 			__get_user_asmx(val, ea, err, "lwarx");  in emulate_loadstore()
3350 			__get_user_asmx(val, ea, err, "ldarx");  in emulate_loadstore()
3353 			err = do_lqarx(ea, &regs->gpr[op->reg]);  in emulate_loadstore()
3360 			regs->dar = ea;  in emulate_loadstore()
3368 		if (ea & (size - 1))  in emulate_loadstore()
3370 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
3376 			__put_user_asmx(op->val, ea, err, "stbcx.", cr);  in emulate_loadstore()
3379 			__put_user_asmx(op->val, ea, err, "sthcx.", cr);  in emulate_loadstore()
3383 			__put_user_asmx(op->val, ea, err, "stwcx.", cr);  in emulate_loadstore()
3387 			__put_user_asmx(op->val, ea, err, "stdcx.", cr);  in emulate_loadstore()
3390 			err = do_stqcx(ea, regs->gpr[op->reg],  in emulate_loadstore()
3402 			regs->dar = ea;  in emulate_loadstore()
3408 			err = emulate_lq(regs, ea, op->reg, cross_endian);  in emulate_loadstore()
3412 		err = read_mem(&regs->gpr[op->reg], ea, size, regs);  in emulate_loadstore()
3431 		err = do_fp_load(op, ea, regs, cross_endian);  in emulate_loadstore()
3438 		err = do_vec_load(op->reg, ea, size, regs, cross_endian);  in emulate_loadstore()
3453 		err = do_vsx_load(op, ea, regs, cross_endian);  in emulate_loadstore()
3458 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
3467 			err = copy_mem_in((u8 *) &v32, ea, nb, regs);  in emulate_loadstore()
3473 			ea += 4;  in emulate_loadstore()
3482 			err = emulate_stq(regs, ea, op->reg, cross_endian);  in emulate_loadstore()
3489 		    ea >= regs->gpr[1] - STACK_INT_FRAME_SIZE) {  in emulate_loadstore()
3490 			err = handle_stack_update(ea, regs);  in emulate_loadstore()
3495 		err = write_mem(op->val, ea, size, regs);  in emulate_loadstore()
3502 		err = do_fp_store(op, ea, regs, cross_endian);  in emulate_loadstore()
3509 		err = do_vec_store(op->reg, ea, size, regs, cross_endian);  in emulate_loadstore()
3524 		err = do_vsx_store(op, ea, regs, cross_endian);  in emulate_loadstore()
3529 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
3540 			err = copy_mem_out((u8 *) &v32, ea, nb, regs);  in emulate_loadstore()
3543 			ea += 4;  in emulate_loadstore()
3557 		regs->gpr[op->update_reg] = op->ea;  in emulate_loadstore()
3575 	unsigned long ea;  in emulate_step()  local
3597 		ea = truncate_if_32bit(regs->msr, op.ea);  in emulate_step()
3598 		if (!address_ok(regs, ea, 8))  in emulate_step()
3602 			__cacheop_user_asmx(ea, err, "dcbst");  in emulate_step()
3605 			__cacheop_user_asmx(ea, err, "dcbf");  in emulate_step()
3609 				prefetchw((void *) ea);  in emulate_step()
3613 				prefetch((void *) ea);  in emulate_step()
3616 			__cacheop_user_asmx(ea, err, "icbi");  in emulate_step()
3619 			err = emulate_dcbz(ea, regs);  in emulate_step()
3623 			regs->dar = ea;  in emulate_step()