51 extern int do_lq(unsigned long ea, unsigned long *regs);
52 extern int do_stq(unsigned long ea, unsigned long val0, unsigned long val1);
53 extern int do_lqarx(unsigned long ea, unsigned long *regs);
54 extern int do_stqcx(unsigned long ea, unsigned long val0, unsigned long val1,
105 				       unsigned long ea, int nb)  in address_ok()  argument
109 	if (__access_ok(ea, nb, USER_DS))  in address_ok()
111 	if (__access_ok(ea, 1, USER_DS))  in address_ok()
115 		regs->dar = ea;  in address_ok()
126 	unsigned long ea;  in dform_ea()  local
129 	ea = (signed short) instr;		/* sign-extend */  in dform_ea()
131 		ea += regs->gpr[ra];  in dform_ea()
133 	return ea;  in dform_ea()
144 	unsigned long ea;  in dsform_ea()  local
147 	ea = (signed short) (instr & ~3);	/* sign-extend */  in dsform_ea()
149 		ea += regs->gpr[ra];  in dsform_ea()
151 	return ea;  in dsform_ea()
161 	unsigned long ea;  in dqform_ea()  local
164 	ea = (signed short) (instr & ~0xf);	/* sign-extend */  in dqform_ea()
166 		ea += regs->gpr[ra];  in dqform_ea()
168 	return ea;  in dqform_ea()
179 	unsigned long ea;  in xform_ea()  local
183 	ea = regs->gpr[rb];  in xform_ea()
185 		ea += regs->gpr[ra];  in xform_ea()
187 	return ea;  in xform_ea()
246 					    unsigned long ea, int nb,  in read_mem_aligned()  argument
254 		err = __get_user(x, (unsigned char __user *) ea);  in read_mem_aligned()
257 		err = __get_user(x, (unsigned short __user *) ea);  in read_mem_aligned()
260 		err = __get_user(x, (unsigned int __user *) ea);  in read_mem_aligned()
264 		err = __get_user(x, (unsigned long __user *) ea);  in read_mem_aligned()
271 		regs->dar = ea;  in read_mem_aligned()
279 static nokprobe_inline int copy_mem_in(u8 *dest, unsigned long ea, int nb,  in copy_mem_in()  argument
286 		c = max_align(ea);  in copy_mem_in()
291 			err = __get_user(*dest, (unsigned char __user *) ea);  in copy_mem_in()
295 					 (unsigned short __user *) ea);  in copy_mem_in()
299 					 (unsigned int __user *) ea);  in copy_mem_in()
304 					 (unsigned long __user *) ea);  in copy_mem_in()
309 			regs->dar = ea;  in copy_mem_in()
313 		ea += c;  in copy_mem_in()
319 					      unsigned long ea, int nb,  in read_mem_unaligned()  argument
331 	err = copy_mem_in(&u.b[i], ea, nb, regs);  in read_mem_unaligned()
342 static int read_mem(unsigned long *dest, unsigned long ea, int nb,  in read_mem()  argument
345 	if (!address_ok(regs, ea, nb))  in read_mem()
347 	if ((ea & (nb - 1)) == 0)  in read_mem()
348 		return read_mem_aligned(dest, ea, nb, regs);  in read_mem()
349 	return read_mem_unaligned(dest, ea, nb, regs);  in read_mem()
354 					     unsigned long ea, int nb,  in write_mem_aligned()  argument
361 		err = __put_user(val, (unsigned char __user *) ea);  in write_mem_aligned()
364 		err = __put_user(val, (unsigned short __user *) ea);  in write_mem_aligned()
367 		err = __put_user(val, (unsigned int __user *) ea);  in write_mem_aligned()
371 		err = __put_user(val, (unsigned long __user *) ea);  in write_mem_aligned()
376 		regs->dar = ea;  in write_mem_aligned()
384 static nokprobe_inline int copy_mem_out(u8 *dest, unsigned long ea, int nb,  in copy_mem_out()  argument
391 		c = max_align(ea);  in copy_mem_out()
396 			err = __put_user(*dest, (unsigned char __user *) ea);  in copy_mem_out()
400 					 (unsigned short __user *) ea);  in copy_mem_out()
404 					 (unsigned int __user *) ea);  in copy_mem_out()
409 					 (unsigned long __user *) ea);  in copy_mem_out()
414 			regs->dar = ea;  in copy_mem_out()
418 		ea += c;  in copy_mem_out()
424 					       unsigned long ea, int nb,  in write_mem_unaligned()  argument
435 	return copy_mem_out(&u.b[i], ea, nb, regs);  in write_mem_unaligned()
442 static int write_mem(unsigned long val, unsigned long ea, int nb,  in write_mem()  argument
445 	if (!address_ok(regs, ea, nb))  in write_mem()
447 	if ((ea & (nb - 1)) == 0)  in write_mem()
448 		return write_mem_aligned(val, ea, nb, regs);  in write_mem()
449 	return write_mem_unaligned(val, ea, nb, regs);  in write_mem()
458 static int do_fp_load(struct instruction_op *op, unsigned long ea,  in do_fp_load()  argument
472 	if (!address_ok(regs, ea, nb))  in do_fp_load()
475 	err = copy_mem_in(u.b, ea, nb, regs);  in do_fp_load()
509 static int do_fp_store(struct instruction_op *op, unsigned long ea,  in do_fp_store()  argument
522 	if (!address_ok(regs, ea, nb))  in do_fp_store()
549 	return copy_mem_out(u.b, ea, nb, regs);  in do_fp_store()
556 static nokprobe_inline int do_vec_load(int rn, unsigned long ea,  in do_vec_load()  argument
566 	if (!address_ok(regs, ea & ~0xfUL, 16))  in do_vec_load()
569 	ea &= ~(size - 1);  in do_vec_load()
570 	err = copy_mem_in(&u.b[ea & 0xf], ea, size, regs);  in do_vec_load()
574 		do_byte_reverse(&u.b[ea & 0xf], size);  in do_vec_load()
584 static nokprobe_inline int do_vec_store(int rn, unsigned long ea,  in do_vec_store()  argument
593 	if (!address_ok(regs, ea & ~0xfUL, 16))  in do_vec_store()
596 	ea &= ~(size - 1);  in do_vec_store()
605 		do_byte_reverse(&u.b[ea & 0xf], size);  in do_vec_store()
606 	return copy_mem_out(&u.b[ea & 0xf], ea, size, regs);  in do_vec_store()
611 static nokprobe_inline int emulate_lq(struct pt_regs *regs, unsigned long ea,  in emulate_lq()  argument
616 	if (!address_ok(regs, ea, 16))  in emulate_lq()
619 	if ((ea & 0xf) == 0) {  in emulate_lq()
620 		err = do_lq(ea, &regs->gpr[reg]);  in emulate_lq()
622 		err = read_mem(&regs->gpr[reg + IS_LE], ea, 8, regs);  in emulate_lq()
624 			err = read_mem(&regs->gpr[reg + IS_BE], ea + 8, 8, regs);  in emulate_lq()
631 static nokprobe_inline int emulate_stq(struct pt_regs *regs, unsigned long ea,  in emulate_stq()  argument
637 	if (!address_ok(regs, ea, 16))  in emulate_stq()
645 	if ((ea & 0xf) == 0)  in emulate_stq()
646 		return do_stq(ea, vals[0], vals[1]);  in emulate_stq()
648 	err = write_mem(vals[IS_LE], ea, 8, regs);  in emulate_stq()
650 		err = write_mem(vals[IS_BE], ea + 8, 8, regs);  in emulate_stq()
817 				       unsigned long ea, struct pt_regs *regs,  in do_vsx_load()  argument
825 	if (!address_ok(regs, ea, size) || copy_mem_in(mem, ea, size, regs))  in do_vsx_load()
849 					unsigned long ea, struct pt_regs *regs,  in do_vsx_store()  argument
857 	if (!address_ok(regs, ea, size))  in do_vsx_store()
877 	return  copy_mem_out(mem, ea, size, regs);  in do_vsx_store()
881 int emulate_dcbz(unsigned long ea, struct pt_regs *regs)  in emulate_dcbz()  argument
889 		ea &= 0xffffffffUL;  in emulate_dcbz()
893 	ea &= ~(size - 1);  in emulate_dcbz()
894 	if (!address_ok(regs, ea, size))  in emulate_dcbz()
897 		err = __put_user(0, (unsigned long __user *) (ea + i));  in emulate_dcbz()
899 			regs->dar = ea;  in emulate_dcbz()
1982 			op->ea = xform_ea(instr, regs);  in analyse_instr()
1987 			op->ea = xform_ea(instr, regs);  in analyse_instr()
1992 			op->ea = xform_ea(instr, regs);  in analyse_instr()
1998 			op->ea = xform_ea(instr, regs);  in analyse_instr()
2004 			op->ea = xform_ea(instr, regs);  in analyse_instr()
2009 			op->ea = xform_ea(instr, regs);  in analyse_instr()
2028 		op->ea = xform_ea(instr, regs);  in analyse_instr()
2193 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2260 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2302 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2337 			op->ea = ra ? regs->gpr[ra] : 0;  in analyse_instr()
2460 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2466 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2472 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2478 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2484 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2490 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2496 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2503 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2508 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2515 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2521 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2527 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2533 		op->ea = dform_ea(instr, regs);  in analyse_instr()
2541 		op->ea = dqform_ea(instr, regs);  in analyse_instr()
2547 		op->ea = dsform_ea(instr, regs);  in analyse_instr()
2572 		op->ea = dsform_ea(instr, regs);  in analyse_instr()
2592 			op->ea = dsform_ea(instr, regs);  in analyse_instr()
2597 			op->ea = dqform_ea(instr, regs);  in analyse_instr()
2607 			op->ea = dsform_ea(instr, regs);  in analyse_instr()
2616 			op->ea = dsform_ea(instr, regs);  in analyse_instr()
2624 			op->ea = dqform_ea(instr, regs);  in analyse_instr()
2637 		op->ea = dsform_ea(instr, regs);  in analyse_instr()
2702 static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs)  in handle_stack_update()  argument
2708 	if (ea - STACK_INT_FRAME_SIZE <= current->thread.ksp_limit) {  in handle_stack_update()
2853 	unsigned long ea;  in emulate_loadstore()  local
2860 	ea = truncate_if_32bit(regs->msr, op->ea);  in emulate_loadstore()
2864 		if (ea & (size - 1))  in emulate_loadstore()
2866 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
2873 			__get_user_asmx(val, ea, err, "lbarx");  in emulate_loadstore()
2876 			__get_user_asmx(val, ea, err, "lharx");  in emulate_loadstore()
2880 			__get_user_asmx(val, ea, err, "lwarx");  in emulate_loadstore()
2884 			__get_user_asmx(val, ea, err, "ldarx");  in emulate_loadstore()
2887 			err = do_lqarx(ea, &regs->gpr[op->reg]);  in emulate_loadstore()
2894 			regs->dar = ea;  in emulate_loadstore()
2902 		if (ea & (size - 1))  in emulate_loadstore()
2904 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
2910 			__put_user_asmx(op->val, ea, err, "stbcx.", cr);  in emulate_loadstore()
2913 			__put_user_asmx(op->val, ea, err, "stbcx.", cr);  in emulate_loadstore()
2917 			__put_user_asmx(op->val, ea, err, "stwcx.", cr);  in emulate_loadstore()
2921 			__put_user_asmx(op->val, ea, err, "stdcx.", cr);  in emulate_loadstore()
2924 			err = do_stqcx(ea, regs->gpr[op->reg],  in emulate_loadstore()
2936 			regs->dar = ea;  in emulate_loadstore()
2942 			err = emulate_lq(regs, ea, op->reg, cross_endian);  in emulate_loadstore()
2946 		err = read_mem(&regs->gpr[op->reg], ea, size, regs);  in emulate_loadstore()
2965 		err = do_fp_load(op, ea, regs, cross_endian);  in emulate_loadstore()
2972 		err = do_vec_load(op->reg, ea, size, regs, cross_endian);  in emulate_loadstore()
2987 		err = do_vsx_load(op, ea, regs, cross_endian);  in emulate_loadstore()
2992 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
3001 			err = copy_mem_in((u8 *) &v32, ea, nb, regs);  in emulate_loadstore()
3007 			ea += 4;  in emulate_loadstore()
3016 			err = emulate_stq(regs, ea, op->reg, cross_endian);  in emulate_loadstore()
3023 		    ea >= regs->gpr[1] - STACK_INT_FRAME_SIZE) {  in emulate_loadstore()
3024 			err = handle_stack_update(ea, regs);  in emulate_loadstore()
3029 		err = write_mem(op->val, ea, size, regs);  in emulate_loadstore()
3036 		err = do_fp_store(op, ea, regs, cross_endian);  in emulate_loadstore()
3043 		err = do_vec_store(op->reg, ea, size, regs, cross_endian);  in emulate_loadstore()
3058 		err = do_vsx_store(op, ea, regs, cross_endian);  in emulate_loadstore()
3063 		if (!address_ok(regs, ea, size))  in emulate_loadstore()
3074 			err = copy_mem_out((u8 *) &v32, ea, nb, regs);  in emulate_loadstore()
3077 			ea += 4;  in emulate_loadstore()
3091 		regs->gpr[op->update_reg] = op->ea;  in emulate_loadstore()
3109 	unsigned long ea;  in emulate_step()  local
3131 		ea = truncate_if_32bit(regs->msr, op.ea);  in emulate_step()
3132 		if (!address_ok(regs, ea, 8))  in emulate_step()
3136 			__cacheop_user_asmx(ea, err, "dcbst");  in emulate_step()
3139 			__cacheop_user_asmx(ea, err, "dcbf");  in emulate_step()
3143 				prefetchw((void *) ea);  in emulate_step()
3147 				prefetch((void *) ea);  in emulate_step()
3150 			__cacheop_user_asmx(ea, err, "icbi");  in emulate_step()
3153 			err = emulate_dcbz(ea, regs);  in emulate_step()
3157 			regs->dar = ea;  in emulate_step()