276 static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)  in uprobe_init_insn()  argument
280 	insn_init(insn, auprobe->insn, sizeof(auprobe->insn), x86_64);  in uprobe_init_insn()
334 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)  in riprel_analyze()  argument
349 		cursor = auprobe->insn + insn_offset_rex_prefix(insn);  in riprel_analyze()
368 		cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;  in riprel_analyze()
432 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;  in riprel_analyze()
435 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;  in riprel_analyze()
439 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;  in riprel_analyze()
446 	cursor = auprobe->insn + insn_offset_modrm(insn);  in riprel_analyze()
456 scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)  in scratch_reg()  argument
458 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)  in scratch_reg()
460 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)  in scratch_reg()
469 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_pre_xol()  argument
471 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {  in riprel_pre_xol()
473 		unsigned long *sr = scratch_reg(auprobe, regs);  in riprel_pre_xol()
476 		*sr = utask->vaddr + auprobe->defparam.ilen;  in riprel_pre_xol()
480 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_post_xol()  argument
482 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {  in riprel_post_xol()
484 		unsigned long *sr = scratch_reg(auprobe, regs);  in riprel_post_xol()
493 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)  in riprel_analyze()  argument
496 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_pre_xol()  argument
499 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_post_xol()  argument
519 static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_pre_xol_op()  argument
521 	riprel_pre_xol(auprobe, regs);  in default_pre_xol_op()
553 static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_post_xol_op()  argument
557 	riprel_post_xol(auprobe, regs);  in default_post_xol_op()
558 	if (auprobe->defparam.fixups & UPROBE_FIX_IP) {  in default_post_xol_op()
561 	} else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {  in default_post_xol_op()
563 		if (emulate_push_stack(regs, utask->vaddr + auprobe->defparam.ilen))  in default_post_xol_op()
567 	if (auprobe->defparam.fixups & UPROBE_FIX_SETF)  in default_post_xol_op()
573 static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_abort_op()  argument
575 	riprel_post_xol(auprobe, regs);  in default_abort_op()
584 static bool branch_is_call(struct arch_uprobe *auprobe)  in branch_is_call()  argument
586 	return auprobe->branch.opc1 == 0xe8;  in branch_is_call()
618 static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)  in check_jmp_cond()  argument
622 	switch (auprobe->branch.opc1) {  in check_jmp_cond()
637 static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in branch_emulate_op()  argument
639 	unsigned long new_ip = regs->ip += auprobe->branch.ilen;  in branch_emulate_op()
640 	unsigned long offs = (long)auprobe->branch.offs;  in branch_emulate_op()
642 	if (branch_is_call(auprobe)) {  in branch_emulate_op()
654 	} else if (!check_jmp_cond(auprobe, regs)) {  in branch_emulate_op()
662 static bool push_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in push_emulate_op()  argument
664 	unsigned long *src_ptr = (void *)regs + auprobe->push.reg_offset;  in push_emulate_op()
668 	regs->ip += auprobe->push.ilen;  in push_emulate_op()
672 static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in branch_post_xol_op()  argument
674 	BUG_ON(!branch_is_call(auprobe));  in branch_post_xol_op()
685 static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)  in branch_clear_offset()  argument
701 	memset(auprobe->insn + insn_offset_immediate(insn),  in branch_clear_offset()
715 static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)  in branch_setup_xol_ops()  argument
727 		branch_clear_offset(auprobe, insn);  in branch_setup_xol_ops()
754 	auprobe->branch.opc1 = opc1;  in branch_setup_xol_ops()
755 	auprobe->branch.ilen = insn->length;  in branch_setup_xol_ops()
756 	auprobe->branch.offs = insn->immediate.value;  in branch_setup_xol_ops()
758 	auprobe->ops = &branch_xol_ops;  in branch_setup_xol_ops()
763 static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)  in push_setup_xol_ops()  argument
837 	auprobe->push.reg_offset = reg_offset;  in push_setup_xol_ops()
838 	auprobe->push.ilen = insn->length;  in push_setup_xol_ops()
839 	auprobe->ops = &push_xol_ops;  in push_setup_xol_ops()
850 int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)  in arch_uprobe_analyze_insn()  argument
856 	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));  in arch_uprobe_analyze_insn()
860 	ret = branch_setup_xol_ops(auprobe, &insn);  in arch_uprobe_analyze_insn()
864 	ret = push_setup_xol_ops(auprobe, &insn);  in arch_uprobe_analyze_insn()
874 		auprobe->defparam.fixups |= UPROBE_FIX_SETF;  in arch_uprobe_analyze_insn()
897 		riprel_analyze(auprobe, &insn);  in arch_uprobe_analyze_insn()
900 	auprobe->defparam.ilen = insn.length;  in arch_uprobe_analyze_insn()
901 	auprobe->defparam.fixups |= fix_ip_or_call;  in arch_uprobe_analyze_insn()
903 	auprobe->ops = &default_xol_ops;  in arch_uprobe_analyze_insn()
912 int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_pre_xol()  argument
916 	if (auprobe->ops->pre_xol) {  in arch_uprobe_pre_xol()
917 		int err = auprobe->ops->pre_xol(auprobe, regs);  in arch_uprobe_pre_xol()
959 int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_post_xol()  argument
968 	if (auprobe->ops->post_xol) {  in arch_uprobe_post_xol()
969 		err = auprobe->ops->post_xol(auprobe, regs);  in arch_uprobe_post_xol()
1030 void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_abort_xol()  argument
1034 	if (auprobe->ops->abort)  in arch_uprobe_abort_xol()
1035 		auprobe->ops->abort(auprobe, regs);  in arch_uprobe_abort_xol()
1044 static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)  in __skip_sstep()  argument
1046 	if (auprobe->ops->emulate)  in __skip_sstep()
1047 		return auprobe->ops->emulate(auprobe, regs);  in __skip_sstep()
1051 bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_skip_sstep()  argument
1053 	bool ret = __skip_sstep(auprobe, regs);  in arch_uprobe_skip_sstep()