57 static int ftrace_calc_offset(long ip, long addr)  in ftrace_calc_offset()  argument
59 	return (int)(addr - ip);  in ftrace_calc_offset()
62 static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)  in ftrace_call_replace()  argument
67 	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);  in ftrace_call_replace()
82 static unsigned long text_ip_addr(unsigned long ip)  in text_ip_addr()  argument
92 	if (within(ip, (unsigned long)_text, (unsigned long)_etext))  in text_ip_addr()
93 		ip = (unsigned long)__va(__pa_symbol(ip));  in text_ip_addr()
95 	return ip;  in text_ip_addr()
104 ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,  in ftrace_modify_code_direct()  argument
120 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))  in ftrace_modify_code_direct()
127 	ip = text_ip_addr(ip);  in ftrace_modify_code_direct()
130 	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))  in ftrace_modify_code_direct()
142 	unsigned long ip = rec->ip;  in ftrace_make_nop()  local
144 	old = ftrace_call_replace(ip, addr);  in ftrace_make_nop()
156 		return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_nop()
168 	unsigned long ip = rec->ip;  in ftrace_make_call()  local
171 	new = ftrace_call_replace(ip, addr);  in ftrace_make_call()
174 	return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_call()
211 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
232 static int update_ftrace_func(unsigned long ip, void *new)  in update_ftrace_func()  argument
237 	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);  in update_ftrace_func()
239 	ftrace_update_func = ip;  in update_ftrace_func()
246 	ret = ftrace_modify_code(ip, old, new);  in update_ftrace_func()
255 	unsigned long ip = (unsigned long)(&ftrace_call);  in ftrace_update_ftrace_func()  local
259 	new = ftrace_call_replace(ip, (unsigned long)func);  in ftrace_update_ftrace_func()
260 	ret = update_ftrace_func(ip, new);  in ftrace_update_ftrace_func()
264 		ip = (unsigned long)(&ftrace_regs_call);  in ftrace_update_ftrace_func()
265 		new = ftrace_call_replace(ip, (unsigned long)func);  in ftrace_update_ftrace_func()
266 		ret = update_ftrace_func(ip, new);  in ftrace_update_ftrace_func()
272 static int is_ftrace_caller(unsigned long ip)  in is_ftrace_caller()  argument
274 	if (ip == ftrace_update_func)  in is_ftrace_caller()
289 	unsigned long ip;  in ftrace_int3_handler()  local
294 	ip = regs->ip - 1;  in ftrace_int3_handler()
295 	if (!ftrace_location(ip) && !is_ftrace_caller(ip))  in ftrace_int3_handler()
298 	regs->ip += MCOUNT_INSN_SIZE - 1;  in ftrace_int3_handler()
303 static int ftrace_write(unsigned long ip, const char *val, int size)  in ftrace_write()  argument
305 	ip = text_ip_addr(ip);  in ftrace_write()
307 	if (probe_kernel_write((void *)ip, val, size))  in ftrace_write()
313 static int add_break(unsigned long ip, const char *old)  in add_break()  argument
318 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))  in add_break()
327 	return ftrace_write(ip, &brk, 1);  in add_break()
333 	unsigned long ip = rec->ip;  in add_brk_on_call()  local
335 	old = ftrace_call_replace(ip, addr);  in add_brk_on_call()
337 	return add_break(rec->ip, old);  in add_brk_on_call()
347 	return add_break(rec->ip, old);  in add_brk_on_nop()
389 	unsigned long ip = rec->ip;  in remove_breakpoint()  local
392 	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))  in remove_breakpoint()
413 		nop = ftrace_call_replace(ip, ftrace_addr);  in remove_breakpoint()
420 		nop = ftrace_call_replace(ip, ftrace_addr);  in remove_breakpoint()
429 	return ftrace_write(ip, nop, 1);  in remove_breakpoint()
432 static int add_update_code(unsigned long ip, unsigned const char *new)  in add_update_code()  argument
435 	ip++;  in add_update_code()
437 	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);  in add_update_code()
442 	unsigned long ip = rec->ip;  in add_update_call()  local
445 	new = ftrace_call_replace(ip, addr);  in add_update_call()
446 	return add_update_code(ip, new);  in add_update_call()
451 	unsigned long ip = rec->ip;  in add_update_nop()  local
455 	return add_update_code(ip, new);  in add_update_nop()
486 	unsigned long ip = rec->ip;  in finish_update_call()  local
489 	new = ftrace_call_replace(ip, addr);  in finish_update_call()
491 	return ftrace_write(ip, new, 1);  in finish_update_call()
496 	unsigned long ip = rec->ip;  in finish_update_nop()  local
501 	return ftrace_write(ip, new, 1);  in finish_update_nop()
618 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,  in ftrace_modify_code()  argument
623 	ret = add_break(ip, old_code);  in ftrace_modify_code()
629 	ret = add_update_code(ip, new_code);  in ftrace_modify_code()
635 	ret = ftrace_write(ip, new_code, 1);  in ftrace_modify_code()
647 	if (ftrace_write(ip, old_code, 1))  in ftrace_modify_code()
668 static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)  in ftrace_jmp_replace()  argument
674 	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);  in ftrace_jmp_replace()
745 	unsigned long ip;  in create_trampoline()  local
783 	ip = (unsigned long)trampoline + size;  in create_trampoline()
786 	jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);  in create_trampoline()
845 	unsigned long ip;  in arch_ftrace_update_trampoline()  local
867 	ip = ops->trampoline + offset;  in arch_ftrace_update_trampoline()
872 	new = ftrace_call_replace(ip, (unsigned long)func);  in arch_ftrace_update_trampoline()
873 	ret = update_ftrace_func(ip, new);  in arch_ftrace_update_trampoline()
963 static int ftrace_mod_jmp(unsigned long ip, void *func)  in ftrace_mod_jmp()  argument
967 	new = ftrace_jmp_replace(ip, (unsigned long)func);  in ftrace_mod_jmp()
969 	return update_ftrace_func(ip, new);  in ftrace_mod_jmp()
974 	unsigned long ip = (unsigned long)(&ftrace_graph_call);  in ftrace_enable_ftrace_graph_caller()  local
976 	return ftrace_mod_jmp(ip, &ftrace_graph_caller);  in ftrace_enable_ftrace_graph_caller()
981 	unsigned long ip = (unsigned long)(&ftrace_graph_call);  in ftrace_disable_ftrace_graph_caller()  local
983 	return ftrace_mod_jmp(ip, &ftrace_stub);  in ftrace_disable_ftrace_graph_caller()