68 static int ftrace_calc_offset(long ip, long addr)  in ftrace_calc_offset()  argument
70 	return (int)(addr - ip);  in ftrace_calc_offset()
74 ftrace_text_replace(unsigned char op, unsigned long ip, unsigned long addr)  in ftrace_text_replace()  argument
79 	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);  in ftrace_text_replace()
85 ftrace_call_replace(unsigned long ip, unsigned long addr)  in ftrace_call_replace()  argument
87 	return ftrace_text_replace(0xe8, ip, addr);  in ftrace_call_replace()
96 static unsigned long text_ip_addr(unsigned long ip)  in text_ip_addr()  argument
106 	if (within(ip, (unsigned long)_text, (unsigned long)_etext))  in text_ip_addr()
107 		ip = (unsigned long)__va(__pa_symbol(ip));  in text_ip_addr()
109 	return ip;  in text_ip_addr()
118 ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,  in ftrace_modify_code_direct()  argument
134 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))  in ftrace_modify_code_direct()
141 	ip = text_ip_addr(ip);  in ftrace_modify_code_direct()
144 	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))  in ftrace_modify_code_direct()
156 	unsigned long ip = rec->ip;  in ftrace_make_nop()  local
158 	old = ftrace_call_replace(ip, addr);  in ftrace_make_nop()
170 		return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_nop()
182 	unsigned long ip = rec->ip;  in ftrace_make_call()  local
185 	new = ftrace_call_replace(ip, addr);  in ftrace_make_call()
188 	return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_call()
225 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
247 static int update_ftrace_func(unsigned long ip, void *new)  in update_ftrace_func()  argument
252 	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);  in update_ftrace_func()
254 	ftrace_update_func = ip;  in update_ftrace_func()
261 	ret = ftrace_modify_code(ip, old, new);  in update_ftrace_func()
270 	unsigned long ip = (unsigned long)(&ftrace_call);  in ftrace_update_ftrace_func()  local
276 	new = ftrace_call_replace(ip, (unsigned long)func);  in ftrace_update_ftrace_func()
277 	ret = update_ftrace_func(ip, new);  in ftrace_update_ftrace_func()
281 		ip = (unsigned long)(&ftrace_regs_call);  in ftrace_update_ftrace_func()
282 		new = ftrace_call_replace(ip, (unsigned long)func);  in ftrace_update_ftrace_func()
283 		ret = update_ftrace_func(ip, new);  in ftrace_update_ftrace_func()
289 static nokprobe_inline int is_ftrace_caller(unsigned long ip)  in is_ftrace_caller()  argument
291 	if (ip == ftrace_update_func)  in is_ftrace_caller()
306 	unsigned long ip;  in ftrace_int3_handler()  local
311 	ip = regs->ip - INT3_INSN_SIZE;  in ftrace_int3_handler()
313 	if (ftrace_location(ip)) {  in ftrace_int3_handler()
316 	} else if (is_ftrace_caller(ip)) {  in ftrace_int3_handler()
318 			int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);  in ftrace_int3_handler()
329 static int ftrace_write(unsigned long ip, const char *val, int size)  in ftrace_write()  argument
331 	ip = text_ip_addr(ip);  in ftrace_write()
333 	if (probe_kernel_write((void *)ip, val, size))  in ftrace_write()
339 static int add_break(unsigned long ip, const char *old)  in add_break()  argument
344 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))  in add_break()
353 	return ftrace_write(ip, &brk, 1);  in add_break()
359 	unsigned long ip = rec->ip;  in add_brk_on_call()  local
361 	old = ftrace_call_replace(ip, addr);  in add_brk_on_call()
363 	return add_break(rec->ip, old);  in add_brk_on_call()
373 	return add_break(rec->ip, old);  in add_brk_on_nop()
415 	unsigned long ip = rec->ip;  in remove_breakpoint()  local
418 	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))  in remove_breakpoint()
439 		nop = ftrace_call_replace(ip, ftrace_addr);  in remove_breakpoint()
446 		nop = ftrace_call_replace(ip, ftrace_addr);  in remove_breakpoint()
455 	return ftrace_write(ip, nop, 1);  in remove_breakpoint()
458 static int add_update_code(unsigned long ip, unsigned const char *new)  in add_update_code()  argument
461 	ip++;  in add_update_code()
463 	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);  in add_update_code()
468 	unsigned long ip = rec->ip;  in add_update_call()  local
471 	new = ftrace_call_replace(ip, addr);  in add_update_call()
472 	return add_update_code(ip, new);  in add_update_call()
477 	unsigned long ip = rec->ip;  in add_update_nop()  local
481 	return add_update_code(ip, new);  in add_update_nop()
512 	unsigned long ip = rec->ip;  in finish_update_call()  local
515 	new = ftrace_call_replace(ip, addr);  in finish_update_call()
517 	return ftrace_write(ip, new, 1);  in finish_update_call()
522 	unsigned long ip = rec->ip;  in finish_update_nop()  local
527 	return ftrace_write(ip, new, 1);  in finish_update_nop()
644 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,  in ftrace_modify_code()  argument
649 	ret = add_break(ip, old_code);  in ftrace_modify_code()
655 	ret = add_update_code(ip, new_code);  in ftrace_modify_code()
661 	ret = ftrace_write(ip, new_code, 1);  in ftrace_modify_code()
673 	if (ftrace_write(ip, old_code, 1))  in ftrace_modify_code()
756 	void *ip;  in create_trampoline()  local
791 	ip = trampoline + size;  in create_trampoline()
795 	ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);  in create_trampoline()
864 	unsigned long ip;  in arch_ftrace_update_trampoline()  local
886 	ip = ops->trampoline + offset;  in arch_ftrace_update_trampoline()
893 	new = ftrace_call_replace(ip, (unsigned long)func);  in arch_ftrace_update_trampoline()
894 	ret = update_ftrace_func(ip, new);  in arch_ftrace_update_trampoline()
984 static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)  in ftrace_jmp_replace()  argument
986 	return ftrace_text_replace(0xe9, ip, addr);  in ftrace_jmp_replace()
989 static int ftrace_mod_jmp(unsigned long ip, void *func)  in ftrace_mod_jmp()  argument
994 	new = ftrace_jmp_replace(ip, (unsigned long)func);  in ftrace_mod_jmp()
996 	return update_ftrace_func(ip, new);  in ftrace_mod_jmp()
1001 	unsigned long ip = (unsigned long)(&ftrace_graph_call);  in ftrace_enable_ftrace_graph_caller()  local
1003 	return ftrace_mod_jmp(ip, &ftrace_graph_caller);  in ftrace_enable_ftrace_graph_caller()
1008 	unsigned long ip = (unsigned long)(&ftrace_graph_call);  in ftrace_disable_ftrace_graph_caller()  local
1010 	return ftrace_mod_jmp(ip, &ftrace_stub);  in ftrace_disable_ftrace_graph_caller()