70 static bool trace_fprobe_is_return(struct trace_fprobe *tf)  in trace_fprobe_is_return()  argument
72 	return tf->fp.exit_handler != NULL;  in trace_fprobe_is_return()
75 static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)  in trace_fprobe_is_tracepoint()  argument
77 	return tf->tpoint != NULL;  in trace_fprobe_is_tracepoint()
80 static const char *trace_fprobe_symbol(struct trace_fprobe *tf)  in trace_fprobe_symbol()  argument
82 	return tf->symbol ? tf->symbol : "unknown";  in trace_fprobe_symbol()
87 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_is_busy()  local
89 	return trace_probe_is_enabled(&tf->tp);  in trace_fprobe_is_busy()
92 static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,  in trace_fprobe_match_command_head()  argument
100 	snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));  in trace_fprobe_match_command_head()
105 	return trace_probe_match_command_args(&tf->tp, argc, argv);  in trace_fprobe_match_command_head()
111 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_match()  local
113 	if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))  in trace_fprobe_match()
116 	if (system && strcmp(trace_probe_group_name(&tf->tp), system))  in trace_fprobe_match()
119 	return trace_fprobe_match_command_head(tf, argc, argv);  in trace_fprobe_match()
122 static bool trace_fprobe_is_registered(struct trace_fprobe *tf)  in trace_fprobe_is_registered()  argument
124 	return fprobe_is_registered(&tf->fp);  in trace_fprobe_is_registered()
172 __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in NOKPROBE_SYMBOL()
177 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in NOKPROBE_SYMBOL()
187 	dsize = __get_data_size(&tf->tp, regs);  in NOKPROBE_SYMBOL()
190 					   sizeof(*entry) + tf->tp.size + dsize);  in NOKPROBE_SYMBOL()
197 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);  in NOKPROBE_SYMBOL()
203 fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fentry_trace_func()  argument
208 	trace_probe_for_each_link_rcu(link, &tf->tp)  in fentry_trace_func()
209 		__fentry_trace_func(tf, entry_ip, regs, link->file);  in fentry_trace_func()
215 __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in __fexit_trace_func()  argument
221 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in __fexit_trace_func()
230 	dsize = __get_data_size(&tf->tp, regs);  in __fexit_trace_func()
233 					   sizeof(*entry) + tf->tp.size + dsize);  in __fexit_trace_func()
241 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);  in __fexit_trace_func()
247 fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fexit_trace_func()  argument
252 	trace_probe_for_each_link_rcu(link, &tf->tp)  in fexit_trace_func()
253 		__fexit_trace_func(tf, entry_ip, ret_ip, regs, link->file);  in fexit_trace_func()
259 static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fentry_perf_func()  argument
262 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in fentry_perf_func()
272 	dsize = __get_data_size(&tf->tp, regs);  in fentry_perf_func()
273 	__size = sizeof(*entry) + tf->tp.size + dsize;  in fentry_perf_func()
283 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);  in fentry_perf_func()
291 fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fexit_perf_func()  argument
294 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in fexit_perf_func()
304 	dsize = __get_data_size(&tf->tp, regs);  in fexit_perf_func()
305 	__size = sizeof(*entry) + tf->tp.size + dsize;  in fexit_perf_func()
315 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);  in fexit_perf_func()
326 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);  in fentry_dispatcher()  local
329 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))  in fentry_dispatcher()
330 		fentry_trace_func(tf, entry_ip, regs);  in fentry_dispatcher()
332 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))  in fentry_dispatcher()
333 		ret = fentry_perf_func(tf, entry_ip, regs);  in fentry_dispatcher()
343 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);  in fexit_dispatcher()  local
345 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))  in fexit_dispatcher()
346 		fexit_trace_func(tf, entry_ip, ret_ip, regs);  in fexit_dispatcher()
348 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))  in fexit_dispatcher()
349 		fexit_perf_func(tf, entry_ip, ret_ip, regs);  in fexit_dispatcher()
354 static void free_trace_fprobe(struct trace_fprobe *tf)  in free_trace_fprobe()  argument
356 	if (tf) {  in free_trace_fprobe()
357 		trace_probe_cleanup(&tf->tp);  in free_trace_fprobe()
358 		kfree(tf->symbol);  in free_trace_fprobe()
359 		kfree(tf);  in free_trace_fprobe()
373 	struct trace_fprobe *tf;  in alloc_trace_fprobe()  local
376 	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);  in alloc_trace_fprobe()
377 	if (!tf)  in alloc_trace_fprobe()
380 	tf->symbol = kstrdup(symbol, GFP_KERNEL);  in alloc_trace_fprobe()
381 	if (!tf->symbol)  in alloc_trace_fprobe()
385 		tf->fp.exit_handler = fexit_dispatcher;  in alloc_trace_fprobe()
387 		tf->fp.entry_handler = fentry_dispatcher;  in alloc_trace_fprobe()
389 	tf->tpoint = tpoint;  in alloc_trace_fprobe()
390 	tf->fp.nr_maxactive = maxactive;  in alloc_trace_fprobe()
392 	ret = trace_probe_init(&tf->tp, event, group, false);  in alloc_trace_fprobe()
396 	dyn_event_init(&tf->devent, &trace_fprobe_ops);  in alloc_trace_fprobe()
397 	return tf;  in alloc_trace_fprobe()
399 	free_trace_fprobe(tf);  in alloc_trace_fprobe()
407 	struct trace_fprobe *tf;  in find_trace_fprobe()  local
409 	for_each_trace_fprobe(tf, pos)  in find_trace_fprobe()
410 		if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&  in find_trace_fprobe()
411 		    strcmp(trace_probe_group_name(&tf->tp), group) == 0)  in find_trace_fprobe()
412 			return tf;  in find_trace_fprobe()
416 static inline int __enable_trace_fprobe(struct trace_fprobe *tf)  in __enable_trace_fprobe()  argument
418 	if (trace_fprobe_is_registered(tf))  in __enable_trace_fprobe()
419 		enable_fprobe(&tf->fp);  in __enable_trace_fprobe()
426 	struct trace_fprobe *tf;  in __disable_trace_fprobe()  local
428 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in __disable_trace_fprobe()
429 		if (!trace_fprobe_is_registered(tf))  in __disable_trace_fprobe()
431 		disable_fprobe(&tf->fp);  in __disable_trace_fprobe()
443 	struct trace_fprobe *tf;  in enable_trace_fprobe()  local
461 		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in enable_trace_fprobe()
463 			__enable_trace_fprobe(tf);  in enable_trace_fprobe()
629 static inline void init_trace_event_call(struct trace_fprobe *tf)  in init_trace_event_call()  argument
631 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in init_trace_event_call()
633 	if (trace_fprobe_is_return(tf)) {  in init_trace_event_call()
645 static int register_fprobe_event(struct trace_fprobe *tf)  in register_fprobe_event()  argument
647 	init_trace_event_call(tf);  in register_fprobe_event()
649 	return trace_probe_register_event_call(&tf->tp);  in register_fprobe_event()
652 static int unregister_fprobe_event(struct trace_fprobe *tf)  in unregister_fprobe_event()  argument
654 	return trace_probe_unregister_event_call(&tf->tp);  in unregister_fprobe_event()
658 static int __register_trace_fprobe(struct trace_fprobe *tf)  in __register_trace_fprobe()  argument
667 	if (trace_fprobe_is_registered(tf))  in __register_trace_fprobe()
670 	for (i = 0; i < tf->tp.nr_args; i++) {  in __register_trace_fprobe()
671 		ret = traceprobe_update_arg(&tf->tp.args[i]);  in __register_trace_fprobe()
677 	if (trace_probe_is_enabled(&tf->tp))  in __register_trace_fprobe()
678 		tf->fp.flags &= ~FPROBE_FL_DISABLED;  in __register_trace_fprobe()
680 		tf->fp.flags |= FPROBE_FL_DISABLED;  in __register_trace_fprobe()
682 	if (trace_fprobe_is_tracepoint(tf)) {  in __register_trace_fprobe()
683 		struct tracepoint *tpoint = tf->tpoint;  in __register_trace_fprobe()
694 		return register_fprobe_ips(&tf->fp, &ip, 1);  in __register_trace_fprobe()
698 	return register_fprobe(&tf->fp, tf->symbol, NULL);  in __register_trace_fprobe()
702 static void __unregister_trace_fprobe(struct trace_fprobe *tf)  in __unregister_trace_fprobe()  argument
704 	if (trace_fprobe_is_registered(tf)) {  in __unregister_trace_fprobe()
705 		unregister_fprobe(&tf->fp);  in __unregister_trace_fprobe()
706 		memset(&tf->fp, 0, sizeof(tf->fp));  in __unregister_trace_fprobe()
707 		if (trace_fprobe_is_tracepoint(tf)) {  in __unregister_trace_fprobe()
708 			tracepoint_probe_unregister(tf->tpoint,  in __unregister_trace_fprobe()
709 					tf->tpoint->probestub, NULL);  in __unregister_trace_fprobe()
710 			tf->tpoint = NULL;  in __unregister_trace_fprobe()
711 			tf->mod = NULL;  in __unregister_trace_fprobe()
718 static int unregister_trace_fprobe(struct trace_fprobe *tf)  in unregister_trace_fprobe()  argument
721 	if (trace_probe_has_sibling(&tf->tp))  in unregister_trace_fprobe()
725 	if (trace_probe_is_enabled(&tf->tp))  in unregister_trace_fprobe()
729 	if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))  in unregister_trace_fprobe()
733 	if (unregister_fprobe_event(tf))  in unregister_trace_fprobe()
737 	__unregister_trace_fprobe(tf);  in unregister_trace_fprobe()
738 	dyn_event_remove(&tf->devent);  in unregister_trace_fprobe()
739 	trace_probe_unlink(&tf->tp);  in unregister_trace_fprobe()
772 static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to)  in append_trace_fprobe()  argument
776 	if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||  in append_trace_fprobe()
777 	    trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {  in append_trace_fprobe()
782 	ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);  in append_trace_fprobe()
789 	if (trace_fprobe_has_same_fprobe(to, tf)) {  in append_trace_fprobe()
796 	ret = trace_probe_append(&tf->tp, &to->tp);  in append_trace_fprobe()
800 	ret = __register_trace_fprobe(tf);  in append_trace_fprobe()
802 		trace_probe_unlink(&tf->tp);  in append_trace_fprobe()
804 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));  in append_trace_fprobe()
810 static int register_trace_fprobe(struct trace_fprobe *tf)  in register_trace_fprobe()  argument
817 	old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),  in register_trace_fprobe()
818 				   trace_probe_group_name(&tf->tp));  in register_trace_fprobe()
820 		ret = append_trace_fprobe(tf, old_tf);  in register_trace_fprobe()
825 	ret = register_fprobe_event(tf);  in register_trace_fprobe()
836 	ret = __register_trace_fprobe(tf);  in register_trace_fprobe()
838 		unregister_fprobe_event(tf);  in register_trace_fprobe()
840 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));  in register_trace_fprobe()
852 	struct trace_fprobe *tf;  in __tracepoint_probe_module_cb()  local
859 	for_each_trace_fprobe(tf, pos) {  in __tracepoint_probe_module_cb()
860 		if (tp_mod->mod == tf->mod) {  in __tracepoint_probe_module_cb()
861 			tracepoint_probe_unregister(tf->tpoint,  in __tracepoint_probe_module_cb()
862 					tf->tpoint->probestub, NULL);  in __tracepoint_probe_module_cb()
863 			tf->tpoint = NULL;  in __tracepoint_probe_module_cb()
864 			tf->mod = NULL;  in __tracepoint_probe_module_cb()
967 	struct trace_fprobe *tf = NULL;  in __trace_fprobe_create()  local
1089 	tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive,  in __trace_fprobe_create()
1091 	if (IS_ERR(tf)) {  in __trace_fprobe_create()
1092 		ret = PTR_ERR(tf);  in __trace_fprobe_create()
1099 		tf->mod = __module_text_address(  in __trace_fprobe_create()
1100 				(unsigned long)tf->tpoint->probestub);  in __trace_fprobe_create()
1106 		ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx);  in __trace_fprobe_create()
1111 	ret = traceprobe_set_print_fmt(&tf->tp,  in __trace_fprobe_create()
1116 	ret = register_trace_fprobe(tf);  in __trace_fprobe_create()
1138 	free_trace_fprobe(tf);  in __trace_fprobe_create()
1149 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_release()  local
1150 	int ret = unregister_trace_fprobe(tf);  in trace_fprobe_release()
1153 		free_trace_fprobe(tf);  in trace_fprobe_release()
1159 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_show()  local
1162 	if (trace_fprobe_is_tracepoint(tf))  in trace_fprobe_show()
1166 	if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive)  in trace_fprobe_show()
1167 		seq_printf(m, "%d", tf->fp.nr_maxactive);  in trace_fprobe_show()
1168 	seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),  in trace_fprobe_show()
1169 				trace_probe_name(&tf->tp));  in trace_fprobe_show()
1171 	seq_printf(m, " %s%s", trace_fprobe_symbol(tf),  in trace_fprobe_show()
1172 			       trace_fprobe_is_return(tf) ? "%return" : "");  in trace_fprobe_show()
1174 	for (i = 0; i < tf->tp.nr_args; i++)  in trace_fprobe_show()
1175 		seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);  in trace_fprobe_show()