61 #define do_for_each_event_file(tr, file)			\  argument
62 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
63 		list_for_each_entry(file, &tr->events, list)
65 #define do_for_each_event_file_safe(tr, file)			\  argument
66 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
68 		list_for_each_entry_safe(file, ___n, &tr->events, list)
239 	struct trace_array *tr = trace_file->tr;  in trace_event_ignore_this_pid()  local
243 	pid_list = rcu_dereference_raw(tr->filtered_pids);  in trace_event_ignore_this_pid()
247 	data = this_cpu_ptr(tr->trace_buffer.data);  in trace_event_ignore_this_pid()
328 	struct trace_array *tr;  in trace_event_enable_cmd_record()  local
331 	do_for_each_event_file(tr, file) {  in trace_event_enable_cmd_record()
350 	struct trace_array *tr;  in trace_event_enable_tgid_record()  local
353 	do_for_each_event_file(tr, file) {  in trace_event_enable_tgid_record()
373 	struct trace_array *tr = file->tr;  in __ftrace_event_enable_disable()  local
445 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {  in __ftrace_event_enable_disable()
451 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {  in __ftrace_event_enable_disable()
499 static void ftrace_clear_events(struct trace_array *tr)  in ftrace_clear_events()  argument
504 	list_for_each_entry(file, &tr->events, list) {  in ftrace_clear_events()
514 	struct trace_array *tr = data;  in event_filter_pid_sched_process_exit()  local
516 	pid_list = rcu_dereference_raw(tr->filtered_pids);  in event_filter_pid_sched_process_exit()
526 	struct trace_array *tr = data;  in event_filter_pid_sched_process_fork()  local
528 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_process_fork()
532 void trace_event_follow_fork(struct trace_array *tr, bool enable)  in trace_event_follow_fork()  argument
536 						       tr, INT_MIN);  in trace_event_follow_fork()
538 						       tr, INT_MAX);  in trace_event_follow_fork()
541 						    tr);  in trace_event_follow_fork()
543 						    tr);  in trace_event_follow_fork()
551 	struct trace_array *tr = data;  in event_filter_pid_sched_switch_probe_pre()  local
554 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_switch_probe_pre()
556 	this_cpu_write(tr->trace_buffer.data->ignore_pid,  in event_filter_pid_sched_switch_probe_pre()
565 	struct trace_array *tr = data;  in event_filter_pid_sched_switch_probe_post()  local
568 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_switch_probe_post()
570 	this_cpu_write(tr->trace_buffer.data->ignore_pid,  in event_filter_pid_sched_switch_probe_post()
577 	struct trace_array *tr = data;  in event_filter_pid_sched_wakeup_probe_pre()  local
581 	if (!this_cpu_read(tr->trace_buffer.data->ignore_pid))  in event_filter_pid_sched_wakeup_probe_pre()
584 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_wakeup_probe_pre()
586 	this_cpu_write(tr->trace_buffer.data->ignore_pid,  in event_filter_pid_sched_wakeup_probe_pre()
593 	struct trace_array *tr = data;  in event_filter_pid_sched_wakeup_probe_post()  local
597 	if (this_cpu_read(tr->trace_buffer.data->ignore_pid))  in event_filter_pid_sched_wakeup_probe_post()
600 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_wakeup_probe_post()
603 	this_cpu_write(tr->trace_buffer.data->ignore_pid,  in event_filter_pid_sched_wakeup_probe_post()
607 static void __ftrace_clear_event_pids(struct trace_array *tr)  in __ftrace_clear_event_pids()  argument
613 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in __ftrace_clear_event_pids()
618 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);  in __ftrace_clear_event_pids()
619 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);  in __ftrace_clear_event_pids()
621 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);  in __ftrace_clear_event_pids()
622 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);  in __ftrace_clear_event_pids()
624 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);  in __ftrace_clear_event_pids()
625 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);  in __ftrace_clear_event_pids()
627 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);  in __ftrace_clear_event_pids()
628 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);  in __ftrace_clear_event_pids()
630 	list_for_each_entry(file, &tr->events, list) {  in __ftrace_clear_event_pids()
635 		per_cpu_ptr(tr->trace_buffer.data, cpu)->ignore_pid = false;  in __ftrace_clear_event_pids()
637 	rcu_assign_pointer(tr->filtered_pids, NULL);  in __ftrace_clear_event_pids()
645 static void ftrace_clear_event_pids(struct trace_array *tr)  in ftrace_clear_event_pids()  argument
648 	__ftrace_clear_event_pids(tr);  in ftrace_clear_event_pids()
739 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,  in __ftrace_set_clr_event_nolock()  argument
748 	list_for_each_entry(file, &tr->events, list) {  in __ftrace_set_clr_event_nolock()
786 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,  in __ftrace_set_clr_event()  argument
792 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);  in __ftrace_set_clr_event()
798 static int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)  in ftrace_set_clr_event()  argument
827 	ret = __ftrace_set_clr_event(tr, match, sub, event, set);  in ftrace_set_clr_event()
850 	struct trace_array *tr = top_trace_array();  in trace_set_clr_event()  local
852 	if (!tr)  in trace_set_clr_event()
855 	return __ftrace_set_clr_event(tr, NULL, system, event, set);  in trace_set_clr_event()
868 	struct trace_array *tr = m->private;  in ftrace_event_write()  local
889 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);  in ftrace_event_write()
907 	struct trace_array *tr = m->private;  in t_next()  local
911 	list_for_each_entry_continue(file, &tr->events, list) {  in t_next()
928 	struct trace_array *tr = m->private;  in t_start()  local
933 	file = list_entry(&tr->events, struct trace_event_file, list);  in t_start()
946 	struct trace_array *tr = m->private;  in s_next()  local
950 	list_for_each_entry_continue(file, &tr->events, list) {  in s_next()
961 	struct trace_array *tr = m->private;  in s_start()  local
966 	file = list_entry(&tr->events, struct trace_event_file, list);  in s_start()
995 	struct trace_array *tr = m->private;  in p_next()  local
996 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->filtered_pids);  in p_next()
1005 	struct trace_array *tr = m->private;  in p_start()  local
1016 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in p_start()
1106 	struct trace_array *tr = dir->tr;  in system_enable_read()  local
1112 	list_for_each_entry(file, &tr->events, list) {  in system_enable_read()
1171 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);  in system_enable_write()
1401 	struct trace_array *tr;  in subsystem_open()  local
1410 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {  in subsystem_open()
1411 		list_for_each_entry(dir, &tr->systems, list) {  in subsystem_open()
1433 	if (trace_array_get(tr) < 0) {  in subsystem_open()
1440 		trace_array_put(tr);  in subsystem_open()
1450 	struct trace_array *tr = inode->i_private;  in system_tr_open()  local
1456 	if (trace_array_get(tr) < 0)  in system_tr_open()
1462 		trace_array_put(tr);  in system_tr_open()
1466 	dir->tr = tr;  in system_tr_open()
1470 		trace_array_put(tr);  in system_tr_open()
1484 	trace_array_put(dir->tr);  in subsystem_release()
1578 	struct trace_array *tr = data;  in ignore_task_cpu()  local
1585 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in ignore_task_cpu()
1588 	this_cpu_write(tr->trace_buffer.data->ignore_pid,  in ignore_task_cpu()
1597 	struct trace_array *tr = m->private;  in ftrace_event_pid_write()  local
1612 	filtered_pids = rcu_dereference_protected(tr->filtered_pids,  in ftrace_event_pid_write()
1619 	rcu_assign_pointer(tr->filtered_pids, pid_list);  in ftrace_event_pid_write()
1621 	list_for_each_entry(file, &tr->events, list) {  in ftrace_event_pid_write()
1636 						 tr, INT_MAX);  in ftrace_event_pid_write()
1638 						 tr, 0);  in ftrace_event_pid_write()
1641 						 tr, INT_MAX);  in ftrace_event_pid_write()
1643 						 tr, 0);  in ftrace_event_pid_write()
1646 						     tr, INT_MAX);  in ftrace_event_pid_write()
1648 						     tr, 0);  in ftrace_event_pid_write()
1651 						 tr, INT_MAX);  in ftrace_event_pid_write()
1653 						 tr, 0);  in ftrace_event_pid_write()
1661 	on_each_cpu(ignore_task_cpu, tr, 1);  in ftrace_event_pid_write()
1796 	struct trace_array *tr = inode->i_private;  in ftrace_event_release()  local
1798 	trace_array_put(tr);  in ftrace_event_release()
1815 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_open()  local
1818 	if (trace_array_get(tr) < 0)  in ftrace_event_set_open()
1823 		ftrace_clear_events(tr);  in ftrace_event_set_open()
1827 		trace_array_put(tr);  in ftrace_event_set_open()
1835 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_pid_open()  local
1838 	if (trace_array_get(tr) < 0)  in ftrace_event_set_pid_open()
1843 		ftrace_clear_event_pids(tr);  in ftrace_event_set_pid_open()
1847 		trace_array_put(tr);  in ftrace_event_set_pid_open()
1885 event_subsystem_dir(struct trace_array *tr, const char *name,  in event_subsystem_dir()  argument
1893 	list_for_each_entry(dir, &tr->systems, list) {  in event_subsystem_dir()
1929 	dir->tr = tr;  in event_subsystem_dir()
1946 	list_add(&dir->list, &tr->systems);  in event_subsystem_dir()
1963 	struct trace_array *tr = file->tr;  in event_create_dir()  local
1974 		d_events = event_subsystem_dir(tr, call->class->system, file, parent);  in event_create_dir()
2037 	struct trace_array *tr;  in remove_event_from_tracers()  local
2039 	do_for_each_event_file_safe(tr, file) {  in remove_event_from_tracers()
2056 	struct trace_array *tr;  in event_remove()  local
2059 	do_for_each_event_file(tr, file) {  in event_remove()
2064 			tr->clear_trace = true;  in event_remove()
2256 		       struct trace_array *tr)  in trace_create_new_event()  argument
2265 	file->tr = tr;  in trace_create_new_event()
2269 	list_add(&file->list, &tr->events);  in trace_create_new_event()
2276 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)  in __trace_add_new_event()  argument
2280 	file = trace_create_new_event(call, tr);  in __trace_add_new_event()
2284 	return event_create_dir(tr->event_dir, file);  in __trace_add_new_event()
2294 			    struct trace_array *tr)  in __trace_early_add_new_event()  argument
2298 	file = trace_create_new_event(call, tr);  in __trace_early_add_new_event()
2338 	struct trace_array *tr;  in probe_remove_event_call()  local
2345 	do_for_each_event_file(tr, file) {  in probe_remove_event_call()
2466 __trace_add_event_dirs(struct trace_array *tr)  in __trace_add_event_dirs()  argument
2472 		ret = __trace_add_new_event(call, tr);  in __trace_add_event_dirs()
2481 __find_event_file(struct trace_array *tr, const char *system, const char *event)  in __find_event_file()  argument
2487 	list_for_each_entry(file, &tr->events, list) {  in __find_event_file()
2504 find_event_file(struct trace_array *tr, const char *system, const char *event)  in find_event_file()  argument
2508 	file = __find_event_file(tr, system, event);  in find_event_file()
2539 		   struct trace_array *tr, struct ftrace_probe_ops *ops,  in event_enable_probe()  argument
2556 			 struct trace_array *tr, struct ftrace_probe_ops *ops,  in event_enable_count_probe()  argument
2613 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,  in event_enable_init()  argument
2651 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,  in event_enable_free()  argument
2704 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,  in event_enable_func()  argument
2716 	if (!tr)  in event_enable_func()
2732 	file = find_event_file(tr, system, event);  in event_enable_func()
2744 		ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);  in event_enable_func()
2787 	ret = register_ftrace_function_probe(glob, tr, ops, data);  in event_enable_func()
2847 __trace_early_add_event_dirs(struct trace_array *tr)  in __trace_early_add_event_dirs()  argument
2853 	list_for_each_entry(file, &tr->events, list) {  in __trace_early_add_event_dirs()
2854 		ret = event_create_dir(tr->event_dir, file);  in __trace_early_add_event_dirs()
2868 __trace_early_add_events(struct trace_array *tr)  in __trace_early_add_events()  argument
2878 		ret = __trace_early_add_new_event(call, tr);  in __trace_early_add_events()
2887 __trace_remove_event_dirs(struct trace_array *tr)  in __trace_remove_event_dirs()  argument
2891 	list_for_each_entry_safe(file, next, &tr->events, list)  in __trace_remove_event_dirs()
2897 	struct trace_array *tr;  in __add_event_to_tracers()  local
2899 	list_for_each_entry(tr, &ftrace_trace_arrays, list)  in __add_event_to_tracers()
2900 		__trace_add_new_event(call, tr);  in __add_event_to_tracers()
2920 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)  in create_event_toplevel_files()  argument
2926 				    tr, &ftrace_set_event_fops);  in create_event_toplevel_files()
2939 				  tr, &ftrace_tr_enable_fops);  in create_event_toplevel_files()
2948 				    tr, &ftrace_set_event_pid_fops);  in create_event_toplevel_files()
2965 	tr->event_dir = d_events;  in create_event_toplevel_files()
2983 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)  in event_trace_add_tracer()  argument
2989 	ret = create_event_toplevel_files(parent, tr);  in event_trace_add_tracer()
2994 	__trace_add_event_dirs(tr);  in event_trace_add_tracer()
3006 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)  in early_event_add_tracer()  argument
3012 	ret = create_event_toplevel_files(parent, tr);  in early_event_add_tracer()
3017 	__trace_early_add_event_dirs(tr);  in early_event_add_tracer()
3027 int event_trace_del_tracer(struct trace_array *tr)  in event_trace_del_tracer()  argument
3032 	clear_event_triggers(tr);  in event_trace_del_tracer()
3035 	__ftrace_clear_event_pids(tr);  in event_trace_del_tracer()
3038 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);  in event_trace_del_tracer()
3044 	__trace_remove_event_dirs(tr);  in event_trace_del_tracer()
3045 	tracefs_remove_recursive(tr->event_dir);  in event_trace_del_tracer()
3048 	tr->event_dir = NULL;  in event_trace_del_tracer()
3061 early_enable_events(struct trace_array *tr, bool disable_first)  in early_enable_events()  argument
3076 				ftrace_set_clr_event(tr, token, 0);  in early_enable_events()
3078 			ret = ftrace_set_clr_event(tr, token, 1);  in early_enable_events()
3091 	struct trace_array *tr = top_trace_array();  in event_trace_enable()  local
3095 	if (!tr)  in event_trace_enable()
3112 	__trace_early_add_events(tr);  in event_trace_enable()
3114 	early_enable_events(tr, false);  in event_trace_enable()
3137 	struct trace_array *tr;  in event_trace_enable_again()  local
3139 	tr = top_trace_array();  in event_trace_enable_again()
3140 	if (!tr)  in event_trace_enable_again()
3143 	early_enable_events(tr, true);  in event_trace_enable_again()
3152 	struct trace_array *tr;  in event_trace_init()  local
3157 	tr = top_trace_array();  in event_trace_init()
3158 	if (!tr)  in event_trace_init()
3166 				    tr, &ftrace_avail_fops);  in event_trace_init()
3176 	ret = early_event_add_tracer(d_tracer, tr);  in event_trace_init()
3258 	struct trace_array *tr;  in event_trace_self_tests()  local
3261 	tr = top_trace_array();  in event_trace_self_tests()
3262 	if (!tr)  in event_trace_self_tests()
3267 	list_for_each_entry(file, &tr->events, list) {  in event_trace_self_tests()
3310 	list_for_each_entry(dir, &tr->systems, list) {  in event_trace_self_tests()
3320 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);  in event_trace_self_tests()
3329 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);  in event_trace_self_tests()
3344 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);  in event_trace_self_tests()
3353 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);  in event_trace_self_tests()
3416 	event_trace_file.tr = top_trace_array();  in event_trace_self_test_with_function()
3417 	if (WARN_ON(!event_trace_file.tr))  in event_trace_self_test_with_function()