64 #define do_for_each_event_file(tr, file)			\  argument
65 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
66 		list_for_each_entry(file, &tr->events, list)
68 #define do_for_each_event_file_safe(tr, file)			\  argument
69 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
71 		list_for_each_entry_safe(file, ___n, &tr->events, list)
234 	struct trace_array *tr = trace_file->tr;  in trace_event_ignore_this_pid()  local
239 	pid_list = rcu_dereference_raw(tr->filtered_pids);  in trace_event_ignore_this_pid()
240 	no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);  in trace_event_ignore_this_pid()
245 	data = this_cpu_ptr(tr->array_buffer.data);  in trace_event_ignore_this_pid()
327 	struct trace_array *tr;  in trace_event_enable_cmd_record()  local
331 	do_for_each_event_file(tr, file) {  in trace_event_enable_cmd_record()
349 	struct trace_array *tr;  in trace_event_enable_tgid_record()  local
353 	do_for_each_event_file(tr, file) {  in trace_event_enable_tgid_record()
372 	struct trace_array *tr = file->tr;  in __ftrace_event_enable_disable()  local
444 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {  in __ftrace_event_enable_disable()
450 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {  in __ftrace_event_enable_disable()
498 static void ftrace_clear_events(struct trace_array *tr)  in ftrace_clear_events()  argument
503 	list_for_each_entry(file, &tr->events, list) {  in ftrace_clear_events()
513 	struct trace_array *tr = data;  in event_filter_pid_sched_process_exit()  local
515 	pid_list = rcu_dereference_raw(tr->filtered_pids);  in event_filter_pid_sched_process_exit()
518 	pid_list = rcu_dereference_raw(tr->filtered_no_pids);  in event_filter_pid_sched_process_exit()
528 	struct trace_array *tr = data;  in event_filter_pid_sched_process_fork()  local
530 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_process_fork()
533 	pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_process_fork()
537 void trace_event_follow_fork(struct trace_array *tr, bool enable)  in trace_event_follow_fork()  argument
541 						       tr, INT_MIN);  in trace_event_follow_fork()
543 						       tr, INT_MAX);  in trace_event_follow_fork()
546 						    tr);  in trace_event_follow_fork()
548 						    tr);  in trace_event_follow_fork()
556 	struct trace_array *tr = data;  in event_filter_pid_sched_switch_probe_pre()  local
561 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_switch_probe_pre()
562 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_switch_probe_pre()
571 	this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||  in event_filter_pid_sched_switch_probe_pre()
580 	struct trace_array *tr = data;  in event_filter_pid_sched_switch_probe_post()  local
584 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_switch_probe_post()
585 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_switch_probe_post()
587 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in event_filter_pid_sched_switch_probe_post()
594 	struct trace_array *tr = data;  in event_filter_pid_sched_wakeup_probe_pre()  local
599 	if (!this_cpu_read(tr->array_buffer.data->ignore_pid))  in event_filter_pid_sched_wakeup_probe_pre()
602 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_wakeup_probe_pre()
603 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_wakeup_probe_pre()
605 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in event_filter_pid_sched_wakeup_probe_pre()
612 	struct trace_array *tr = data;  in event_filter_pid_sched_wakeup_probe_post()  local
617 	if (this_cpu_read(tr->array_buffer.data->ignore_pid))  in event_filter_pid_sched_wakeup_probe_post()
620 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_wakeup_probe_post()
621 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_wakeup_probe_post()
624 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in event_filter_pid_sched_wakeup_probe_post()
628 static void unregister_pid_events(struct trace_array *tr)  in unregister_pid_events()  argument
630 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);  in unregister_pid_events()
631 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);  in unregister_pid_events()
633 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);  in unregister_pid_events()
634 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);  in unregister_pid_events()
636 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);  in unregister_pid_events()
637 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);  in unregister_pid_events()
639 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);  in unregister_pid_events()
640 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);  in unregister_pid_events()
643 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)  in __ftrace_clear_event_pids()  argument
650 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in __ftrace_clear_event_pids()
652 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,  in __ftrace_clear_event_pids()
660 		unregister_pid_events(tr);  in __ftrace_clear_event_pids()
662 		list_for_each_entry(file, &tr->events, list) {  in __ftrace_clear_event_pids()
667 			per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;  in __ftrace_clear_event_pids()
671 		rcu_assign_pointer(tr->filtered_pids, NULL);  in __ftrace_clear_event_pids()
674 		rcu_assign_pointer(tr->filtered_no_pids, NULL);  in __ftrace_clear_event_pids()
686 static void ftrace_clear_event_pids(struct trace_array *tr, int type)  in ftrace_clear_event_pids()  argument
689 	__ftrace_clear_event_pids(tr, type);  in ftrace_clear_event_pids()
780 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,  in __ftrace_set_clr_event_nolock()  argument
789 	list_for_each_entry(file, &tr->events, list) {  in __ftrace_set_clr_event_nolock()
827 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,  in __ftrace_set_clr_event()  argument
833 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);  in __ftrace_set_clr_event()
839 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)  in ftrace_set_clr_event()  argument
844 	if (!tr)  in ftrace_set_clr_event()
870 	ret = __ftrace_set_clr_event(tr, match, sub, event, set);  in ftrace_set_clr_event()
893 	struct trace_array *tr = top_trace_array();  in trace_set_clr_event()  local
895 	if (!tr)  in trace_set_clr_event()
898 	return __ftrace_set_clr_event(tr, NULL, system, event, set);  in trace_set_clr_event()
915 int trace_array_set_clr_event(struct trace_array *tr, const char *system,  in trace_array_set_clr_event()  argument
920 	if (!tr)  in trace_array_set_clr_event()
924 	return __ftrace_set_clr_event(tr, NULL, system, event, set);  in trace_array_set_clr_event()
937 	struct trace_array *tr = m->private;  in ftrace_event_write()  local
958 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);  in ftrace_event_write()
976 	struct trace_array *tr = m->private;  in t_next()  local
980 	list_for_each_entry_continue(file, &tr->events, list) {  in t_next()
997 	struct trace_array *tr = m->private;  in t_start()  local
1002 	file = list_entry(&tr->events, struct trace_event_file, list);  in t_start()
1015 	struct trace_array *tr = m->private;  in s_next()  local
1019 	list_for_each_entry_continue(file, &tr->events, list) {  in s_next()
1030 	struct trace_array *tr = m->private;  in s_start()  local
1035 	file = list_entry(&tr->events, struct trace_event_file, list);  in s_start()
1064 	struct trace_array *tr = m->private;  in __next()  local
1068 		pid_list = rcu_dereference_sched(tr->filtered_pids);  in __next()
1070 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in __next()
1091 	struct trace_array *tr = m->private;  in __start()  local
1103 		pid_list = rcu_dereference_sched(tr->filtered_pids);  in __start()
1105 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in __start()
1207 	struct trace_array *tr = dir->tr;  in system_enable_read()  local
1213 	list_for_each_entry(file, &tr->events, list) {  in system_enable_read()
1272 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);  in system_enable_write()
1501 	struct trace_array *tr;  in subsystem_open()  local
1510 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {  in subsystem_open()
1511 		list_for_each_entry(dir, &tr->systems, list) {  in subsystem_open()
1533 	if (trace_array_get(tr) < 0) {  in subsystem_open()
1540 		trace_array_put(tr);  in subsystem_open()
1550 	struct trace_array *tr = inode->i_private;  in system_tr_open()  local
1563 	dir->tr = tr;  in system_tr_open()
1573 	trace_array_put(dir->tr);  in subsystem_release()
1667 	struct trace_array *tr = data;  in ignore_task_cpu()  local
1675 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in ignore_task_cpu()
1677 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,  in ignore_task_cpu()
1680 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in ignore_task_cpu()
1684 static void register_pid_events(struct trace_array *tr)  in register_pid_events()  argument
1693 					 tr, INT_MAX);  in register_pid_events()
1695 					 tr, 0);  in register_pid_events()
1698 					 tr, INT_MAX);  in register_pid_events()
1700 					 tr, 0);  in register_pid_events()
1703 					     tr, INT_MAX);  in register_pid_events()
1705 					     tr, 0);  in register_pid_events()
1708 					 tr, INT_MAX);  in register_pid_events()
1710 					 tr, 0);  in register_pid_events()
1718 	struct trace_array *tr = m->private;  in event_pid_write()  local
1735 		filtered_pids = rcu_dereference_protected(tr->filtered_pids,  in event_pid_write()
1737 		other_pids = rcu_dereference_protected(tr->filtered_no_pids,  in event_pid_write()
1740 		filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,  in event_pid_write()
1742 		other_pids = rcu_dereference_protected(tr->filtered_pids,  in event_pid_write()
1751 		rcu_assign_pointer(tr->filtered_pids, pid_list);  in event_pid_write()
1753 		rcu_assign_pointer(tr->filtered_no_pids, pid_list);  in event_pid_write()
1755 	list_for_each_entry(file, &tr->events, list) {  in event_pid_write()
1763 		register_pid_events(tr);  in event_pid_write()
1771 	on_each_cpu(ignore_task_cpu, tr, 1);  in event_pid_write()
1940 	struct trace_array *tr = inode->i_private;  in ftrace_event_release()  local
1942 	trace_array_put(tr);  in ftrace_event_release()
1960 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_open()  local
1963 	ret = tracing_check_open_get_tr(tr);  in ftrace_event_set_open()
1969 		ftrace_clear_events(tr);  in ftrace_event_set_open()
1973 		trace_array_put(tr);  in ftrace_event_set_open()
1981 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_pid_open()  local
1984 	ret = tracing_check_open_get_tr(tr);  in ftrace_event_set_pid_open()
1990 		ftrace_clear_event_pids(tr, TRACE_PIDS);  in ftrace_event_set_pid_open()
1994 		trace_array_put(tr);  in ftrace_event_set_pid_open()
2002 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_npid_open()  local
2005 	ret = tracing_check_open_get_tr(tr);  in ftrace_event_set_npid_open()
2011 		ftrace_clear_event_pids(tr, TRACE_NO_PIDS);  in ftrace_event_set_npid_open()
2015 		trace_array_put(tr);  in ftrace_event_set_npid_open()
2053 event_subsystem_dir(struct trace_array *tr, const char *name,  in event_subsystem_dir()  argument
2061 	list_for_each_entry(dir, &tr->systems, list) {  in event_subsystem_dir()
2097 	dir->tr = tr;  in event_subsystem_dir()
2114 	list_add(&dir->list, &tr->systems);  in event_subsystem_dir()
2168 	struct trace_array *tr = file->tr;  in event_create_dir()  local
2178 		d_events = event_subsystem_dir(tr, call->class->system, file, parent);  in event_create_dir()
2243 	struct trace_array *tr;  in remove_event_from_tracers()  local
2245 	do_for_each_event_file_safe(tr, file) {  in remove_event_from_tracers()
2262 	struct trace_array *tr;  in event_remove()  local
2265 	do_for_each_event_file(tr, file) {  in event_remove()
2270 			tr->clear_trace = true;  in event_remove()
2462 		       struct trace_array *tr)  in trace_create_new_event()  argument
2471 	file->tr = tr;  in trace_create_new_event()
2475 	list_add(&file->list, &tr->events);  in trace_create_new_event()
2482 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)  in __trace_add_new_event()  argument
2486 	file = trace_create_new_event(call, tr);  in __trace_add_new_event()
2491 		return event_create_dir(tr->event_dir, file);  in __trace_add_new_event()
2503 			    struct trace_array *tr)  in __trace_early_add_new_event()  argument
2507 	file = trace_create_new_event(call, tr);  in __trace_early_add_new_event()
2547 	struct trace_array *tr;  in probe_remove_event_call()  local
2554 	do_for_each_event_file(tr, file) {  in probe_remove_event_call()
2675 __trace_add_event_dirs(struct trace_array *tr)  in __trace_add_event_dirs()  argument
2681 		ret = __trace_add_new_event(call, tr);  in __trace_add_event_dirs()
2690 __find_event_file(struct trace_array *tr, const char *system, const char *event)  in __find_event_file()  argument
2696 	list_for_each_entry(file, &tr->events, list) {  in __find_event_file()
2713 find_event_file(struct trace_array *tr, const char *system, const char *event)  in find_event_file()  argument
2717 	file = __find_event_file(tr, system, event);  in find_event_file()
2749 	struct trace_array *tr = top_trace_array();  in trace_get_event_file()  local
2754 		tr = trace_array_find_get(instance);  in trace_get_event_file()
2755 		if (!tr)  in trace_get_event_file()
2758 		ret = trace_array_get(tr);  in trace_get_event_file()
2765 	file = find_event_file(tr, system, event);  in trace_get_event_file()
2767 		trace_array_put(tr);  in trace_get_event_file()
2775 		trace_array_put(tr);  in trace_get_event_file()
2806 	trace_array_put(file->tr);  in trace_put_event_file()
2833 		   struct trace_array *tr, struct ftrace_probe_ops *ops,  in event_enable_probe()  argument
2850 			 struct trace_array *tr, struct ftrace_probe_ops *ops,  in event_enable_count_probe()  argument
2907 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,  in event_enable_init()  argument
2945 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,  in event_enable_free()  argument
2998 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,  in event_enable_func()  argument
3010 	if (!tr)  in event_enable_func()
3026 	file = find_event_file(tr, system, event);  in event_enable_func()
3038 		ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);  in event_enable_func()
3081 	ret = register_ftrace_function_probe(glob, tr, ops, data);  in event_enable_func()
3140 static void __trace_early_add_event_dirs(struct trace_array *tr)  in __trace_early_add_event_dirs()  argument
3146 	list_for_each_entry(file, &tr->events, list) {  in __trace_early_add_event_dirs()
3147 		ret = event_create_dir(tr->event_dir, file);  in __trace_early_add_event_dirs()
3160 void __trace_early_add_events(struct trace_array *tr)  in __trace_early_add_events()  argument
3170 		ret = __trace_early_add_new_event(call, tr);  in __trace_early_add_events()
3179 __trace_remove_event_dirs(struct trace_array *tr)  in __trace_remove_event_dirs()  argument
3183 	list_for_each_entry_safe(file, next, &tr->events, list)  in __trace_remove_event_dirs()
3189 	struct trace_array *tr;  in __add_event_to_tracers()  local
3191 	list_for_each_entry(tr, &ftrace_trace_arrays, list)  in __add_event_to_tracers()
3192 		__trace_add_new_event(call, tr);  in __add_event_to_tracers()
3212 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)  in create_event_toplevel_files()  argument
3218 				    tr, &ftrace_set_event_fops);  in create_event_toplevel_files()
3231 				  tr, &ftrace_tr_enable_fops);  in create_event_toplevel_files()
3240 				    tr, &ftrace_set_event_pid_fops);  in create_event_toplevel_files()
3245 				    tr, &ftrace_set_event_notrace_pid_fops);  in create_event_toplevel_files()
3262 	tr->event_dir = d_events;  in create_event_toplevel_files()
3280 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)  in event_trace_add_tracer()  argument
3286 	ret = create_event_toplevel_files(parent, tr);  in event_trace_add_tracer()
3292 	if (unlikely(!list_empty(&tr->events)))  in event_trace_add_tracer()
3293 		__trace_early_add_event_dirs(tr);  in event_trace_add_tracer()
3295 		__trace_add_event_dirs(tr);  in event_trace_add_tracer()
3307 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)  in early_event_add_tracer()  argument
3313 	ret = create_event_toplevel_files(parent, tr);  in early_event_add_tracer()
3318 	__trace_early_add_event_dirs(tr);  in early_event_add_tracer()
3328 int event_trace_del_tracer(struct trace_array *tr)  in event_trace_del_tracer()  argument
3333 	clear_event_triggers(tr);  in event_trace_del_tracer()
3336 	__ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);  in event_trace_del_tracer()
3339 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);  in event_trace_del_tracer()
3345 	__trace_remove_event_dirs(tr);  in event_trace_del_tracer()
3346 	tracefs_remove(tr->event_dir);  in event_trace_del_tracer()
3349 	tr->event_dir = NULL;  in event_trace_del_tracer()
3362 early_enable_events(struct trace_array *tr, bool disable_first)  in early_enable_events()  argument
3377 				ftrace_set_clr_event(tr, token, 0);  in early_enable_events()
3379 			ret = ftrace_set_clr_event(tr, token, 1);  in early_enable_events()
3392 	struct trace_array *tr = top_trace_array();  in event_trace_enable()  local
3396 	if (!tr)  in event_trace_enable()
3413 	__trace_early_add_events(tr);  in event_trace_enable()
3415 	early_enable_events(tr, false);  in event_trace_enable()
3438 	struct trace_array *tr;  in event_trace_enable_again()  local
3440 	tr = top_trace_array();  in event_trace_enable_again()
3441 	if (!tr)  in event_trace_enable_again()
3444 	early_enable_events(tr, true);  in event_trace_enable_again()
3465 	struct trace_array *tr;  in event_trace_init()  local
3469 	tr = top_trace_array();  in event_trace_init()
3470 	if (!tr)  in event_trace_init()
3474 				    tr, &ftrace_avail_fops);  in event_trace_init()
3478 	ret = early_event_add_tracer(NULL, tr);  in event_trace_init()
3564 	struct trace_array *tr;  in event_trace_self_tests()  local
3567 	tr = top_trace_array();  in event_trace_self_tests()
3568 	if (!tr)  in event_trace_self_tests()
3573 	list_for_each_entry(file, &tr->events, list) {  in event_trace_self_tests()
3616 	list_for_each_entry(dir, &tr->systems, list) {  in event_trace_self_tests()
3626 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);  in event_trace_self_tests()
3635 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);  in event_trace_self_tests()
3650 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);  in event_trace_self_tests()
3659 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);  in event_trace_self_tests()
3722 	event_trace_file.tr = top_trace_array();  in event_trace_self_test_with_function()
3723 	if (WARN_ON(!event_trace_file.tr))  in event_trace_self_test_with_function()