98 	struct trace_array *tr;  in ftrace_pids_enabled()  local
103 	tr = ops->private;  in ftrace_pids_enabled()
105 	return tr->function_pids != NULL || tr->function_no_pids != NULL;  in ftrace_pids_enabled()
145 	struct trace_array *tr = op->private;  in ftrace_pid_func()  local
148 	if (tr) {  in ftrace_pid_func()
149 		pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);  in ftrace_pid_func()
1020 	struct trace_array	*tr;  member
1287 static int ftrace_add_mod(struct trace_array *tr,  in ftrace_add_mod()  argument
1292 	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;  in ftrace_add_mod()
3257 	struct trace_array		*tr;  member
3268 	struct trace_array *tr = iter->ops->private;  in t_probe_next()  local
3279 	if (!tr)  in t_probe_next()
3282 	func_probes = &tr->func_probes;  in t_probe_next()
3399 	struct trace_array *tr = iter->tr;  in t_mod_next()  local
3406 	if (iter->mod_list == &tr->mod_trace ||  in t_mod_next()
3407 	    iter->mod_list == &tr->mod_notrace) {  in t_mod_next()
3429 	if (!iter->tr)  in t_mod_start()
3452 	struct trace_array *tr = iter->tr;  in t_mod_show()  local
3455 			 iter->mod_list == &tr->mod_trace ||  in t_mod_show()
3456 			 iter->mod_list == &tr->mod_notrace)  in t_mod_show()
3761 	struct trace_array *tr = ops->private;  in ftrace_regex_open()  local
3769 	if (tracing_check_open_get_tr(tr))  in ftrace_regex_open()
3781 	iter->tr = tr;  in ftrace_regex_open()
3787 		mod_head = tr ? &tr->mod_notrace : NULL;  in ftrace_regex_open()
3790 		mod_head = tr ? &tr->mod_trace : NULL;  in ftrace_regex_open()
3835 		if (tr)  in ftrace_regex_open()
3836 			trace_array_put(tr);  in ftrace_regex_open()
4130 static int cache_mod(struct trace_array *tr,  in cache_mod()  argument
4134 	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;  in cache_mod()
4167 	ret = ftrace_add_mod(tr, func, module, enable);  in cache_mod()
4251 	struct trace_array *tr;  in process_cached_mods()  local
4259 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {  in process_cached_mods()
4260 		if (!list_empty(&tr->mod_trace))  in process_cached_mods()
4261 			process_mod_list(&tr->mod_trace, tr->ops, mod, true);  in process_cached_mods()
4262 		if (!list_empty(&tr->mod_notrace))  in process_cached_mods()
4263 			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);  in process_cached_mods()
4277 ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,  in ftrace_mod_callback()  argument
4299 		return cache_mod(tr, func_orig, module, enable);  in ftrace_mod_callback()
4331 	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);  in function_trace_probe_call()
4497 			probe_ops->free(probe_ops, probe->tr, 0, probe->data);  in release_probe()
4514 register_ftrace_function_probe(char *glob, struct trace_array *tr,  in register_ftrace_function_probe()  argument
4528 	if (WARN_ON(!tr))  in register_ftrace_function_probe()
4538 	list_for_each_entry(probe, &tr->func_probes, list) {  in register_ftrace_function_probe()
4542 	if (&probe->list == &tr->func_probes) {  in register_ftrace_function_probe()
4550 		probe->tr = tr;  in register_ftrace_function_probe()
4552 		list_add(&probe->list, &tr->func_probes);  in register_ftrace_function_probe()
4594 				ret = probe_ops->init(probe_ops, tr,  in register_ftrace_function_probe()
4599 						probe_ops->free(probe_ops, tr,  in register_ftrace_function_probe()
4650 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);  in register_ftrace_function_probe()
4657 unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,  in unregister_ftrace_function_probe_func()  argument
4690 	list_for_each_entry(probe, &tr->func_probes, list) {  in unregister_ftrace_function_probe_func()
4694 	if (&probe->list == &tr->func_probes)  in unregister_ftrace_function_probe_func()
4767 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);  in unregister_ftrace_function_probe_func()
4785 void clear_ftrace_function_probes(struct trace_array *tr)  in clear_ftrace_function_probes()  argument
4789 	list_for_each_entry_safe(probe, n, &tr->func_probes, list)  in clear_ftrace_function_probes()
4790 		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);  in clear_ftrace_function_probes()
4846 	struct trace_array *tr = iter->ops->private;  in ftrace_process_regex()  local
4869 			ret = p->func(tr, hash, func, command, next, enable);  in ftrace_process_regex()
5613 			if (iter->tr && !list_empty(&iter->tr->mod_trace))  in ftrace_regex_release()
5629 	if (iter->tr)  in ftrace_regex_release()
5630 		trace_array_put(iter->tr);  in ftrace_regex_release()
6332 	struct trace_array *tr;  in clear_mod_from_hashes()  local
6335 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {  in clear_mod_from_hashes()
6336 		if (!tr->ops || !tr->ops->func_hash)  in clear_mod_from_hashes()
6338 		mutex_lock(&tr->ops->func_hash->regex_lock);  in clear_mod_from_hashes()
6339 		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);  in clear_mod_from_hashes()
6340 		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);  in clear_mod_from_hashes()
6341 		mutex_unlock(&tr->ops->func_hash->regex_lock);  in clear_mod_from_hashes()
6703 	struct trace_array *tr;  in clear_func_from_hashes()  local
6706 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {  in clear_func_from_hashes()
6707 		if (!tr->ops || !tr->ops->func_hash)  in clear_func_from_hashes()
6709 		mutex_lock(&tr->ops->func_hash->regex_lock);  in clear_func_from_hashes()
6710 		clear_func_from_hash(func, tr->ops->func_hash->filter_hash);  in clear_func_from_hashes()
6711 		clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);  in clear_func_from_hashes()
6712 		mutex_unlock(&tr->ops->func_hash->regex_lock);  in clear_func_from_hashes()
6877 void ftrace_init_trace_array(struct trace_array *tr)  in ftrace_init_trace_array()  argument
6879 	INIT_LIST_HEAD(&tr->func_probes);  in ftrace_init_trace_array()
6880 	INIT_LIST_HEAD(&tr->mod_trace);  in ftrace_init_trace_array()
6881 	INIT_LIST_HEAD(&tr->mod_notrace);  in ftrace_init_trace_array()
6912 __init void ftrace_init_global_array_ops(struct trace_array *tr)  in ftrace_init_global_array_ops()  argument
6914 	tr->ops = &global_ops;  in ftrace_init_global_array_ops()
6915 	tr->ops->private = tr;  in ftrace_init_global_array_ops()
6916 	ftrace_init_trace_array(tr);  in ftrace_init_global_array_ops()
6919 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)  in ftrace_init_array_ops()  argument
6922 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {  in ftrace_init_array_ops()
6923 		if (WARN_ON(tr->ops->func != ftrace_stub))  in ftrace_init_array_ops()
6925 			       tr->ops->func);  in ftrace_init_array_ops()
6927 	tr->ops->func = func;  in ftrace_init_array_ops()
6928 	tr->ops->private = tr;  in ftrace_init_array_ops()
6931 void ftrace_reset_array_ops(struct trace_array *tr)  in ftrace_reset_array_ops()  argument
6933 	tr->ops->func = ftrace_stub;  in ftrace_reset_array_ops()
7060 	struct trace_array *tr = data;  in ftrace_filter_pid_sched_switch_probe()  local
7064 	pid_list = rcu_dereference_sched(tr->function_pids);  in ftrace_filter_pid_sched_switch_probe()
7065 	no_pid_list = rcu_dereference_sched(tr->function_no_pids);  in ftrace_filter_pid_sched_switch_probe()
7068 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,  in ftrace_filter_pid_sched_switch_probe()
7071 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,  in ftrace_filter_pid_sched_switch_probe()
7081 	struct trace_array *tr = data;  in ftrace_pid_follow_sched_process_fork()  local
7083 	pid_list = rcu_dereference_sched(tr->function_pids);  in ftrace_pid_follow_sched_process_fork()
7086 	pid_list = rcu_dereference_sched(tr->function_no_pids);  in ftrace_pid_follow_sched_process_fork()
7094 	struct trace_array *tr = data;  in ftrace_pid_follow_sched_process_exit()  local
7096 	pid_list = rcu_dereference_sched(tr->function_pids);  in ftrace_pid_follow_sched_process_exit()
7099 	pid_list = rcu_dereference_sched(tr->function_no_pids);  in ftrace_pid_follow_sched_process_exit()
7103 void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)  in ftrace_pid_follow_fork()  argument
7107 						  tr);  in ftrace_pid_follow_fork()
7109 						  tr);  in ftrace_pid_follow_fork()
7112 						    tr);  in ftrace_pid_follow_fork()
7114 						    tr);  in ftrace_pid_follow_fork()
7118 static void clear_ftrace_pids(struct trace_array *tr, int type)  in clear_ftrace_pids()  argument
7124 	pid_list = rcu_dereference_protected(tr->function_pids,  in clear_ftrace_pids()
7126 	no_pid_list = rcu_dereference_protected(tr->function_no_pids,  in clear_ftrace_pids()
7135 		unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);  in clear_ftrace_pids()
7137 			per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE;  in clear_ftrace_pids()
7141 		rcu_assign_pointer(tr->function_pids, NULL);  in clear_ftrace_pids()
7144 		rcu_assign_pointer(tr->function_no_pids, NULL);  in clear_ftrace_pids()
7156 void ftrace_clear_pids(struct trace_array *tr)  in ftrace_clear_pids()  argument
7160 	clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);  in ftrace_clear_pids()
7165 static void ftrace_pid_reset(struct trace_array *tr, int type)  in ftrace_pid_reset()  argument
7168 	clear_ftrace_pids(tr, type);  in ftrace_pid_reset()
7183 	struct trace_array *tr = m->private;  in fpid_start()  local
7188 	pid_list = rcu_dereference_sched(tr->function_pids);  in fpid_start()
7198 	struct trace_array *tr = m->private;  in fpid_next()  local
7199 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);  in fpid_next()
7236 	struct trace_array *tr = m->private;  in fnpid_start()  local
7241 	pid_list = rcu_dereference_sched(tr->function_no_pids);  in fnpid_start()
7251 	struct trace_array *tr = m->private;  in fnpid_next()  local
7252 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids);  in fnpid_next()
7271 	struct trace_array *tr = inode->i_private;  in pid_open()  local
7275 	ret = tracing_check_open_get_tr(tr);  in pid_open()
7281 		ftrace_pid_reset(tr, type);  in pid_open()
7291 		trace_array_put(tr);  in pid_open()
7298 		trace_array_put(tr);  in pid_open()
7302 		m->private = tr;  in pid_open()
7322 	struct trace_array *tr = data;  in ignore_task_cpu()  local
7330 	pid_list = rcu_dereference_protected(tr->function_pids,  in ignore_task_cpu()
7332 	no_pid_list = rcu_dereference_protected(tr->function_no_pids,  in ignore_task_cpu()
7336 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,  in ignore_task_cpu()
7339 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,  in ignore_task_cpu()
7348 	struct trace_array *tr = m->private;  in pid_write()  local
7361 		filtered_pids = rcu_dereference_protected(tr->function_pids,  in pid_write()
7363 		other_pids = rcu_dereference_protected(tr->function_no_pids,  in pid_write()
7367 		filtered_pids = rcu_dereference_protected(tr->function_no_pids,  in pid_write()
7369 		other_pids = rcu_dereference_protected(tr->function_pids,  in pid_write()
7384 		rcu_assign_pointer(tr->function_pids, pid_list);  in pid_write()
7387 		rcu_assign_pointer(tr->function_no_pids, pid_list);  in pid_write()
7397 		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);  in pid_write()
7405 	on_each_cpu(ignore_task_cpu, tr, 1);  in pid_write()
7435 	struct trace_array *tr = inode->i_private;  in ftrace_pid_release()  local
7437 	trace_array_put(tr);  in ftrace_pid_release()
7458 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)  in ftrace_init_tracefs()  argument
7461 			    tr, &ftrace_pid_fops);  in ftrace_init_tracefs()
7463 			    tr, &ftrace_no_pid_fops);  in ftrace_init_tracefs()
7466 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,  in ftrace_init_tracefs_toplevel()  argument
7470 	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));  in ftrace_init_tracefs_toplevel()