456 	struct ftrace_profile *rec = v;  in function_stat_next()  local
459 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);  in function_stat_next()
463 		rec++;  in function_stat_next()
465 	if ((void *)rec >= (void *)&pg->records[pg->index]) {  in function_stat_next()
469 		rec = &pg->records[0];  in function_stat_next()
470 		if (!rec->counter)  in function_stat_next()
474 	return rec;  in function_stat_next()
534 	struct ftrace_profile *rec = v;  in function_stat_show()  local
545 	if (unlikely(rec->counter == 0)) {  in function_stat_show()
551 	avg = div64_ul(rec->time, rec->counter);  in function_stat_show()
556 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);  in function_stat_show()
557 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);  in function_stat_show()
563 	if (rec->counter <= 1)  in function_stat_show()
570 		stddev = rec->counter * rec->time_squared -  in function_stat_show()
571 			 rec->time * rec->time;  in function_stat_show()
578 				  rec->counter * (rec->counter - 1) * 1000);  in function_stat_show()
582 	trace_print_graph_duration(rec->time, &s);  in function_stat_show()
720 	struct ftrace_profile *rec;  in ftrace_find_profiled_func()  local
730 	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {  in ftrace_find_profiled_func()
731 		if (rec->ip == ip)  in ftrace_find_profiled_func()
732 			return rec;  in ftrace_find_profiled_func()
739 			       struct ftrace_profile *rec)  in ftrace_add_profile()  argument
743 	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);  in ftrace_add_profile()
744 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);  in ftrace_add_profile()
753 	struct ftrace_profile *rec = NULL;  in ftrace_profile_alloc()  local
763 	rec = ftrace_find_profiled_func(stat, ip);  in ftrace_profile_alloc()
764 	if (rec)  in ftrace_profile_alloc()
773 	rec = &stat->pages->records[stat->pages->index++];  in ftrace_profile_alloc()
774 	rec->ip = ip;  in ftrace_profile_alloc()
775 	ftrace_add_profile(stat, rec);  in ftrace_profile_alloc()
780 	return rec;  in ftrace_profile_alloc()
788 	struct ftrace_profile *rec;  in function_profile_call()  local
800 	rec = ftrace_find_profiled_func(stat, ip);  in function_profile_call()
801 	if (!rec) {  in function_profile_call()
802 		rec = ftrace_profile_alloc(stat, ip);  in function_profile_call()
803 		if (!rec)  in function_profile_call()
807 	rec->counter++;  in function_profile_call()
842 	struct ftrace_profile *rec;  in profile_graph_return()  local
870 	rec = ftrace_find_profiled_func(stat, trace->func);  in profile_graph_return()
871 	if (rec) {  in profile_graph_return()
872 		rec->time += calltime;  in profile_graph_return()
873 		rec->time_squared += calltime * calltime;  in profile_graph_return()
1538 #define do_for_each_ftrace_rec(pg, rec)					\  argument
1542 			rec = &pg->records[_____i];
1552 	const struct dyn_ftrace *rec = b;  in ftrace_cmp_recs()  local
1554 	if (key->flags < rec->ip)  in ftrace_cmp_recs()
1556 	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)  in ftrace_cmp_recs()
1564 	struct dyn_ftrace *rec = NULL;  in lookup_rec()  local
1575 		rec = bsearch(&key, pg->records, pg->index,  in lookup_rec()
1578 		if (rec)  in lookup_rec()
1581 	return rec;  in lookup_rec()
1598 	struct dyn_ftrace *rec;  in ftrace_location_range()  local
1600 	rec = lookup_rec(start, end);  in ftrace_location_range()
1601 	if (rec)  in ftrace_location_range()
1602 		return rec->ip;  in ftrace_location_range()
1617 	struct dyn_ftrace *rec;  in ftrace_location()  local
1621 	rec = lookup_rec(ip, ip);  in ftrace_location()
1622 	if (!rec) {  in ftrace_location()
1628 			rec = lookup_rec(ip, ip + size - 1);  in ftrace_location()
1631 	if (rec)  in ftrace_location()
1632 		return rec->ip;  in ftrace_location()
1659 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)  in test_rec_ops_needs_regs()  argument
1667 		if (ftrace_ops_test(ops, rec->ip, rec)) {  in test_rec_ops_needs_regs()
1679 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1681 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
1683 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1685 static bool skip_record(struct dyn_ftrace *rec)  in skip_record()  argument
1693 	return rec->flags & FTRACE_FL_DISABLED &&  in skip_record()
1694 		!(rec->flags & FTRACE_FL_ENABLED);  in skip_record()
1704 	struct dyn_ftrace *rec;  in __ftrace_hash_rec_update()  local
1741 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_rec_update()
1746 		if (skip_record(rec))  in __ftrace_hash_rec_update()
1754 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))  in __ftrace_hash_rec_update()
1757 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);  in __ftrace_hash_rec_update()
1758 			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);  in __ftrace_hash_rec_update()
1780 			rec->flags++;  in __ftrace_hash_rec_update()
1781 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))  in __ftrace_hash_rec_update()
1785 				rec->flags |= FTRACE_FL_DIRECT;  in __ftrace_hash_rec_update()
1792 			if (ftrace_rec_count(rec) == 1 && ops->trampoline)  in __ftrace_hash_rec_update()
1793 				rec->flags |= FTRACE_FL_TRAMP;  in __ftrace_hash_rec_update()
1801 				rec->flags &= ~FTRACE_FL_TRAMP;  in __ftrace_hash_rec_update()
1808 				rec->flags |= FTRACE_FL_REGS;  in __ftrace_hash_rec_update()
1810 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))  in __ftrace_hash_rec_update()
1812 			rec->flags--;  in __ftrace_hash_rec_update()
1821 				rec->flags &= ~FTRACE_FL_DIRECT;  in __ftrace_hash_rec_update()
1829 			if (ftrace_rec_count(rec) > 0 &&  in __ftrace_hash_rec_update()
1830 			    rec->flags & FTRACE_FL_REGS &&  in __ftrace_hash_rec_update()
1832 				if (!test_rec_ops_needs_regs(rec))  in __ftrace_hash_rec_update()
1833 					rec->flags &= ~FTRACE_FL_REGS;  in __ftrace_hash_rec_update()
1843 			if (ftrace_rec_count(rec) == 1 &&  in __ftrace_hash_rec_update()
1844 			    ftrace_find_tramp_ops_any_other(rec, ops))  in __ftrace_hash_rec_update()
1845 				rec->flags |= FTRACE_FL_TRAMP;  in __ftrace_hash_rec_update()
1847 				rec->flags &= ~FTRACE_FL_TRAMP;  in __ftrace_hash_rec_update()
1860 		    ftrace_rec_count(rec) == 1 &&  in __ftrace_hash_rec_update()
1862 			rec->flags |= FTRACE_FL_CALL_OPS;  in __ftrace_hash_rec_update()
1864 			rec->flags &= ~FTRACE_FL_CALL_OPS;  in __ftrace_hash_rec_update()
1869 		update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;  in __ftrace_hash_rec_update()
1947 	struct dyn_ftrace *rec, *end = NULL;  in __ftrace_hash_update_ipmodify()  local
1974 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_update_ipmodify()
1976 		if (rec->flags & FTRACE_FL_DISABLED)  in __ftrace_hash_update_ipmodify()
1980 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);  in __ftrace_hash_update_ipmodify()
1981 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);  in __ftrace_hash_update_ipmodify()
1986 			if (rec->flags & FTRACE_FL_IPMODIFY) {  in __ftrace_hash_update_ipmodify()
1993 				FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT);  in __ftrace_hash_update_ipmodify()
2007 				rec->flags |= FTRACE_FL_IPMODIFY;  in __ftrace_hash_update_ipmodify()
2010 			rec->flags &= ~FTRACE_FL_IPMODIFY;  in __ftrace_hash_update_ipmodify()
2017 	end = rec;  in __ftrace_hash_update_ipmodify()
2020 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_update_ipmodify()
2022 		if (rec->flags & FTRACE_FL_DISABLED)  in __ftrace_hash_update_ipmodify()
2025 		if (rec == end)  in __ftrace_hash_update_ipmodify()
2028 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);  in __ftrace_hash_update_ipmodify()
2029 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);  in __ftrace_hash_update_ipmodify()
2034 			rec->flags &= ~FTRACE_FL_IPMODIFY;  in __ftrace_hash_update_ipmodify()
2036 			rec->flags |= FTRACE_FL_IPMODIFY;  in __ftrace_hash_update_ipmodify()
2126 void ftrace_bug(int failed, struct dyn_ftrace *rec)  in ftrace_bug()  argument
2128 	unsigned long ip = rec ? rec->ip : 0;  in ftrace_bug()
2156 	if (rec) {  in ftrace_bug()
2159 		pr_info("ftrace record flags: %lx\n", rec->flags);  in ftrace_bug()
2160 		pr_cont(" (%ld)%s%s", ftrace_rec_count(rec),  in ftrace_bug()
2161 			rec->flags & FTRACE_FL_REGS ? " R" : "  ",  in ftrace_bug()
2162 			rec->flags & FTRACE_FL_CALL_OPS ? " O" : "  ");  in ftrace_bug()
2163 		if (rec->flags & FTRACE_FL_TRAMP_EN) {  in ftrace_bug()
2164 			ops = ftrace_find_tramp_ops_any(rec);  in ftrace_bug()
2170 					ops = ftrace_find_tramp_ops_next(rec, ops);  in ftrace_bug()
2176 		ip = ftrace_get_addr_curr(rec);  in ftrace_bug()
2183 static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)  in ftrace_check_record()  argument
2189 	if (skip_record(rec))  in ftrace_check_record()
2203 	if (enable && ftrace_rec_count(rec))  in ftrace_check_record()
2213 		if (!(rec->flags & FTRACE_FL_REGS) !=  in ftrace_check_record()
2214 		    !(rec->flags & FTRACE_FL_REGS_EN))  in ftrace_check_record()
2217 		if (!(rec->flags & FTRACE_FL_TRAMP) !=  in ftrace_check_record()
2218 		    !(rec->flags & FTRACE_FL_TRAMP_EN))  in ftrace_check_record()
2232 		if (ftrace_rec_count(rec) == 1) {  in ftrace_check_record()
2233 			if (!(rec->flags & FTRACE_FL_DIRECT) !=  in ftrace_check_record()
2234 			    !(rec->flags & FTRACE_FL_DIRECT_EN))  in ftrace_check_record()
2236 		} else if (rec->flags & FTRACE_FL_DIRECT_EN) {  in ftrace_check_record()
2245 		if (ftrace_rec_count(rec) == 1) {  in ftrace_check_record()
2246 			if (!(rec->flags & FTRACE_FL_CALL_OPS) !=  in ftrace_check_record()
2247 			    !(rec->flags & FTRACE_FL_CALL_OPS_EN))  in ftrace_check_record()
2249 		} else if (rec->flags & FTRACE_FL_CALL_OPS_EN) {  in ftrace_check_record()
2255 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)  in ftrace_check_record()
2260 		flag ^= rec->flags & FTRACE_FL_ENABLED;  in ftrace_check_record()
2263 			rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED;  in ftrace_check_record()
2265 				if (rec->flags & FTRACE_FL_REGS)  in ftrace_check_record()
2266 					rec->flags |= FTRACE_FL_REGS_EN;  in ftrace_check_record()
2268 					rec->flags &= ~FTRACE_FL_REGS_EN;  in ftrace_check_record()
2271 				if (rec->flags & FTRACE_FL_TRAMP)  in ftrace_check_record()
2272 					rec->flags |= FTRACE_FL_TRAMP_EN;  in ftrace_check_record()
2274 					rec->flags &= ~FTRACE_FL_TRAMP_EN;  in ftrace_check_record()
2278 			if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY))  in ftrace_check_record()
2279 				rec->flags |= FTRACE_FL_MODIFIED;  in ftrace_check_record()
2287 				if (ftrace_rec_count(rec) == 1) {  in ftrace_check_record()
2288 					if (rec->flags & FTRACE_FL_DIRECT)  in ftrace_check_record()
2289 						rec->flags |= FTRACE_FL_DIRECT_EN;  in ftrace_check_record()
2291 						rec->flags &= ~FTRACE_FL_DIRECT_EN;  in ftrace_check_record()
2297 					rec->flags &= ~FTRACE_FL_DIRECT_EN;  in ftrace_check_record()
2302 				if (ftrace_rec_count(rec) == 1) {  in ftrace_check_record()
2303 					if (rec->flags & FTRACE_FL_CALL_OPS)  in ftrace_check_record()
2304 						rec->flags |= FTRACE_FL_CALL_OPS_EN;  in ftrace_check_record()
2306 						rec->flags &= ~FTRACE_FL_CALL_OPS_EN;  in ftrace_check_record()
2312 					rec->flags &= ~FTRACE_FL_CALL_OPS_EN;  in ftrace_check_record()
2336 		if (!ftrace_rec_count(rec))  in ftrace_check_record()
2337 			rec->flags &= FTRACE_NOCLEAR_FLAGS;  in ftrace_check_record()
2343 			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |  in ftrace_check_record()
2360 int ftrace_update_record(struct dyn_ftrace *rec, bool enable)  in ftrace_update_record()  argument
2362 	return ftrace_check_record(rec, enable, true);  in ftrace_update_record()
2374 int ftrace_test_record(struct dyn_ftrace *rec, bool enable)  in ftrace_test_record()  argument
2376 	return ftrace_check_record(rec, enable, false);  in ftrace_test_record()
2380 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)  in ftrace_find_tramp_ops_any()  argument
2383 	unsigned long ip = rec->ip;  in ftrace_find_tramp_ops_any()
2398 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)  in ftrace_find_tramp_ops_any_other()  argument
2401 	unsigned long ip = rec->ip;  in ftrace_find_tramp_ops_any_other()
2416 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,  in ftrace_find_tramp_ops_next()  argument
2419 	unsigned long ip = rec->ip;  in ftrace_find_tramp_ops_next()
2434 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)  in ftrace_find_tramp_ops_curr()  argument
2437 	unsigned long ip = rec->ip;  in ftrace_find_tramp_ops_curr()
2504 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)  in ftrace_find_tramp_ops_new()  argument
2507 	unsigned long ip = rec->ip;  in ftrace_find_tramp_ops_new()
2519 ftrace_find_unique_ops(struct dyn_ftrace *rec)  in ftrace_find_unique_ops()  argument
2522 	unsigned long ip = rec->ip;  in ftrace_find_unique_ops()
2613 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)  in ftrace_get_addr_new()  argument
2618 	if ((rec->flags & FTRACE_FL_DIRECT) &&  in ftrace_get_addr_new()
2619 	    (ftrace_rec_count(rec) == 1)) {  in ftrace_get_addr_new()
2620 		addr = ftrace_find_rec_direct(rec->ip);  in ftrace_get_addr_new()
2627 	if (rec->flags & FTRACE_FL_TRAMP) {  in ftrace_get_addr_new()
2628 		ops = ftrace_find_tramp_ops_new(rec);  in ftrace_get_addr_new()
2631 				(void *)rec->ip, (void *)rec->ip, rec->flags);  in ftrace_get_addr_new()
2638 	if (rec->flags & FTRACE_FL_REGS)  in ftrace_get_addr_new()
2654 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)  in ftrace_get_addr_curr()  argument
2660 	if (rec->flags & FTRACE_FL_DIRECT_EN) {  in ftrace_get_addr_curr()
2661 		addr = ftrace_find_rec_direct(rec->ip);  in ftrace_get_addr_curr()
2668 	if (rec->flags & FTRACE_FL_TRAMP_EN) {  in ftrace_get_addr_curr()
2669 		ops = ftrace_find_tramp_ops_curr(rec);  in ftrace_get_addr_curr()
2672 				(void *)rec->ip, (void *)rec->ip);  in ftrace_get_addr_curr()
2679 	if (rec->flags & FTRACE_FL_REGS_EN)  in ftrace_get_addr_curr()
2686 __ftrace_replace_code(struct dyn_ftrace *rec, bool enable)  in __ftrace_replace_code()  argument
2692 	ftrace_addr = ftrace_get_addr_new(rec);  in __ftrace_replace_code()
2695 	ftrace_old_addr = ftrace_get_addr_curr(rec);  in __ftrace_replace_code()
2697 	ret = ftrace_update_record(rec, enable);  in __ftrace_replace_code()
2707 		return ftrace_make_call(rec, ftrace_addr);  in __ftrace_replace_code()
2711 		return ftrace_make_nop(NULL, rec, ftrace_old_addr);  in __ftrace_replace_code()
2715 		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);  in __ftrace_replace_code()
2723 	struct dyn_ftrace *rec;  in ftrace_replace_code()  local
2732 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_replace_code()
2734 		if (skip_record(rec))  in ftrace_replace_code()
2737 		failed = __ftrace_replace_code(rec, enable);  in ftrace_replace_code()
2739 			ftrace_bug(failed, rec);  in ftrace_replace_code()
2821 ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)  in ftrace_nop_initialize()  argument
2828 	ret = ftrace_init_nop(mod, rec);  in ftrace_nop_initialize()
2831 		ftrace_bug(ret, rec);  in ftrace_nop_initialize()
3155 		struct dyn_ftrace *rec;  in ftrace_shutdown()  local
3157 		do_for_each_ftrace_rec(pg, rec) {  in ftrace_shutdown()
3158 			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS))  in ftrace_shutdown()
3160 					(void *)rec->ip, rec->flags);  in ftrace_shutdown()
3598 	struct dyn_ftrace *rec = NULL;  in t_func_next()  local
3610 		rec = &iter->pg->records[iter->idx++];  in t_func_next()
3612 		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||  in t_func_next()
3615 		     !(rec->flags & FTRACE_FL_ENABLED)) ||  in t_func_next()
3618 		     !(rec->flags & FTRACE_FL_TOUCHED))) {  in t_func_next()
3620 			rec = NULL;  in t_func_next()
3625 	if (!rec)  in t_func_next()
3629 	iter->func = rec;  in t_func_next()
3732 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)  in arch_ftrace_trampoline_func()  argument
3738 				struct dyn_ftrace *rec)  in add_trampoline_func()  argument
3742 	ptr = arch_ftrace_trampoline_func(ops, rec);  in add_trampoline_func()
3758 static int test_for_valid_rec(struct dyn_ftrace *rec)  in test_for_valid_rec()  argument
3764 	ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str);  in test_for_valid_rec()
3768 		rec->flags |= FTRACE_FL_DISABLED;  in test_for_valid_rec()
3783 	struct dyn_ftrace *rec;  in ftrace_check_work_func()  local
3786 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_check_work_func()
3787 		test_for_valid_rec(rec);  in ftrace_check_work_func()
3834 static inline int test_for_valid_rec(struct dyn_ftrace *rec)  in test_for_valid_rec()  argument
3849 	struct dyn_ftrace *rec;  in t_show()  local
3865 	rec = iter->func;  in t_show()
3867 	if (!rec)  in t_show()
3871 		seq_printf(m, "%lx ", rec->ip);  in t_show()
3873 	if (print_rec(m, rec->ip)) {  in t_show()
3875 		WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED));  in t_show()
3884 			   ftrace_rec_count(rec),  in t_show()
3885 			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",  in t_show()
3886 			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ",  in t_show()
3887 			   rec->flags & FTRACE_FL_DIRECT ? " D" : "  ",  in t_show()
3888 			   rec->flags & FTRACE_FL_CALL_OPS ? " O" : "  ",  in t_show()
3889 			   rec->flags & FTRACE_FL_MODIFIED ? " M " : "   ");  in t_show()
3890 		if (rec->flags & FTRACE_FL_TRAMP_EN) {  in t_show()
3891 			ops = ftrace_find_tramp_ops_any(rec);  in t_show()
3897 					add_trampoline_func(m, ops, rec);  in t_show()
3898 					ops = ftrace_find_tramp_ops_next(rec, ops);  in t_show()
3903 			add_trampoline_func(m, NULL, rec);  in t_show()
3905 		if (rec->flags & FTRACE_FL_CALL_OPS_EN) {  in t_show()
3906 			ops = ftrace_find_unique_ops(rec);  in t_show()
3914 		if (rec->flags & FTRACE_FL_DIRECT) {  in t_show()
3917 			direct = ftrace_find_rec_direct(rec->ip);  in t_show()
4210 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)  in enter_record()  argument
4215 	entry = ftrace_lookup_ip(hash, rec->ip);  in enter_record()
4227 		ret = add_hash_entry(hash, rec->ip);  in enter_record()
4238 	struct dyn_ftrace *rec;  in add_rec_by_index()  local
4244 	do_for_each_ftrace_rec(pg, rec) {  in add_rec_by_index()
4250 		rec = &pg->records[index];  in add_rec_by_index()
4251 		enter_record(hash, rec, clear_filter);  in add_rec_by_index()
4276 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,  in ftrace_match_record()  argument
4282 	if (lookup_ip(rec->ip, &modname, str)) {  in ftrace_match_record()
4285 			     !(rec->flags & FTRACE_FL_DISABLED));  in ftrace_match_record()
4322 	struct dyn_ftrace *rec;  in match_records()  local
4353 	do_for_each_ftrace_rec(pg, rec) {  in match_records()
4355 		if (rec->flags & FTRACE_FL_DISABLED)  in match_records()
4358 		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {  in match_records()
4359 			ret = enter_record(hash, rec, clear_filter);  in match_records()
6280 	struct dyn_ftrace *rec;  in ftrace_graph_set_hash()  local
6299 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_graph_set_hash()
6301 		if (rec->flags & FTRACE_FL_DISABLED)  in ftrace_graph_set_hash()
6304 		if (ftrace_match_record(rec, &func_g, NULL, 0)) {  in ftrace_graph_set_hash()
6305 			entry = ftrace_lookup_ip(hash, rec->ip);  in ftrace_graph_set_hash()
6312 				if (add_hash_entry(hash, rec->ip) < 0)  in ftrace_graph_set_hash()
6483 	struct dyn_ftrace *rec;  in ftrace_process_locs()  local
6560 		rec = &pg->records[pg->index++];  in ftrace_process_locs()
6561 		rec->ip = addr;  in ftrace_process_locs()
6673 static int referenced_filters(struct dyn_ftrace *rec)  in referenced_filters()  argument
6679 		if (ops_references_ip(ops, rec->ip)) {  in referenced_filters()
6686 				rec->flags |= FTRACE_FL_REGS;  in referenced_filters()
6688 				rec->flags |= FTRACE_FL_TRAMP;  in referenced_filters()
6690 				rec->flags &= ~FTRACE_FL_TRAMP;  in referenced_filters()
6701 	struct dyn_ftrace *rec;  in clear_mod_from_hash()  local
6708 		rec = &pg->records[i];  in clear_mod_from_hash()
6709 		entry = __ftrace_lookup_ip(hash, rec->ip);  in clear_mod_from_hash()
6757 	struct dyn_ftrace *rec;  in ftrace_release_mod()  local
6781 		rec = &pg->records[0];  in ftrace_release_mod()
6782 		if (within_module(rec->ip, mod)) {  in ftrace_release_mod()
6822 	struct dyn_ftrace *rec;  in ftrace_module_enable()  local
6846 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_module_enable()
6854 		if (!within_module(rec->ip, mod))  in ftrace_module_enable()
6858 		if (!test_for_valid_rec(rec)) {  in ftrace_module_enable()
6860 			rec->flags = FTRACE_FL_DISABLED;  in ftrace_module_enable()
6873 			cnt += referenced_filters(rec);  in ftrace_module_enable()
6875 		rec->flags &= ~FTRACE_FL_DISABLED;  in ftrace_module_enable()
6876 		rec->flags += cnt;  in ftrace_module_enable()
6879 			int failed = __ftrace_replace_code(rec, 1);  in ftrace_module_enable()
6881 				ftrace_bug(failed, rec);  in ftrace_module_enable()
6913 				struct dyn_ftrace *rec)  in save_ftrace_mod_rec()  argument
6922 	ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);  in save_ftrace_mod_rec()
6936 	mod_func->ip = rec->ip - offset;  in save_ftrace_mod_rec()
7059 				struct dyn_ftrace *rec) { }  in save_ftrace_mod_rec()  argument
7119 				   struct dyn_ftrace *rec)  in add_to_clear_hash_list()  argument
7129 	func->ip = rec->ip;  in add_to_clear_hash_list()
7139 	struct dyn_ftrace *rec;  in ftrace_free_mem()  local
7163 		rec = bsearch(&key, pg->records, pg->index,  in ftrace_free_mem()
7166 		if (!rec)  in ftrace_free_mem()
7170 		add_to_clear_hash_list(&clear_hash, rec);  in ftrace_free_mem()
7173 			save_ftrace_mod_rec(mod_map, rec);  in ftrace_free_mem()
7190 		memmove(rec, rec + 1,  in ftrace_free_mem()
7191 			(pg->index - (rec - pg->records)) * sizeof(*rec));  in ftrace_free_mem()