433 	struct ftrace_profile_page *pg;  in function_stat_next()  local
435 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);  in function_stat_next()
441 	if ((void *)rec >= (void *)&pg->records[pg->index]) {  in function_stat_next()
442 		pg = pg->next;  in function_stat_next()
443 		if (!pg)  in function_stat_next()
445 		rec = &pg->records[0];  in function_stat_next()
574 	struct ftrace_profile_page *pg;  in ftrace_profile_reset()  local
576 	pg = stat->pages = stat->start;  in ftrace_profile_reset()
578 	while (pg) {  in ftrace_profile_reset()
579 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);  in ftrace_profile_reset()
580 		pg->index = 0;  in ftrace_profile_reset()
581 		pg = pg->next;  in ftrace_profile_reset()
590 	struct ftrace_profile_page *pg;  in ftrace_profile_pages_init()  local
616 	pg = stat->start = stat->pages;  in ftrace_profile_pages_init()
621 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);  in ftrace_profile_pages_init()
622 		if (!pg->next)  in ftrace_profile_pages_init()
624 		pg = pg->next;  in ftrace_profile_pages_init()
630 	pg = stat->start;  in ftrace_profile_pages_init()
631 	while (pg) {  in ftrace_profile_pages_init()
632 		unsigned long tmp = (unsigned long)pg;  in ftrace_profile_pages_init()
634 		pg = pg->next;  in ftrace_profile_pages_init()
1514 #define do_for_each_ftrace_rec(pg, rec)					\  argument
1515 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1517 		for (_____i = 0; _____i < pg->index; _____i++) {	\
1518 			rec = &pg->records[_____i];
1551 	struct ftrace_page *pg;  in ftrace_location_range()  local
1558 	for (pg = ftrace_pages_start; pg; pg = pg->next) {  in ftrace_location_range()
1559 		if (end < pg->records[0].ip ||  in ftrace_location_range()
1560 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))  in ftrace_location_range()
1562 		rec = bsearch(&key, pg->records, pg->index,  in ftrace_location_range()
1637 	struct ftrace_page *pg;  in __ftrace_hash_rec_update()  local
1675 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_rec_update()
1849 	struct ftrace_page *pg;  in __ftrace_hash_update_ipmodify()  local
1868 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_update_ipmodify()
1894 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_update_ipmodify()
2400 	struct ftrace_page *pg;  in ftrace_replace_code()  local
2408 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_replace_code()
2425 	struct ftrace_page	*pg;  member
2447 	iter->pg = ftrace_pages_start;  in ftrace_rec_iter_start()
2451 	while (iter->pg && !iter->pg->index)  in ftrace_rec_iter_start()
2452 		iter->pg = iter->pg->next;  in ftrace_rec_iter_start()
2454 	if (!iter->pg)  in ftrace_rec_iter_start()
2470 	if (iter->index >= iter->pg->index) {  in ftrace_rec_iter_next()
2471 		iter->pg = iter->pg->next;  in ftrace_rec_iter_next()
2475 		while (iter->pg && !iter->pg->index)  in ftrace_rec_iter_next()
2476 			iter->pg = iter->pg->next;  in ftrace_rec_iter_next()
2479 	if (!iter->pg)  in ftrace_rec_iter_next()
2493 	return &iter->pg->records[iter->index];  in ftrace_rec_iter_record()
2778 		struct ftrace_page *pg;  in ftrace_shutdown()  local
2781 		do_for_each_ftrace_rec(pg, rec) {  in ftrace_shutdown()
2907 	struct ftrace_page *pg;  in ftrace_update_code()  local
2930 	for (pg = new_pgs; pg; pg = pg->next) {  in ftrace_update_code()
2932 		for (i = 0; i < pg->index; i++) {  in ftrace_update_code()
2938 			p = &pg->records[i];  in ftrace_update_code()
2960 static int ftrace_allocate_records(struct ftrace_page *pg, int count)  in ftrace_allocate_records()  argument
2978 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);  in ftrace_allocate_records()
2980 	if (!pg->records) {  in ftrace_allocate_records()
2989 	pg->size = cnt;  in ftrace_allocate_records()
3001 	struct ftrace_page *pg;  in ftrace_allocate_pages()  local
3008 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);  in ftrace_allocate_pages()
3009 	if (!pg)  in ftrace_allocate_pages()
3018 		cnt = ftrace_allocate_records(pg, num_to_init);  in ftrace_allocate_pages()
3026 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);  in ftrace_allocate_pages()
3027 		if (!pg->next)  in ftrace_allocate_pages()
3030 		pg = pg->next;  in ftrace_allocate_pages()
3036 	pg = start_pg;  in ftrace_allocate_pages()
3037 	while (pg) {  in ftrace_allocate_pages()
3038 		order = get_count_order(pg->size / ENTRIES_PER_PAGE);  in ftrace_allocate_pages()
3039 		free_pages((unsigned long)pg->records, order);  in ftrace_allocate_pages()
3040 		start_pg = pg->next;  in ftrace_allocate_pages()
3041 		kfree(pg);  in ftrace_allocate_pages()
3042 		pg = start_pg;  in ftrace_allocate_pages()
3054 	struct ftrace_page		*pg;  member
3284 	if (iter->idx >= iter->pg->index) {  in t_func_next()
3285 		if (iter->pg->next) {  in t_func_next()
3286 			iter->pg = iter->pg->next;  in t_func_next()
3291 		rec = &iter->pg->records[iter->idx++];  in t_func_next()
3390 	iter->pg = ftrace_pages_start;  in t_start()
3503 	iter->pg = ftrace_pages_start;  in ftrace_avail_open()
3527 	iter->pg = ftrace_pages_start;  in ftrace_enabled_open()
3611 		iter->pg = ftrace_pages_start;  in ftrace_regex_open()
3739 	struct ftrace_page *pg;  in add_rec_by_index()  local
3746 	do_for_each_ftrace_rec(pg, rec) {  in add_rec_by_index()
3747 		if (pg->index <= index) {  in add_rec_by_index()
3748 			index -= pg->index;  in add_rec_by_index()
3752 		rec = &pg->records[index];  in add_rec_by_index()
3800 	struct ftrace_page *pg;  in match_records()  local
3832 	do_for_each_ftrace_rec(pg, rec) {  in match_records()
5399 	struct ftrace_page *pg;  in ftrace_graph_set_hash()  local
5417 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_graph_set_hash()
5572 	struct ftrace_page *pg;  in ftrace_process_locs()  local
5617 	pg = start_pg;  in ftrace_process_locs()
5629 		if (pg->index == pg->size) {  in ftrace_process_locs()
5631 			if (WARN_ON(!pg->next))  in ftrace_process_locs()
5633 			pg = pg->next;  in ftrace_process_locs()
5636 		rec = &pg->records[pg->index++];  in ftrace_process_locs()
5641 	WARN_ON(pg->next);  in ftrace_process_locs()
5644 	ftrace_pages = pg;  in ftrace_process_locs()
5703 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)  in clear_mod_from_hash()  argument
5712 	for (i = 0; i < pg->index; i++) {  in clear_mod_from_hash()
5713 		rec = &pg->records[i];  in clear_mod_from_hash()
5726 static void clear_mod_from_hashes(struct ftrace_page *pg)  in clear_mod_from_hashes()  argument
5735 		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);  in clear_mod_from_hashes()
5736 		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);  in clear_mod_from_hashes()
5765 	struct ftrace_page *pg;  in ftrace_release_mod()  local
5786 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {  in ftrace_release_mod()
5787 		rec = &pg->records[0];  in ftrace_release_mod()
5794 			if (WARN_ON(pg == ftrace_pages_start))  in ftrace_release_mod()
5798 			if (pg == ftrace_pages)  in ftrace_release_mod()
5801 			ftrace_update_tot_cnt -= pg->index;  in ftrace_release_mod()
5802 			*last_pg = pg->next;  in ftrace_release_mod()
5804 			pg->next = tmp_page;  in ftrace_release_mod()
5805 			tmp_page = pg;  in ftrace_release_mod()
5807 			last_pg = &pg->next;  in ftrace_release_mod()
5812 	for (pg = tmp_page; pg; pg = tmp_page) {  in ftrace_release_mod()
5815 		clear_mod_from_hashes(pg);  in ftrace_release_mod()
5817 		order = get_count_order(pg->size / ENTRIES_PER_PAGE);  in ftrace_release_mod()
5818 		free_pages((unsigned long)pg->records, order);  in ftrace_release_mod()
5819 		tmp_page = pg->next;  in ftrace_release_mod()
5820 		kfree(pg);  in ftrace_release_mod()
5827 	struct ftrace_page *pg;  in ftrace_module_enable()  local
5850 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_module_enable()
6116 	struct ftrace_page *pg;  in ftrace_free_mem()  local
6139 	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {  in ftrace_free_mem()
6140 		if (end < pg->records[0].ip ||  in ftrace_free_mem()
6141 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))  in ftrace_free_mem()
6144 		rec = bsearch(&key, pg->records, pg->index,  in ftrace_free_mem()
6156 		pg->index--;  in ftrace_free_mem()
6158 		if (!pg->index) {  in ftrace_free_mem()
6159 			*last_pg = pg->next;  in ftrace_free_mem()
6160 			order = get_count_order(pg->size / ENTRIES_PER_PAGE);  in ftrace_free_mem()
6161 			free_pages((unsigned long)pg->records, order);  in ftrace_free_mem()
6162 			kfree(pg);  in ftrace_free_mem()
6163 			pg = container_of(last_pg, struct ftrace_page, next);  in ftrace_free_mem()
6165 				ftrace_pages = pg;  in ftrace_free_mem()
6169 			(pg->index - (rec - pg->records)) * sizeof(*rec));  in ftrace_free_mem()