352 static void rb_init_page(struct buffer_data_page *bpage)  in rb_init_page()  argument
354 	local_set(&bpage->commit, 0);  in rb_init_page()
357 static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage)  in rb_page_commit()  argument
359 	return local_read(&bpage->page->commit);  in rb_page_commit()
362 static void free_buffer_page(struct buffer_page *bpage)  in free_buffer_page()  argument
364 	free_page((unsigned long)bpage->page);  in free_buffer_page()
365 	kfree(bpage);  in free_buffer_page()
1450 static inline void rb_inc_page(struct buffer_page **bpage)  in rb_inc_page()  argument
1452 	struct list_head *p = rb_list_head((*bpage)->list.next);  in rb_inc_page()
1454 	*bpage = list_entry(p, struct buffer_page, list);  in rb_inc_page()
1572 			  struct buffer_page *bpage)  in rb_check_bpage()  argument
1574 	unsigned long val = (unsigned long)bpage;  in rb_check_bpage()
1613 	struct buffer_page *bpage, *tmp;  in __rb_allocate_pages()  local
1650 		bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),  in __rb_allocate_pages()
1652 		if (!bpage)  in __rb_allocate_pages()
1655 		rb_check_bpage(cpu_buffer, bpage);  in __rb_allocate_pages()
1657 		list_add(&bpage->list, pages);  in __rb_allocate_pages()
1662 		bpage->page = page_address(page);  in __rb_allocate_pages()
1663 		rb_init_page(bpage->page);  in __rb_allocate_pages()
1674 	list_for_each_entry_safe(bpage, tmp, pages, list) {  in __rb_allocate_pages()
1675 		list_del_init(&bpage->list);  in __rb_allocate_pages()
1676 		free_buffer_page(bpage);  in __rb_allocate_pages()
1713 	struct buffer_page *bpage;  in rb_allocate_cpu_buffer()  local
1733 	bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),  in rb_allocate_cpu_buffer()
1735 	if (!bpage)  in rb_allocate_cpu_buffer()
1738 	rb_check_bpage(cpu_buffer, bpage);  in rb_allocate_cpu_buffer()
1740 	cpu_buffer->reader_page = bpage;  in rb_allocate_cpu_buffer()
1744 	bpage->page = page_address(page);  in rb_allocate_cpu_buffer()
1745 	rb_init_page(bpage->page);  in rb_allocate_cpu_buffer()
1773 	struct buffer_page *bpage, *tmp;  in rb_free_cpu_buffer()  local
1782 		list_for_each_entry_safe(bpage, tmp, head, list) {  in rb_free_cpu_buffer()
1783 			list_del_init(&bpage->list);  in rb_free_cpu_buffer()
1784 			free_buffer_page(bpage);  in rb_free_cpu_buffer()
1786 		bpage = list_entry(head, struct buffer_page, list);  in rb_free_cpu_buffer()
1787 		free_buffer_page(bpage);  in rb_free_cpu_buffer()
1913 static inline unsigned long rb_page_entries(struct buffer_page *bpage)  in rb_page_entries()  argument
1915 	return local_read(&bpage->entries) & RB_WRITE_MASK;  in rb_page_entries()
1918 static inline unsigned long rb_page_write(struct buffer_page *bpage)  in rb_page_write()  argument
1920 	return local_read(&bpage->write) & RB_WRITE_MASK;  in rb_page_write()
2103 		struct buffer_page *bpage, *tmp;  in rb_insert_pages()  local
2104 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,  in rb_insert_pages()
2106 			list_del_init(&bpage->list);  in rb_insert_pages()
2107 			free_buffer_page(bpage);  in rb_insert_pages()
2334 		struct buffer_page *bpage, *tmp;  in ring_buffer_resize()  local
2342 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,  in ring_buffer_resize()
2344 			list_del_init(&bpage->list);  in ring_buffer_resize()
2345 			free_buffer_page(bpage);  in ring_buffer_resize()
2366 static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)  in __rb_page_index()  argument
2368 	return bpage->page->data + index;  in __rb_page_index()
2438 static __always_inline unsigned rb_page_size(struct buffer_page *bpage)  in rb_page_size()  argument
2440 	return rb_page_commit(bpage);  in rb_page_size()
3008 	struct buffer_page *bpage;  in rb_try_to_discard()  local
3018 	bpage = READ_ONCE(cpu_buffer->tail_page);  in rb_try_to_discard()
3028 	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {  in rb_try_to_discard()
3030 			local_read(&bpage->write) & ~RB_WRITE_MASK;  in rb_try_to_discard()
3067 		if (local_try_cmpxchg(&bpage->write, &old_index, new_index)) {  in rb_try_to_discard()
3410 static void dump_buffer_page(struct buffer_data_page *bpage,  in dump_buffer_page()  argument
3418 	ts = bpage->time_stamp;  in dump_buffer_page()
3423 		event = (struct ring_buffer_event *)(bpage->data + e);  in dump_buffer_page()
3467 	struct buffer_data_page *bpage;  in check_buffer()  local
3472 	bpage = info->tail_page->page;  in check_buffer()
3476 		tail = local_read(&bpage->commit);  in check_buffer()
3487 	if (tail <= 8 || tail > local_read(&bpage->commit))  in check_buffer()
3496 	ts = bpage->time_stamp;  in check_buffer()
3500 		event = (struct ring_buffer_event *)(bpage->data + e);  in check_buffer()
3541 		dump_buffer_page(bpage, info, tail);  in check_buffer()
3847 	struct buffer_page *bpage = cpu_buffer->commit_page;  in rb_decrement_entry()  local
3853 	if (likely(bpage->page == (void *)addr)) {  in rb_decrement_entry()
3854 		local_dec(&bpage->entries);  in rb_decrement_entry()
3862 	rb_inc_page(&bpage);  in rb_decrement_entry()
3863 	start = bpage;  in rb_decrement_entry()
3865 		if (bpage->page == (void *)addr) {  in rb_decrement_entry()
3866 			local_dec(&bpage->entries);  in rb_decrement_entry()
3869 		rb_inc_page(&bpage);  in rb_decrement_entry()
3870 	} while (bpage != start);  in rb_decrement_entry()
4197 	struct buffer_page *bpage;  in ring_buffer_oldest_event_ts()  local
4210 		bpage = cpu_buffer->reader_page;  in ring_buffer_oldest_event_ts()
4212 		bpage = rb_set_head_page(cpu_buffer);  in ring_buffer_oldest_event_ts()
4213 	if (bpage)  in ring_buffer_oldest_event_ts()
4214 		ret = bpage->page->time_stamp;  in ring_buffer_oldest_event_ts()
5597 	struct buffer_data_page *bpage = NULL;  in ring_buffer_alloc_read_page()  local
5609 		bpage = cpu_buffer->free_page;  in ring_buffer_alloc_read_page()
5616 	if (bpage)  in ring_buffer_alloc_read_page()
5624 	bpage = page_address(page);  in ring_buffer_alloc_read_page()
5627 	rb_init_page(bpage);  in ring_buffer_alloc_read_page()
5629 	return bpage;  in ring_buffer_alloc_read_page()
5644 	struct buffer_data_page *bpage = data;  in ring_buffer_free_read_page()  local
5645 	struct page *page = virt_to_page(bpage);  in ring_buffer_free_read_page()
5661 		cpu_buffer->free_page = bpage;  in ring_buffer_free_read_page()
5662 		bpage = NULL;  in ring_buffer_free_read_page()
5669 	free_page((unsigned long)bpage);  in ring_buffer_free_read_page()
5711 	struct buffer_data_page *bpage;  in ring_buffer_read_page()  local
5735 	bpage = *data_page;  in ring_buffer_read_page()
5736 	if (!bpage)  in ring_buffer_read_page()
5799 			memcpy(bpage->data + pos, rpage->data + rpos, size);  in ring_buffer_read_page()
5816 		local_set(&bpage->commit, pos);  in ring_buffer_read_page()
5817 		bpage->time_stamp = save_timestamp;  in ring_buffer_read_page()
5827 		rb_init_page(bpage);  in ring_buffer_read_page()
5828 		bpage = reader->page;  in ring_buffer_read_page()
5833 		*data_page = bpage;  in ring_buffer_read_page()
5841 			local_set(&bpage->commit, reader->real_end);  in ring_buffer_read_page()
5847 	commit = local_read(&bpage->commit);  in ring_buffer_read_page()
5856 			memcpy(&bpage->data[commit], &missed_events,  in ring_buffer_read_page()
5858 			local_add(RB_MISSED_STORED, &bpage->commit);  in ring_buffer_read_page()
5861 		local_add(RB_MISSED_EVENTS, &bpage->commit);  in ring_buffer_read_page()
5868 		memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);  in ring_buffer_read_page()