3  * Generic ring buffer
35  * The ring buffer header is special. We must manually up keep it.
57  * The ring buffer is made up of a list of pages. A separate list of pages is
58  * allocated for each CPU. A writer may only write to a buffer that is
60  * from any per cpu buffer.
62  * The reader is special. For each per cpu buffer, the reader has its own
64  * page is swapped with another page in the ring buffer.
68  * again (as long as it is out of the ring buffer).
73  *   |reader|          RING BUFFER
84  *   |reader|          RING BUFFER
95  *   |reader|          RING BUFFER
106  *   |buffer|          RING BUFFER
118  * and swap that into the ring buffer.
256 /* inline for ring buffer fast paths */
280 #define for_each_buffer_cpu(buffer, cpu)		\  argument
281 	for_each_cpu(cpu, buffer->cpumask)
283 #define for_each_online_buffer_cpu(buffer, cpu)		\  argument
284 	for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
309 	unsigned char	 data[] RB_ALIGN_DATA;	/* data of buffer page */
313  * Note, the buffer_page list must be first. The buffer pages
314  * are allocated in cache lines, which means that each buffer
317  * add flags in the list struct pointers, to make the ring buffer
321 	struct list_head list;		/* list of buffer pages */
330  * The buffer page counters, write and entries, must be reset
430  *  ABSOLUTE - the buffer requests all events to have absolute time stamps
483  * head_page == tail_page && head == tail then buffer is empty.
489 	struct trace_buffer	*buffer;  member
521 	/* ring buffer pages to update, > 0 to add, < 0 to remove */
566  * buffer doesn't need all the features of a true 64 bit atomic,
569  * For the ring buffer, 64 bit required operations for the time is
748  * is on the buffer that it passed in.
787 static inline u64 rb_time_stamp(struct trace_buffer *buffer);
791  * @buffer: The buffer that the event is on
795  * committed to the ring buffer. And must be called from the same
802  * the max nesting, then the write_stamp of the buffer is returned,
806 u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer,  in ring_buffer_event_time_stamp()  argument
809 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()];  in ring_buffer_event_time_stamp()
833 		ts = rb_time_stamp(cpu_buffer->buffer);  in ring_buffer_event_time_stamp()
839  * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
840  * @buffer: The ring_buffer to get the number of pages from
843  * Returns the number of pages used by a per_cpu buffer of the ring buffer.
845 size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu)  in ring_buffer_nr_pages()  argument
847 	return buffer->buffers[cpu]->nr_pages;  in ring_buffer_nr_pages()
851  * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
852  * @buffer: The ring_buffer to get the number of pages from
855  * Returns the number of pages that have content in the ring buffer.
857 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)  in ring_buffer_nr_dirty_pages()  argument
862 	read = local_read(&buffer->buffers[cpu]->pages_read);  in ring_buffer_nr_dirty_pages()
863 	cnt = local_read(&buffer->buffers[cpu]->pages_touched);  in ring_buffer_nr_dirty_pages()
874  * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
877  * ring buffer waiters queue.
891  * ring_buffer_wait - wait for input to the ring buffer
892  * @buffer: buffer to wait on
893  * @cpu: the cpu buffer to wait on
897  * as data is added to any of the @buffer's cpu buffers. Otherwise
898  * it will wait for data to be added to a specific cpu buffer.
900 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full)  in ring_buffer_wait()  argument
909 	 * data in any cpu buffer, or a specific buffer, put the  in ring_buffer_wait()
913 		work = &buffer->irq_work;  in ring_buffer_wait()
917 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_wait()
919 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_wait()
937 		 * We don't clear it even if the buffer is no longer  in ring_buffer_wait()
960 		if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer))  in ring_buffer_wait()
964 		    !ring_buffer_empty_cpu(buffer, cpu)) {  in ring_buffer_wait()
976 			dirty = ring_buffer_nr_dirty_pages(buffer, cpu);  in ring_buffer_wait()
998  * ring_buffer_poll_wait - poll on buffer input
999  * @buffer: buffer to wait on
1000  * @cpu: the cpu buffer to wait on
1005  * as data is added to any of the @buffer's cpu buffers. Otherwise
1006  * it will wait for data to be added to a specific cpu buffer.
1011 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,  in ring_buffer_poll_wait()  argument
1018 		work = &buffer->irq_work;  in ring_buffer_poll_wait()
1020 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_poll_wait()
1023 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_wait()
1031 	 * checking if the ring buffer is empty.  Once the waiters_pending bit  in ring_buffer_poll_wait()
1038 	 * the buffer goes from empty to having content.  But as this race is  in ring_buffer_poll_wait()
1044 	if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||  in ring_buffer_poll_wait()
1045 	    (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))  in ring_buffer_poll_wait()
1050 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
1058 				atomic_inc(&__b->buffer->record_disabled); \
1069 static inline u64 rb_time_stamp(struct trace_buffer *buffer)  in rb_time_stamp()  argument
1074 	if (IS_ENABLED(CONFIG_RETPOLINE) && likely(buffer->clock == trace_clock_local))  in rb_time_stamp()
1077 		ts = buffer->clock();  in rb_time_stamp()
1083 u64 ring_buffer_time_stamp(struct trace_buffer *buffer)  in ring_buffer_time_stamp()  argument
1088 	time = rb_time_stamp(buffer);  in ring_buffer_time_stamp()
1095 void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer,  in ring_buffer_normalize_time_stamp()  argument
1104  * Making the ring buffer lockless makes things tricky.
1109  * The reader page is always off the ring buffer, but when the
1111  * a new one from the buffer. The reader needs to take from
1161  *  the reader page with a page in the buffer, but before it
1477  * rb_check_pages - integrity check of buffer pages
1478  * @cpu_buffer: CPU buffer with pages to test
1603 	 * The ring buffer page list is a circular list that does not  in rb_allocate_pages()
1618 rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)  in rb_allocate_cpu_buffer()  argument
1631 	cpu_buffer->buffer = buffer;  in rb_allocate_cpu_buffer()
1633 	lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);  in rb_allocate_cpu_buffer()
1702  * @flags: attributes to set for the ring buffer.
1703  * @key: ring buffer reader_lock_key.
1706  * flag. This flag means that the buffer will overwrite old data
1707  * when the buffer wraps. If this flag is not set, the buffer will
1713 	struct trace_buffer *buffer;  in __ring_buffer_alloc()  local
1720 	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),  in __ring_buffer_alloc()
1722 	if (!buffer)  in __ring_buffer_alloc()
1725 	if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))  in __ring_buffer_alloc()
1729 	buffer->flags = flags;  in __ring_buffer_alloc()
1730 	buffer->clock = trace_clock_local;  in __ring_buffer_alloc()
1731 	buffer->reader_lock_key = key;  in __ring_buffer_alloc()
1733 	init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);  in __ring_buffer_alloc()
1734 	init_waitqueue_head(&buffer->irq_work.waiters);  in __ring_buffer_alloc()
1740 	buffer->cpus = nr_cpu_ids;  in __ring_buffer_alloc()
1743 	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),  in __ring_buffer_alloc()
1745 	if (!buffer->buffers)  in __ring_buffer_alloc()
1749 	cpumask_set_cpu(cpu, buffer->cpumask);  in __ring_buffer_alloc()
1750 	buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in __ring_buffer_alloc()
1751 	if (!buffer->buffers[cpu])  in __ring_buffer_alloc()
1754 	ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);  in __ring_buffer_alloc()
1758 	mutex_init(&buffer->mutex);  in __ring_buffer_alloc()
1760 	return buffer;  in __ring_buffer_alloc()
1763 	for_each_buffer_cpu(buffer, cpu) {  in __ring_buffer_alloc()
1764 		if (buffer->buffers[cpu])  in __ring_buffer_alloc()
1765 			rb_free_cpu_buffer(buffer->buffers[cpu]);  in __ring_buffer_alloc()
1767 	kfree(buffer->buffers);  in __ring_buffer_alloc()
1770 	free_cpumask_var(buffer->cpumask);  in __ring_buffer_alloc()
1773 	kfree(buffer);  in __ring_buffer_alloc()
1779  * ring_buffer_free - free a ring buffer.
1780  * @buffer: the buffer to free.
1783 ring_buffer_free(struct trace_buffer *buffer)  in ring_buffer_free()  argument
1787 	cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);  in ring_buffer_free()
1789 	for_each_buffer_cpu(buffer, cpu)  in ring_buffer_free()
1790 		rb_free_cpu_buffer(buffer->buffers[cpu]);  in ring_buffer_free()
1792 	kfree(buffer->buffers);  in ring_buffer_free()
1793 	free_cpumask_var(buffer->cpumask);  in ring_buffer_free()
1795 	kfree(buffer);  in ring_buffer_free()
1799 void ring_buffer_set_clock(struct trace_buffer *buffer,  in ring_buffer_set_clock()  argument
1802 	buffer->clock = clock;  in ring_buffer_set_clock()
1805 void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs)  in ring_buffer_set_time_stamp_abs()  argument
1807 	buffer->time_stamp_abs = abs;  in ring_buffer_set_time_stamp_abs()
1810 bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer)  in ring_buffer_time_stamp_abs()  argument
1812 	return buffer->time_stamp_abs;  in ring_buffer_time_stamp_abs()
1854 	 * from the ring buffer  in rb_remove_pages()
1881 	/* make sure pages points to a valid page in the ring buffer */  in rb_remove_pages()
1901 	/* last buffer page to remove */  in rb_remove_pages()
1918 			 * bytes consumed in ring buffer from here.  in rb_remove_pages()
1948 	 * in the ring buffer. Now we are racing with the writer trying to  in rb_insert_pages()
2039  * ring_buffer_resize - resize the ring buffer
2040  * @buffer: the buffer to resize.
2042  * @cpu_id: the cpu buffer to resize
2048 int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size,  in ring_buffer_resize()  argument
2056 	 * Always succeed at resizing a non-existent buffer:  in ring_buffer_resize()
2058 	if (!buffer)  in ring_buffer_resize()
2061 	/* Make sure the requested buffer exists */  in ring_buffer_resize()
2063 	    !cpumask_test_cpu(cpu_id, buffer->cpumask))  in ring_buffer_resize()
2072 	/* prevent another thread from changing buffer sizes */  in ring_buffer_resize()
2073 	mutex_lock(&buffer->mutex);  in ring_buffer_resize()
2079 		 * manipulating the ring buffer and is expecting a sane state while  in ring_buffer_resize()
2082 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2083 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2091 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2092 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2118 		 * since we can change their buffer sizes without any race.  in ring_buffer_resize()
2120 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2121 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2136 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2137 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2148 		cpu_buffer = buffer->buffers[cpu_id];  in ring_buffer_resize()
2155 		 * manipulating the ring buffer and is expecting a sane state while  in ring_buffer_resize()
2191 	 * The ring buffer resize can happen with the ring buffer  in ring_buffer_resize()
2193 	 * as possible. But if the buffer is disabled, we do not need  in ring_buffer_resize()
2195 	 * that the buffer is not corrupt.  in ring_buffer_resize()
2197 	if (atomic_read(&buffer->record_disabled)) {  in ring_buffer_resize()
2198 		atomic_inc(&buffer->record_disabled);  in ring_buffer_resize()
2200 		 * Even though the buffer was disabled, we must make sure  in ring_buffer_resize()
2206 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2207 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2210 		atomic_dec(&buffer->record_disabled);  in ring_buffer_resize()
2213 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
2217 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2220 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2233 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
2238 void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val)  in ring_buffer_change_overwrite()  argument
2240 	mutex_lock(&buffer->mutex);  in ring_buffer_change_overwrite()
2242 		buffer->flags |= RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
2244 		buffer->flags &= ~RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
2245 	mutex_unlock(&buffer->mutex);  in ring_buffer_change_overwrite()
2586 	/* Set write to end of buffer */  in rb_reset_tail()
2602 	struct trace_buffer *buffer = cpu_buffer->buffer;  in rb_move_tail()  local
2612 	 * it all the way around the buffer, bail, and warn  in rb_move_tail()
2625 	 * page with the buffer head.  in rb_move_tail()
2631 	 * the buffer, unless the commit page is still on the  in rb_move_tail()
2645 			if (!(buffer->flags & RB_FL_OVERWRITE)) {  in rb_move_tail()
2661 			 * page. We could have a small buffer, and  in rb_move_tail()
2662 			 * have filled up the buffer with events  in rb_move_tail()
2768 				pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",  in rb_add_timestamp()
2783  * @cpu_buffer: The per cpu buffer of the @event
2788  * is the actual size that is written to the ring buffer,
3056 rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)  in rb_wakeups()  argument
3062 	if (buffer->irq_work.waiters_pending) {  in rb_wakeups()
3063 		buffer->irq_work.waiters_pending = false;  in rb_wakeups()
3065 		irq_work_queue(&buffer->irq_work.work);  in rb_wakeups()
3087 	dirty = ring_buffer_nr_dirty_pages(buffer, cpu_buffer->cpu);  in rb_wakeups()
3153  * if an interrupt comes in while NORMAL bit is set and the ring buffer
3210  * @buffer: The ring buffer to modify
3212  * The ring buffer has a safety mechanism to prevent recursion.
3221 void ring_buffer_nest_start(struct trace_buffer *buffer)  in ring_buffer_nest_start()  argument
3229 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_nest_start()
3236  * @buffer: The ring buffer to modify
3241 void ring_buffer_nest_end(struct trace_buffer *buffer)  in ring_buffer_nest_end()  argument
3248 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_nest_end()
3256  * @buffer: The buffer to commit to
3259  * This commits the data to the ring buffer, and releases any locks held.
3263 int ring_buffer_unlock_commit(struct trace_buffer *buffer,  in ring_buffer_unlock_commit()  argument
3269 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_unlock_commit()
3273 	rb_wakeups(buffer, cpu_buffer);  in ring_buffer_unlock_commit()
3337  * the buffer page.
3452 	info->ts = rb_time_stamp(cpu_buffer->buffer);  in __rb_reserve_next()
3483 	/* See if we shot pass the end of this buffer page */  in __rb_reserve_next()
3536 		ts = rb_time_stamp(cpu_buffer->buffer);  in __rb_reserve_next()
3567 	/* We reserved something on the buffer */  in __rb_reserve_next()
3588 rb_reserve_next_event(struct trace_buffer *buffer,  in rb_reserve_next_event()  argument
3602 	 * Due to the ability to swap a cpu buffer from a buffer  in rb_reserve_next_event()
3608 	if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) {  in rb_reserve_next_event()
3617 	if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {  in rb_reserve_next_event()
3656  * ring_buffer_lock_reserve - reserve a part of the buffer
3657  * @buffer: the ring buffer to reserve from
3660  * Returns a reserved event on the ring buffer to copy directly to.
3671 ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length)  in ring_buffer_lock_reserve()  argument
3680 	if (unlikely(atomic_read(&buffer->record_disabled)))  in ring_buffer_lock_reserve()
3685 	if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask)))  in ring_buffer_lock_reserve()
3688 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_lock_reserve()
3699 	event = rb_reserve_next_event(buffer, cpu_buffer, length);  in ring_buffer_lock_reserve()
3749 	/* commit not part of this buffer?? */  in rb_decrement_entry()
3755  * @buffer: the ring buffer
3758  * Sometimes an event that is in the ring buffer needs to be ignored.
3759  * This function lets the user discard an event in the ring buffer
3763  * committed. It will try to free the event from the ring buffer
3772 void ring_buffer_discard_commit(struct trace_buffer *buffer,  in ring_buffer_discard_commit()  argument
3782 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_discard_commit()
3789 	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));  in ring_buffer_discard_commit()
3806  * ring_buffer_write - write data to the buffer without reserving
3807  * @buffer: The ring buffer to write to.
3809  * @data: The data to write to the buffer.
3812  * one function. If you already have the data to write to the buffer, it
3818 int ring_buffer_write(struct trace_buffer *buffer,  in ring_buffer_write()  argument
3830 	if (atomic_read(&buffer->record_disabled))  in ring_buffer_write()
3835 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_write()
3838 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_write()
3849 	event = rb_reserve_next_event(buffer, cpu_buffer, length);  in ring_buffer_write()
3859 	rb_wakeups(buffer, cpu_buffer);  in ring_buffer_write()
3889 	 * committed content has been read, the ring buffer is empty.  in rb_per_cpu_empty()
3910  * ring_buffer_record_disable - stop all writes into the buffer
3911  * @buffer: The ring buffer to stop writes to.
3913  * This prevents all writes to the buffer. Any attempt to write
3914  * to the buffer after this will fail and return NULL.
3918 void ring_buffer_record_disable(struct trace_buffer *buffer)  in ring_buffer_record_disable()  argument
3920 	atomic_inc(&buffer->record_disabled);  in ring_buffer_record_disable()
3925  * ring_buffer_record_enable - enable writes to the buffer
3926  * @buffer: The ring buffer to enable writes
3931 void ring_buffer_record_enable(struct trace_buffer *buffer)  in ring_buffer_record_enable()  argument
3933 	atomic_dec(&buffer->record_disabled);  in ring_buffer_record_enable()
3938  * ring_buffer_record_off - stop all writes into the buffer
3939  * @buffer: The ring buffer to stop writes to.
3941  * This prevents all writes to the buffer. Any attempt to write
3942  * to the buffer after this will fail and return NULL.
3948 void ring_buffer_record_off(struct trace_buffer *buffer)  in ring_buffer_record_off()  argument
3954 		rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_off()
3956 	} while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);  in ring_buffer_record_off()
3961  * ring_buffer_record_on - restart writes into the buffer
3962  * @buffer: The ring buffer to start writes to.
3964  * This enables all writes to the buffer that was disabled by
3971 void ring_buffer_record_on(struct trace_buffer *buffer)  in ring_buffer_record_on()  argument
3977 		rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_on()
3979 	} while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);  in ring_buffer_record_on()
3984  * ring_buffer_record_is_on - return true if the ring buffer can write
3985  * @buffer: The ring buffer to see if write is enabled
3987  * Returns true if the ring buffer is in a state that it accepts writes.
3989 bool ring_buffer_record_is_on(struct trace_buffer *buffer)  in ring_buffer_record_is_on()  argument
3991 	return !atomic_read(&buffer->record_disabled);  in ring_buffer_record_is_on()
3995  * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
3996  * @buffer: The ring buffer to see if write is set enabled
3998  * Returns true if the ring buffer is set writable by ring_buffer_record_on().
4001  * It may return true when the ring buffer has been disabled by
4003  * the ring buffer.
4005 bool ring_buffer_record_is_set_on(struct trace_buffer *buffer)  in ring_buffer_record_is_set_on()  argument
4007 	return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF);  in ring_buffer_record_is_set_on()
4012  * @buffer: The ring buffer to stop writes to.
4013  * @cpu: The CPU buffer to stop
4015  * This prevents all writes to the buffer. Any attempt to write
4016  * to the buffer after this will fail and return NULL.
4020 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_record_disable_cpu()  argument
4024 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_disable_cpu()
4027 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_disable_cpu()
4033  * ring_buffer_record_enable_cpu - enable writes to the buffer
4034  * @buffer: The ring buffer to enable writes
4040 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_record_enable_cpu()  argument
4044 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_enable_cpu()
4047 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_enable_cpu()
4053  * The total entries in the ring buffer is the running counter
4054  * of entries entered into the ring buffer, minus the sum of
4055  * the entries read from the ring buffer and the number of
4066  * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
4067  * @buffer: The ring buffer
4068  * @cpu: The per CPU buffer to read from.
4070 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu)  in ring_buffer_oldest_event_ts()  argument
4077 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_oldest_event_ts()
4080 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_oldest_event_ts()
4099  * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer
4100  * @buffer: The ring buffer
4101  * @cpu: The per CPU buffer to read from.
4103 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_bytes_cpu()  argument
4108 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_bytes_cpu()
4111 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_bytes_cpu()
4119  * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
4120  * @buffer: The ring buffer
4121  * @cpu: The per CPU buffer to get the entries from.
4123 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_entries_cpu()  argument
4127 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_entries_cpu()
4130 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries_cpu()
4138  * buffer wrapping around (only if RB_FL_OVERWRITE is on).
4139  * @buffer: The ring buffer
4140  * @cpu: The per CPU buffer to get the number of overruns from
4142 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_overrun_cpu()  argument
4147 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_overrun_cpu()
4150 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overrun_cpu()
4159  * commits failing due to the buffer wrapping around while there are uncommitted
4161  * @buffer: The ring buffer
4162  * @cpu: The per CPU buffer to get the number of overruns from
4165 ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_commit_overrun_cpu()  argument
4170 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_commit_overrun_cpu()
4173 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_commit_overrun_cpu()
4182  * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
4183  * @buffer: The ring buffer
4184  * @cpu: The per CPU buffer to get the number of overruns from
4187 ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_dropped_events_cpu()  argument
4192 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_dropped_events_cpu()
4195 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_dropped_events_cpu()
4204  * @buffer: The ring buffer
4205  * @cpu: The per CPU buffer to get the number of events read
4208 ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_read_events_cpu()  argument
4212 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_events_cpu()
4215 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_events_cpu()
4221  * ring_buffer_entries - get the number of entries in a buffer
4222  * @buffer: The ring buffer
4224  * Returns the total number of entries in the ring buffer
4227 unsigned long ring_buffer_entries(struct trace_buffer *buffer)  in ring_buffer_entries()  argument
4233 	/* if you care about this being correct, lock the buffer */  in ring_buffer_entries()
4234 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_entries()
4235 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries()
4244  * ring_buffer_overruns - get the number of overruns in buffer
4245  * @buffer: The ring buffer
4247  * Returns the total number of overruns in the ring buffer
4250 unsigned long ring_buffer_overruns(struct trace_buffer *buffer)  in ring_buffer_overruns()  argument
4256 	/* if you care about this being correct, lock the buffer */  in ring_buffer_overruns()
4257 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_overruns()
4258 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overruns()
4458 	/* Don't bother swapping if the ring buffer is empty */  in rb_get_reader_page()
4481 	 * cpu_buffer->pages just needs to point to the buffer, it  in rb_get_reader_page()
4482 	 *  has no specific buffer page to point to. Lets move it out  in rb_get_reader_page()
4561 	/* This function should not be called when buffer is empty */  in rb_advance_reader()
4592 	 * Check if we are at the end of the buffer.  in rb_advance_iter()
4658 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
4668 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
4686 	struct trace_buffer *buffer;  in rb_iter_peek()  local
4695 	buffer = cpu_buffer->buffer;  in rb_iter_peek()
4699 	 * the buffer. A consuming read invalidates the iterator  in rb_iter_peek()
4714 	 * the ring buffer with an active write as the consumer is.  in rb_iter_peek()
4749 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_iter_peek()
4759 			ring_buffer_normalize_time_stamp(buffer,  in rb_iter_peek()
4780 	 * If an NMI die dumps out the content of the ring buffer  in rb_reader_lock()
4782 	 * preempted a task that holds the ring buffer locks. If  in rb_reader_lock()
4784 	 * to do the read, but this can corrupt the ring buffer,  in rb_reader_lock()
4791 	/* Continue without locking, but disable the ring buffer */  in rb_reader_lock()
4806  * @buffer: The ring buffer to read
4815 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts,  in ring_buffer_peek()  argument
4818 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_peek()
4823 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_peek()
4842  * @iter: The ring buffer iterator
4857  * @iter: The ring buffer iterator
4883  * @buffer: The ring buffer to get the next event from
4884  * @cpu: the cpu to read the buffer from
4888  * Returns the next event in the ring buffer, and that event is consumed.
4890  * and eventually empty the ring buffer if the producer is slower.
4893 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts,  in ring_buffer_consume()  argument
4905 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_consume()
4908 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_consume()
4932  * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
4933  * @buffer: The ring buffer to read from
4934  * @cpu: The cpu buffer to iterate over
4938  * through the buffer.  Memory is allocated, buffer recording
4941  * Disabling buffer recording prevents the reading from being
4953 ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags)  in ring_buffer_read_prepare()  argument
4958 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_prepare()
4971 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_prepare()
4996  * ring_buffer_read_start - start a non consuming read of the buffer
4999  * This finalizes the startup of an iteration through the buffer.
5026  * ring_buffer_read_finish - finish reading the iterator of the buffer
5029  * This re-enables the recording to the buffer, and frees the
5039 	 * Ring buffer is disabled from recording, here's a good place  in ring_buffer_read_finish()
5040 	 * to check the integrity of the ring buffer.  in ring_buffer_read_finish()
5056  * @iter: The ring buffer iterator
5075  * ring_buffer_size - return the size of the ring buffer (in bytes)
5076  * @buffer: The ring buffer.
5077  * @cpu: The CPU to get ring buffer size from.
5079 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)  in ring_buffer_size()  argument
5083 	 *	BUF_PAGE_SIZE * buffer->nr_pages  in ring_buffer_size()
5085 	 * return the per cpu buffer value.  in ring_buffer_size()
5087 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_size()
5090 	return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;  in ring_buffer_size()
5142 /* Must have disabled the cpu buffer then done a synchronize_rcu */
5163  * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
5164  * @buffer: The ring buffer to reset a per cpu buffer of
5165  * @cpu: The CPU buffer to be reset
5167 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_reset_cpu()  argument
5169 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_cpu()
5171 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_reset_cpu()
5174 	/* prevent another thread from changing buffer sizes */  in ring_buffer_reset_cpu()
5175 	mutex_lock(&buffer->mutex);  in ring_buffer_reset_cpu()
5188 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset_cpu()
5193  * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
5194  * @buffer: The ring buffer to reset a per cpu buffer of
5195  * @cpu: The CPU buffer to be reset
5197 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer)  in ring_buffer_reset_online_cpus()  argument
5202 	/* prevent another thread from changing buffer sizes */  in ring_buffer_reset_online_cpus()
5203 	mutex_lock(&buffer->mutex);  in ring_buffer_reset_online_cpus()
5205 	for_each_online_buffer_cpu(buffer, cpu) {  in ring_buffer_reset_online_cpus()
5206 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_online_cpus()
5215 	for_each_online_buffer_cpu(buffer, cpu) {  in ring_buffer_reset_online_cpus()
5216 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_online_cpus()
5224 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset_online_cpus()
5228  * ring_buffer_reset - reset a ring buffer
5229  * @buffer: The ring buffer to reset all cpu buffers
5231 void ring_buffer_reset(struct trace_buffer *buffer)  in ring_buffer_reset()  argument
5236 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_reset()
5237 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset()
5246 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_reset()
5247 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset()
5258  * rind_buffer_empty - is the ring buffer empty?
5259  * @buffer: The ring buffer to test
5261 bool ring_buffer_empty(struct trace_buffer *buffer)  in ring_buffer_empty()  argument
5269 	/* yes this is racy, but if you don't like the race, lock the buffer */  in ring_buffer_empty()
5270 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_empty()
5271 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty()
5287  * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
5288  * @buffer: The ring buffer
5289  * @cpu: The CPU buffer to test
5291 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_empty_cpu()  argument
5298 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_empty_cpu()
5301 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty_cpu()
5314  * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
5315  * @buffer_a: One buffer to swap with
5316  * @buffer_b: The other buffer to swap with
5320  * of a CPU buffer and has another back up buffer lying around.
5321  * it is expected that the tracer handles the cpu buffer not being
5374 	cpu_buffer_b->buffer = buffer_a;  in ring_buffer_swap_cpu()
5375 	cpu_buffer_a->buffer = buffer_b;  in ring_buffer_swap_cpu()
5389  * ring_buffer_alloc_read_page - allocate a page to read from buffer
5390  * @buffer: the buffer to allocate for.
5391  * @cpu: the cpu buffer to allocate.
5394  * When reading a full page from the ring buffer, these functions
5397  * needs to get pages from the ring buffer, it passes the result
5399  * the page that was allocated, with the read page of the buffer.
5404 void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)  in ring_buffer_alloc_read_page()  argument
5411 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_alloc_read_page()
5414 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_alloc_read_page()
5445  * @buffer: the buffer the page was allocate for
5446  * @cpu: the cpu buffer the page came from
5451 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)  in ring_buffer_free_read_page()  argument
5453 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_free_read_page()
5479  * ring_buffer_read_page - extract a page from the ring buffer
5480  * @buffer: buffer to extract from
5483  * @cpu: the cpu of the buffer to extract
5486  * This function will pull out a page from the ring buffer and consume it.
5489  * to swap with a page in the ring buffer.
5492  *	rpage = ring_buffer_alloc_read_page(buffer, cpu);
5495  *	ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
5503  *  The ring buffer can be used anywhere in the kernel and can not
5504  *  blindly call wake_up. The layer that uses the ring buffer must be
5511 int ring_buffer_read_page(struct trace_buffer *buffer,  in ring_buffer_read_page()  argument
5514 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_page()
5525 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_page()
5562 	 * we must copy the data from the page to the buffer.  in ring_buffer_read_page()
5677  * If we were to free the buffer, then the user would lose any trace that was in
5678  * the buffer.
5682 	struct trace_buffer *buffer;  in trace_rb_cpu_prepare()  local
5687 	buffer = container_of(node, struct trace_buffer, node);  in trace_rb_cpu_prepare()
5688 	if (cpumask_test_cpu(cpu, buffer->cpumask))  in trace_rb_cpu_prepare()
5694 	for_each_buffer_cpu(buffer, cpu_i) {  in trace_rb_cpu_prepare()
5697 			nr_pages = buffer->buffers[cpu_i]->nr_pages;  in trace_rb_cpu_prepare()
5698 		if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {  in trace_rb_cpu_prepare()
5706 	buffer->buffers[cpu] =  in trace_rb_cpu_prepare()
5707 		rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in trace_rb_cpu_prepare()
5708 	if (!buffer->buffers[cpu]) {  in trace_rb_cpu_prepare()
5709 		WARN(1, "failed to allocate ring buffer on CPU %u\n",  in trace_rb_cpu_prepare()
5714 	cpumask_set_cpu(cpu, buffer->cpumask);  in trace_rb_cpu_prepare()
5720  * This is a basic integrity check of the ring buffer.
5723  * writing to the per cpu ring buffer various sizes of data.
5727  * IPIs to the other CPUs to also write into the ring buffer.
5728  * this is to test the nesting ability of the buffer.
5731  * ring buffer should happen that's not expected, a big warning
5737 	struct trace_buffer *buffer;  member
5790 	/* read rb_test_started before checking buffer enabled */  in rb_write_something()
5793 	event = ring_buffer_lock_reserve(data->buffer, len);  in rb_write_something()
5807 	if (RB_WARN_ON(data->buffer, event_len < len))  in rb_write_something()
5833 	ring_buffer_unlock_commit(data->buffer, event);  in rb_write_something()
5879 	struct trace_buffer *buffer;  in test_ringbuffer()  local
5884 		pr_warn("Lockdown is enabled, skipping ring buffer tests\n");  in test_ringbuffer()
5888 	pr_info("Running ring buffer tests...\n");  in test_ringbuffer()
5890 	buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);  in test_ringbuffer()
5891 	if (WARN_ON(!buffer))  in test_ringbuffer()
5894 	/* Disable buffer so that threads can't write to it yet */  in test_ringbuffer()
5895 	ring_buffer_record_off(buffer);  in test_ringbuffer()
5898 		rb_data[cpu].buffer = buffer;  in test_ringbuffer()
5921 	ring_buffer_record_on(buffer);  in test_ringbuffer()
5923 	 * Show buffer is enabled before setting rb_test_started.  in test_ringbuffer()
5926 	 * buffer gets enabled, there will always be some kind of  in test_ringbuffer()
5929 	 * the threads see that the buffer is active.  in test_ringbuffer()
5947 		ring_buffer_free(buffer);  in test_ringbuffer()
5987 		if (RB_WARN_ON(buffer, total_dropped))  in test_ringbuffer()
5992 		while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) {  in test_ringbuffer()
5999 				pr_info("buffer had: %.*s\n", item->size, item->str);  in test_ringbuffer()
6001 				RB_WARN_ON(buffer, 1);  in test_ringbuffer()
6021 			if (RB_WARN_ON(buffer, total_len != total_alloc ||  in test_ringbuffer()
6025 		if (RB_WARN_ON(buffer, total_lost + total_read != total_events))  in test_ringbuffer()
6031 		pr_info("Ring buffer PASSED!\n");  in test_ringbuffer()
6033 	ring_buffer_free(buffer);  in test_ringbuffer()