Lines Matching full:buffer
3 * Generic ring buffer
34 * The ring buffer header is special. We must manually up keep it.
56 * The ring buffer is made up of a list of pages. A separate list of pages is
57 * allocated for each CPU. A writer may only write to a buffer that is
59 * from any per cpu buffer.
61 * The reader is special. For each per cpu buffer, the reader has its own
63 * page is swapped with another page in the ring buffer.
67 * again (as long as it is out of the ring buffer).
72 * |reader| RING BUFFER
83 * |reader| RING BUFFER
94 * |reader| RING BUFFER
105 * |buffer| RING BUFFER
117 * and swap that into the ring buffer.
246 /* inline for ring buffer fast paths */
270 #define for_each_buffer_cpu(buffer, cpu) \ argument
271 for_each_cpu(cpu, buffer->cpumask)
273 #define for_each_online_buffer_cpu(buffer, cpu) \ argument
274 for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
287 * contained within a ring buffer event. This function decodes
309 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
313 * Note, the buffer_page list must be first. The buffer pages
314 * are allocated in cache lines, which means that each buffer
317 * add flags in the list struct pointers, to make the ring buffer
321 struct list_head list; /* list of buffer pages */
330 * The buffer page counters, write and entries, must be reset
430 * ABSOLUTE - the buffer requests all events to have absolute time stamps
481 * head_page == tail_page && head == tail then buffer is empty.
487 struct trace_buffer *buffer; member
518 /* ring buffer pages to update, > 0 to add, < 0 to remove */
563 * buffer doesn't need all the features of a true 64 bit atomic,
566 * For the ring buffer, 64 bit required operations for the time is
743 * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
744 * @buffer: The ring_buffer to get the number of pages from
747 * Returns the number of pages used by a per_cpu buffer of the ring buffer.
749 size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu) in ring_buffer_nr_pages() argument
751 return buffer->buffers[cpu]->nr_pages; in ring_buffer_nr_pages()
755 * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
756 * @buffer: The ring_buffer to get the number of pages from
759 * Returns the number of pages that have content in the ring buffer.
761 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu) in ring_buffer_nr_dirty_pages() argument
766 read = local_read(&buffer->buffers[cpu]->pages_read); in ring_buffer_nr_dirty_pages()
767 cnt = local_read(&buffer->buffers[cpu]->pages_touched); in ring_buffer_nr_dirty_pages()
778 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
781 * ring buffer waiters queue.
795 * ring_buffer_wait - wait for input to the ring buffer
796 * @buffer: buffer to wait on
797 * @cpu: the cpu buffer to wait on
801 * as data is added to any of the @buffer's cpu buffers. Otherwise
802 * it will wait for data to be added to a specific cpu buffer.
804 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) in ring_buffer_wait() argument
813 * data in any cpu buffer, or a specific buffer, put the in ring_buffer_wait()
817 work = &buffer->irq_work; in ring_buffer_wait()
821 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_wait()
823 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wait()
841 * We don't clear it even if the buffer is no longer in ring_buffer_wait()
864 if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) in ring_buffer_wait()
868 !ring_buffer_empty_cpu(buffer, cpu)) { in ring_buffer_wait()
880 dirty = ring_buffer_nr_dirty_pages(buffer, cpu); in ring_buffer_wait()
902 * ring_buffer_poll_wait - poll on buffer input
903 * @buffer: buffer to wait on
904 * @cpu: the cpu buffer to wait on
909 * as data is added to any of the @buffer's cpu buffers. Otherwise
910 * it will wait for data to be added to a specific cpu buffer.
915 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, in ring_buffer_poll_wait() argument
922 work = &buffer->irq_work; in ring_buffer_poll_wait()
924 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_poll_wait()
927 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_poll_wait()
935 * checking if the ring buffer is empty. Once the waiters_pending bit in ring_buffer_poll_wait()
942 * the buffer goes from empty to having content. But as this race is in ring_buffer_poll_wait()
948 if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) || in ring_buffer_poll_wait()
949 (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu))) in ring_buffer_poll_wait()
954 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
962 atomic_inc(&__b->buffer->record_disabled); \
973 static inline u64 rb_time_stamp(struct trace_buffer *buffer) in rb_time_stamp() argument
978 if (IS_ENABLED(CONFIG_RETPOLINE) && likely(buffer->clock == trace_clock_local)) in rb_time_stamp()
981 ts = buffer->clock(); in rb_time_stamp()
987 u64 ring_buffer_time_stamp(struct trace_buffer *buffer, int cpu) in ring_buffer_time_stamp() argument
992 time = rb_time_stamp(buffer); in ring_buffer_time_stamp()
999 void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer, in ring_buffer_normalize_time_stamp() argument
1008 * Making the ring buffer lockless makes things tricky.
1013 * The reader page is always off the ring buffer, but when the
1015 * a new one from the buffer. The reader needs to take from
1065 * the reader page with a page in the buffer, but before it
1384 * rb_check_pages - integrity check of buffer pages
1385 * @cpu_buffer: CPU buffer with pages to test
1507 * The ring buffer page list is a circular list that does not in rb_allocate_pages()
1522 rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) in rb_allocate_cpu_buffer() argument
1535 cpu_buffer->buffer = buffer; in rb_allocate_cpu_buffer()
1537 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); in rb_allocate_cpu_buffer()
1606 * @flags: attributes to set for the ring buffer.
1607 * @key: ring buffer reader_lock_key.
1610 * flag. This flag means that the buffer will overwrite old data
1611 * when the buffer wraps. If this flag is not set, the buffer will
1617 struct trace_buffer *buffer; in __ring_buffer_alloc() local
1624 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), in __ring_buffer_alloc()
1626 if (!buffer) in __ring_buffer_alloc()
1629 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) in __ring_buffer_alloc()
1633 buffer->flags = flags; in __ring_buffer_alloc()
1634 buffer->clock = trace_clock_local; in __ring_buffer_alloc()
1635 buffer->reader_lock_key = key; in __ring_buffer_alloc()
1637 init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); in __ring_buffer_alloc()
1638 init_waitqueue_head(&buffer->irq_work.waiters); in __ring_buffer_alloc()
1644 buffer->cpus = nr_cpu_ids; in __ring_buffer_alloc()
1647 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), in __ring_buffer_alloc()
1649 if (!buffer->buffers) in __ring_buffer_alloc()
1653 cpumask_set_cpu(cpu, buffer->cpumask); in __ring_buffer_alloc()
1654 buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in __ring_buffer_alloc()
1655 if (!buffer->buffers[cpu]) in __ring_buffer_alloc()
1658 ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in __ring_buffer_alloc()
1662 mutex_init(&buffer->mutex); in __ring_buffer_alloc()
1664 return buffer; in __ring_buffer_alloc()
1667 for_each_buffer_cpu(buffer, cpu) { in __ring_buffer_alloc()
1668 if (buffer->buffers[cpu]) in __ring_buffer_alloc()
1669 rb_free_cpu_buffer(buffer->buffers[cpu]); in __ring_buffer_alloc()
1671 kfree(buffer->buffers); in __ring_buffer_alloc()
1674 free_cpumask_var(buffer->cpumask); in __ring_buffer_alloc()
1677 kfree(buffer); in __ring_buffer_alloc()
1683 * ring_buffer_free - free a ring buffer.
1684 * @buffer: the buffer to free.
1687 ring_buffer_free(struct trace_buffer *buffer) in ring_buffer_free() argument
1691 cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in ring_buffer_free()
1693 for_each_buffer_cpu(buffer, cpu) in ring_buffer_free()
1694 rb_free_cpu_buffer(buffer->buffers[cpu]); in ring_buffer_free()
1696 kfree(buffer->buffers); in ring_buffer_free()
1697 free_cpumask_var(buffer->cpumask); in ring_buffer_free()
1699 kfree(buffer); in ring_buffer_free()
1703 void ring_buffer_set_clock(struct trace_buffer *buffer, in ring_buffer_set_clock() argument
1706 buffer->clock = clock; in ring_buffer_set_clock()
1709 void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs) in ring_buffer_set_time_stamp_abs() argument
1711 buffer->time_stamp_abs = abs; in ring_buffer_set_time_stamp_abs()
1714 bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer) in ring_buffer_time_stamp_abs() argument
1716 return buffer->time_stamp_abs; in ring_buffer_time_stamp_abs()
1758 * from the ring buffer in rb_remove_pages()
1785 /* make sure pages points to a valid page in the ring buffer */ in rb_remove_pages()
1805 /* last buffer page to remove */ in rb_remove_pages()
1822 * bytes consumed in ring buffer from here. in rb_remove_pages()
1852 * in the ring buffer. Now we are racing with the writer trying to in rb_insert_pages()
1943 * ring_buffer_resize - resize the ring buffer
1944 * @buffer: the buffer to resize.
1946 * @cpu_id: the cpu buffer to resize
1952 int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, in ring_buffer_resize() argument
1960 * Always succeed at resizing a non-existent buffer: in ring_buffer_resize()
1962 if (!buffer) in ring_buffer_resize()
1965 /* Make sure the requested buffer exists */ in ring_buffer_resize()
1967 !cpumask_test_cpu(cpu_id, buffer->cpumask)) in ring_buffer_resize()
1978 /* prevent another thread from changing buffer sizes */ in ring_buffer_resize()
1979 mutex_lock(&buffer->mutex); in ring_buffer_resize()
1985 * manipulating the ring buffer and is expecting a sane state while in ring_buffer_resize()
1988 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
1989 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
1997 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
1998 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2024 * since we can change their buffer sizes without any race. in ring_buffer_resize()
2026 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2027 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2042 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2043 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2055 if (!cpumask_test_cpu(cpu_id, buffer->cpumask)) in ring_buffer_resize()
2058 cpu_buffer = buffer->buffers[cpu_id]; in ring_buffer_resize()
2065 * manipulating the ring buffer and is expecting a sane state while in ring_buffer_resize()
2101 * The ring buffer resize can happen with the ring buffer in ring_buffer_resize()
2103 * as possible. But if the buffer is disabled, we do not need in ring_buffer_resize()
2105 * that the buffer is not corrupt. in ring_buffer_resize()
2107 if (atomic_read(&buffer->record_disabled)) { in ring_buffer_resize()
2108 atomic_inc(&buffer->record_disabled); in ring_buffer_resize()
2110 * Even though the buffer was disabled, we must make sure in ring_buffer_resize()
2116 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2117 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2120 atomic_dec(&buffer->record_disabled); in ring_buffer_resize()
2123 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
2127 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2130 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2143 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
2148 void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val) in ring_buffer_change_overwrite() argument
2150 mutex_lock(&buffer->mutex); in ring_buffer_change_overwrite()
2152 buffer->flags |= RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
2154 buffer->flags &= ~RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
2155 mutex_unlock(&buffer->mutex); in ring_buffer_change_overwrite()
2496 /* Set write to end of buffer */ in rb_reset_tail()
2512 struct trace_buffer *buffer = cpu_buffer->buffer; in rb_move_tail() local
2522 * it all the way around the buffer, bail, and warn in rb_move_tail()
2535 * page with the buffer head. in rb_move_tail()
2541 * the buffer, unless the commit page is still on the in rb_move_tail()
2555 if (!(buffer->flags & RB_FL_OVERWRITE)) { in rb_move_tail()
2571 * page. We could have a small buffer, and in rb_move_tail()
2572 * have filled up the buffer with events in rb_move_tail()
2681 pr_warn("Ring buffer clock went backwards: %llu -> %llu\n", in rb_add_timestamp()
2696 * @cpu_buffer: The per cpu buffer of the @event
2701 * is the actual size that is written to the ring buffer,
2968 rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) in rb_wakeups() argument
2974 if (buffer->irq_work.waiters_pending) { in rb_wakeups()
2975 buffer->irq_work.waiters_pending = false; in rb_wakeups()
2977 irq_work_queue(&buffer->irq_work.work); in rb_wakeups()
2999 dirty = ring_buffer_nr_dirty_pages(buffer, cpu_buffer->cpu); in rb_wakeups()
3058 * if an interrupt comes in while NORMAL bit is set and the ring buffer
3113 * @buffer: The ring buffer to modify
3115 * The ring buffer has a safety mechanism to prevent recursion.
3124 void ring_buffer_nest_start(struct trace_buffer *buffer) in ring_buffer_nest_start() argument
3132 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_start()
3139 * @buffer: The ring buffer to modify
3144 void ring_buffer_nest_end(struct trace_buffer *buffer) in ring_buffer_nest_end() argument
3151 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_end()
3159 * @buffer: The buffer to commit to
3162 * This commits the data to the ring buffer, and releases any locks held.
3166 int ring_buffer_unlock_commit(struct trace_buffer *buffer, in ring_buffer_unlock_commit() argument
3172 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_unlock_commit()
3176 rb_wakeups(buffer, cpu_buffer); in ring_buffer_unlock_commit()
3204 info->ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
3235 /* See if we shot pass the end of this buffer page */ in __rb_reserve_next()
3285 ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
3316 /* We reserved something on the buffer */ in __rb_reserve_next()
3337 rb_reserve_next_event(struct trace_buffer *buffer, in rb_reserve_next_event() argument
3351 * Due to the ability to swap a cpu buffer from a buffer in rb_reserve_next_event()
3357 if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) { in rb_reserve_next_event()
3366 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { in rb_reserve_next_event()
3405 * ring_buffer_lock_reserve - reserve a part of the buffer
3406 * @buffer: the ring buffer to reserve from
3409 * Returns a reserved event on the ring buffer to copy directly to.
3420 ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) in ring_buffer_lock_reserve() argument
3429 if (unlikely(atomic_read(&buffer->record_disabled))) in ring_buffer_lock_reserve()
3434 if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask))) in ring_buffer_lock_reserve()
3437 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_lock_reserve()
3448 event = rb_reserve_next_event(buffer, cpu_buffer, length); in ring_buffer_lock_reserve()
3498 /* commit not part of this buffer?? */ in rb_decrement_entry()
3504 * @buffer: the ring buffer
3507 * Sometimes an event that is in the ring buffer needs to be ignored.
3508 * This function lets the user discard an event in the ring buffer
3512 * committed. It will try to free the event from the ring buffer
3521 void ring_buffer_discard_commit(struct trace_buffer *buffer, in ring_buffer_discard_commit() argument
3531 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_discard_commit()
3538 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); in ring_buffer_discard_commit()
3555 * ring_buffer_write - write data to the buffer without reserving
3556 * @buffer: The ring buffer to write to.
3558 * @data: The data to write to the buffer.
3561 * one function. If you already have the data to write to the buffer, it
3567 int ring_buffer_write(struct trace_buffer *buffer, in ring_buffer_write() argument
3579 if (atomic_read(&buffer->record_disabled)) in ring_buffer_write()
3584 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_write()
3587 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_write()
3598 event = rb_reserve_next_event(buffer, cpu_buffer, length); in ring_buffer_write()
3608 rb_wakeups(buffer, cpu_buffer); in ring_buffer_write()
3639 * ring_buffer_record_disable - stop all writes into the buffer
3640 * @buffer: The ring buffer to stop writes to.
3642 * This prevents all writes to the buffer. Any attempt to write
3643 * to the buffer after this will fail and return NULL.
3647 void ring_buffer_record_disable(struct trace_buffer *buffer) in ring_buffer_record_disable() argument
3649 atomic_inc(&buffer->record_disabled); in ring_buffer_record_disable()
3654 * ring_buffer_record_enable - enable writes to the buffer
3655 * @buffer: The ring buffer to enable writes
3660 void ring_buffer_record_enable(struct trace_buffer *buffer) in ring_buffer_record_enable() argument
3662 atomic_dec(&buffer->record_disabled); in ring_buffer_record_enable()
3667 * ring_buffer_record_off - stop all writes into the buffer
3668 * @buffer: The ring buffer to stop writes to.
3670 * This prevents all writes to the buffer. Any attempt to write
3671 * to the buffer after this will fail and return NULL.
3677 void ring_buffer_record_off(struct trace_buffer *buffer) in ring_buffer_record_off() argument
3683 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_off()
3685 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd); in ring_buffer_record_off()
3690 * ring_buffer_record_on - restart writes into the buffer
3691 * @buffer: The ring buffer to start writes to.
3693 * This enables all writes to the buffer that was disabled by
3700 void ring_buffer_record_on(struct trace_buffer *buffer) in ring_buffer_record_on() argument
3706 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_on()
3708 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd); in ring_buffer_record_on()
3713 * ring_buffer_record_is_on - return true if the ring buffer can write
3714 * @buffer: The ring buffer to see if write is enabled
3716 * Returns true if the ring buffer is in a state that it accepts writes.
3718 bool ring_buffer_record_is_on(struct trace_buffer *buffer) in ring_buffer_record_is_on() argument
3720 return !atomic_read(&buffer->record_disabled); in ring_buffer_record_is_on()
3724 * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
3725 * @buffer: The ring buffer to see if write is set enabled
3727 * Returns true if the ring buffer is set writable by ring_buffer_record_on().
3730 * It may return true when the ring buffer has been disabled by
3732 * the ring buffer.
3734 bool ring_buffer_record_is_set_on(struct trace_buffer *buffer) in ring_buffer_record_is_set_on() argument
3736 return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF); in ring_buffer_record_is_set_on()
3741 * @buffer: The ring buffer to stop writes to.
3742 * @cpu: The CPU buffer to stop
3744 * This prevents all writes to the buffer. Any attempt to write
3745 * to the buffer after this will fail and return NULL.
3749 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_disable_cpu() argument
3753 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_disable_cpu()
3756 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_disable_cpu()
3762 * ring_buffer_record_enable_cpu - enable writes to the buffer
3763 * @buffer: The ring buffer to enable writes
3769 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_enable_cpu() argument
3773 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_enable_cpu()
3776 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_enable_cpu()
3782 * The total entries in the ring buffer is the running counter
3783 * of entries entered into the ring buffer, minus the sum of
3784 * the entries read from the ring buffer and the number of
3795 * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
3796 * @buffer: The ring buffer
3797 * @cpu: The per CPU buffer to read from.
3799 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu) in ring_buffer_oldest_event_ts() argument
3806 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_oldest_event_ts()
3809 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_oldest_event_ts()
3828 * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer
3829 * @buffer: The ring buffer
3830 * @cpu: The per CPU buffer to read from.
3832 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_bytes_cpu() argument
3837 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_bytes_cpu()
3840 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_bytes_cpu()
3848 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
3849 * @buffer: The ring buffer
3850 * @cpu: The per CPU buffer to get the entries from.
3852 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_entries_cpu() argument
3856 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_entries_cpu()
3859 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries_cpu()
3867 * buffer wrapping around (only if RB_FL_OVERWRITE is on).
3868 * @buffer: The ring buffer
3869 * @cpu: The per CPU buffer to get the number of overruns from
3871 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_overrun_cpu() argument
3876 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_overrun_cpu()
3879 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overrun_cpu()
3888 * commits failing due to the buffer wrapping around while there are uncommitted
3890 * @buffer: The ring buffer
3891 * @cpu: The per CPU buffer to get the number of overruns from
3894 ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_commit_overrun_cpu() argument
3899 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_commit_overrun_cpu()
3902 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_commit_overrun_cpu()
3911 * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
3912 * @buffer: The ring buffer
3913 * @cpu: The per CPU buffer to get the number of overruns from
3916 ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_dropped_events_cpu() argument
3921 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_dropped_events_cpu()
3924 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_dropped_events_cpu()
3933 * @buffer: The ring buffer
3934 * @cpu: The per CPU buffer to get the number of events read
3937 ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_read_events_cpu() argument
3941 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_events_cpu()
3944 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_events_cpu()
3950 * ring_buffer_entries - get the number of entries in a buffer
3951 * @buffer: The ring buffer
3953 * Returns the total number of entries in the ring buffer
3956 unsigned long ring_buffer_entries(struct trace_buffer *buffer) in ring_buffer_entries() argument
3962 /* if you care about this being correct, lock the buffer */ in ring_buffer_entries()
3963 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_entries()
3964 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries()
3973 * ring_buffer_overruns - get the number of overruns in buffer
3974 * @buffer: The ring buffer
3976 * Returns the total number of overruns in the ring buffer
3979 unsigned long ring_buffer_overruns(struct trace_buffer *buffer) in ring_buffer_overruns() argument
3985 /* if you care about this being correct, lock the buffer */ in ring_buffer_overruns()
3986 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_overruns()
3987 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overruns()
4187 /* Don't bother swapping if the ring buffer is empty */ in rb_get_reader_page()
4210 * cpu_buffer->pages just needs to point to the buffer, it in rb_get_reader_page()
4211 * has no specific buffer page to point to. Lets move it out in rb_get_reader_page()
4290 /* This function should not be called when buffer is empty */ in rb_advance_reader()
4321 * Check if we are at the end of the buffer. in rb_advance_iter()
4387 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
4397 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
4415 struct trace_buffer *buffer; in rb_iter_peek() local
4424 buffer = cpu_buffer->buffer; in rb_iter_peek()
4428 * the buffer. A consuming read invalidates the iterator in rb_iter_peek()
4443 * the ring buffer with an active write as the consumer is. in rb_iter_peek()
4478 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_iter_peek()
4488 ring_buffer_normalize_time_stamp(buffer, in rb_iter_peek()
4509 * If an NMI die dumps out the content of the ring buffer in rb_reader_lock()
4511 * preempted a task that holds the ring buffer locks. If in rb_reader_lock()
4513 * to do the read, but this can corrupt the ring buffer, in rb_reader_lock()
4520 /* Continue without locking, but disable the ring buffer */ in rb_reader_lock()
4535 * @buffer: The ring buffer to read
4544 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_peek() argument
4547 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_peek()
4552 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_peek()
4571 * @iter: The ring buffer iterator
4586 * @iter: The ring buffer iterator
4612 * @buffer: The ring buffer to get the next event from
4613 * @cpu: the cpu to read the buffer from
4617 * Returns the next event in the ring buffer, and that event is consumed.
4619 * and eventually empty the ring buffer if the producer is slower.
4622 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_consume() argument
4634 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_consume()
4637 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_consume()
4661 * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
4662 * @buffer: The ring buffer to read from
4663 * @cpu: The cpu buffer to iterate over
4667 * through the buffer. Memory is allocated, buffer recording
4670 * Disabling buffer recording prevents the reading from being
4682 ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags) in ring_buffer_read_prepare() argument
4687 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_prepare()
4700 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_prepare()
4725 * ring_buffer_read_start - start a non consuming read of the buffer
4728 * This finalizes the startup of an iteration through the buffer.
4755 * ring_buffer_read_finish - finish reading the iterator of the buffer
4758 * This re-enables the recording to the buffer, and frees the
4768 * Ring buffer is disabled from recording, here's a good place in ring_buffer_read_finish()
4769 * to check the integrity of the ring buffer. in ring_buffer_read_finish()
4785 * @iter: The ring buffer iterator
4804 * ring_buffer_size - return the size of the ring buffer (in bytes)
4805 * @buffer: The ring buffer.
4806 * @cpu: The CPU to get ring buffer size from.
4808 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) in ring_buffer_size() argument
4812 * BUF_PAGE_SIZE * buffer->nr_pages in ring_buffer_size()
4814 * return the per cpu buffer value. in ring_buffer_size()
4816 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_size()
4819 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages; in ring_buffer_size()
4869 /* Must have disabled the cpu buffer then done a synchronize_rcu */
4890 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
4891 * @buffer: The ring buffer to reset a per cpu buffer of
4892 * @cpu: The CPU buffer to be reset
4894 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_reset_cpu() argument
4896 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_cpu()
4898 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_reset_cpu()
4901 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset_cpu()
4902 mutex_lock(&buffer->mutex); in ring_buffer_reset_cpu()
4915 mutex_unlock(&buffer->mutex); in ring_buffer_reset_cpu()
4920 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
4921 * @buffer: The ring buffer to reset a per cpu buffer of
4922 * @cpu: The CPU buffer to be reset
4924 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer) in ring_buffer_reset_online_cpus() argument
4929 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset_online_cpus()
4930 mutex_lock(&buffer->mutex); in ring_buffer_reset_online_cpus()
4932 for_each_online_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
4933 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
4942 for_each_online_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
4943 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
4951 mutex_unlock(&buffer->mutex); in ring_buffer_reset_online_cpus()
4955 * ring_buffer_reset - reset a ring buffer
4956 * @buffer: The ring buffer to reset all cpu buffers
4958 void ring_buffer_reset(struct trace_buffer *buffer) in ring_buffer_reset() argument
4963 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
4964 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
4973 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
4974 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
4985 * rind_buffer_empty - is the ring buffer empty?
4986 * @buffer: The ring buffer to test
4988 bool ring_buffer_empty(struct trace_buffer *buffer) in ring_buffer_empty() argument
4996 /* yes this is racy, but if you don't like the race, lock the buffer */ in ring_buffer_empty()
4997 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_empty()
4998 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty()
5014 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
5015 * @buffer: The ring buffer
5016 * @cpu: The CPU buffer to test
5018 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_empty_cpu() argument
5025 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_empty_cpu()
5028 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty_cpu()
5041 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
5042 * @buffer_a: One buffer to swap with
5043 * @buffer_b: The other buffer to swap with
5047 * of a CPU buffer and has another back up buffer lying around.
5048 * it is expected that the tracer handles the cpu buffer not being
5101 cpu_buffer_b->buffer = buffer_a; in ring_buffer_swap_cpu()
5102 cpu_buffer_a->buffer = buffer_b; in ring_buffer_swap_cpu()
5116 * ring_buffer_alloc_read_page - allocate a page to read from buffer
5117 * @buffer: the buffer to allocate for.
5118 * @cpu: the cpu buffer to allocate.
5121 * When reading a full page from the ring buffer, these functions
5124 * needs to get pages from the ring buffer, it passes the result
5126 * the page that was allocated, with the read page of the buffer.
5131 void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) in ring_buffer_alloc_read_page() argument
5138 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_alloc_read_page()
5141 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_alloc_read_page()
5172 * @buffer: the buffer the page was allocate for
5173 * @cpu: the cpu buffer the page came from
5178 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data) in ring_buffer_free_read_page() argument
5180 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_free_read_page()
5206 * ring_buffer_read_page - extract a page from the ring buffer
5207 * @buffer: buffer to extract from
5210 * @cpu: the cpu of the buffer to extract
5213 * This function will pull out a page from the ring buffer and consume it.
5216 * to swap with a page in the ring buffer.
5219 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
5222 * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
5230 * The ring buffer can be used anywhere in the kernel and can not
5231 * blindly call wake_up. The layer that uses the ring buffer must be
5238 int ring_buffer_read_page(struct trace_buffer *buffer, in ring_buffer_read_page() argument
5241 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_page()
5252 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_page()
5289 * we must copy the data from the page to the buffer. in ring_buffer_read_page()
5404 * If we were to free the buffer, then the user would lose any trace that was in
5405 * the buffer.
5409 struct trace_buffer *buffer; in trace_rb_cpu_prepare() local
5414 buffer = container_of(node, struct trace_buffer, node); in trace_rb_cpu_prepare()
5415 if (cpumask_test_cpu(cpu, buffer->cpumask)) in trace_rb_cpu_prepare()
5421 for_each_buffer_cpu(buffer, cpu_i) { in trace_rb_cpu_prepare()
5424 nr_pages = buffer->buffers[cpu_i]->nr_pages; in trace_rb_cpu_prepare()
5425 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { in trace_rb_cpu_prepare()
5433 buffer->buffers[cpu] = in trace_rb_cpu_prepare()
5434 rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in trace_rb_cpu_prepare()
5435 if (!buffer->buffers[cpu]) { in trace_rb_cpu_prepare()
5436 WARN(1, "failed to allocate ring buffer on CPU %u\n", in trace_rb_cpu_prepare()
5441 cpumask_set_cpu(cpu, buffer->cpumask); in trace_rb_cpu_prepare()
5447 * This is a basic integrity check of the ring buffer.
5450 * writing to the per cpu ring buffer various sizes of data.
5454 * IPIs to the other CPUs to also write into the ring buffer.
5455 * this is to test the nesting ability of the buffer.
5458 * ring buffer should happen that's not expected, a big warning
5464 struct trace_buffer *buffer; member
5517 /* read rb_test_started before checking buffer enabled */ in rb_write_something()
5520 event = ring_buffer_lock_reserve(data->buffer, len); in rb_write_something()
5534 if (RB_WARN_ON(data->buffer, event_len < len)) in rb_write_something()
5560 ring_buffer_unlock_commit(data->buffer, event); in rb_write_something()
5606 struct trace_buffer *buffer; in test_ringbuffer() local
5611 pr_warn("Lockdown is enabled, skipping ring buffer tests\n"); in test_ringbuffer()
5615 pr_info("Running ring buffer tests...\n"); in test_ringbuffer()
5617 buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE); in test_ringbuffer()
5618 if (WARN_ON(!buffer)) in test_ringbuffer()
5621 /* Disable buffer so that threads can't write to it yet */ in test_ringbuffer()
5622 ring_buffer_record_off(buffer); in test_ringbuffer()
5625 rb_data[cpu].buffer = buffer; in test_ringbuffer()
5648 ring_buffer_record_on(buffer); in test_ringbuffer()
5650 * Show buffer is enabled before setting rb_test_started. in test_ringbuffer()
5653 * buffer gets enabled, there will always be some kind of in test_ringbuffer()
5656 * the threads see that the buffer is active. in test_ringbuffer()
5674 ring_buffer_free(buffer); in test_ringbuffer()
5714 if (RB_WARN_ON(buffer, total_dropped)) in test_ringbuffer()
5719 while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) { in test_ringbuffer()
5726 pr_info("buffer had: %.*s\n", item->size, item->str); in test_ringbuffer()
5728 RB_WARN_ON(buffer, 1); in test_ringbuffer()
5748 if (RB_WARN_ON(buffer, total_len != total_alloc || in test_ringbuffer()
5752 if (RB_WARN_ON(buffer, total_lost + total_read != total_events)) in test_ringbuffer()
5758 pr_info("Ring buffer PASSED!\n"); in test_ringbuffer()
5760 ring_buffer_free(buffer); in test_ringbuffer()