Lines Matching +full:cpu +full:- +full:read
1 // SPDX-License-Identifier: GPL-2.0
27 #include <linux/cpu.h>
66 * allocated for each CPU. A writer may only write to a buffer that is
67 * associated with the CPU it is currently executing on. A reader may read
68 * from any per cpu buffer.
70 * The reader is special. For each per cpu buffer, the reader has its own
71 * reader page. When a reader has read the entire reader page, this reader
80 * +------+
83 * +------+ +---+ +---+ +---+
84 * | |-->| |-->| |
85 * +---+ +---+ +---+
88 * +---------------+
91 * +------+
93 * |page |------------------v
94 * +------+ +---+ +---+ +---+
95 * | |-->| |-->| |
96 * +---+ +---+ +---+
99 * +---------------+
102 * +------+
104 * |page |------------------v
105 * +------+ +---+ +---+ +---+
106 * ^ | |-->| |-->| |
107 * | +---+ +---+ +---+
110 * +------------------------------+
113 * +------+
115 * |page |------------------v
116 * +------+ +---+ +---+ +---+
117 * ^ | | | |-->| |
118 * | New +---+ +---+ +---+
119 * | Reader------^ |
121 * +------------------------------+
164 (event->type_len >= RINGBUF_TYPE_TIME_EXTEND)
168 return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta; in rb_null_event()
174 event->type_len = RINGBUF_TYPE_PADDING; in rb_event_set_padding()
175 event->time_delta = 0; in rb_event_set_padding()
183 if (event->type_len) in rb_event_data_length()
184 length = event->type_len * RB_ALIGNMENT; in rb_event_data_length()
186 length = event->array[0]; in rb_event_data_length()
198 switch (event->type_len) { in rb_event_length()
202 return -1; in rb_event_length()
203 return event->array[0] + RB_EVNT_HDR_SIZE; in rb_event_length()
238 * ring_buffer_event_length - return the length of the event
255 if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX) in ring_buffer_event_length()
257 length -= RB_EVNT_HDR_SIZE; in ring_buffer_event_length()
258 if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) in ring_buffer_event_length()
259 length -= sizeof(event->array[0]); in ring_buffer_event_length()
270 WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX); in rb_event_data()
272 if (event->type_len) in rb_event_data()
273 return (void *)&event->array[0]; in rb_event_data()
275 return (void *)&event->array[1]; in rb_event_data()
279 * ring_buffer_event_data - return the data of the event
288 #define for_each_buffer_cpu(buffer, cpu) \ argument
289 for_each_cpu(cpu, buffer->cpumask)
291 #define for_each_online_buffer_cpu(buffer, cpu) \ argument
292 for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
295 #define TS_MASK ((1ULL << TS_SHIFT) - 1)
302 ts = event->array[0]; in rb_event_time_stamp()
304 ts += event->time_delta; in rb_event_time_stamp()
331 unsigned read; /* index for next read */ member
354 local_set(&bpage->commit, 0); in rb_init_page()
363 free_page((unsigned long)bpage->page); in free_buffer_page()
377 #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
379 /* Max payload is BUF_PAGE_SIZE - header (8bytes) */
380 #define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
438 * EXTEND - wants a time extend
439 * ABSOLUTE - the buffer requests all events to have absolute time stamps
440 * FORCE - force a full time stamp.
496 int cpu; member
507 struct buffer_page *head_page; /* read from head */
526 unsigned long read; member
583 * - Reads may fail if it interrupted a modification of the time stamp.
585 * the read itself is interrupted by a write.
588 * - Writes always succeed and will overwrite other writes and writes
591 * - A write followed by a read of the same time stamp will always succeed,
594 * - A cmpxchg will fail if it interrupted another write or cmpxchg.
600 * The two most significant bits of each half holds a 2 bit counter (0-3).
606 #define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1)
631 * If the read is interrupted by a write, then the cnt will in __rb_time_read()
632 * be different. Loop until both top and bottom have been read in __rb_time_read()
636 c = local_read(&t->cnt); in __rb_time_read()
637 top = local_read(&t->top); in __rb_time_read()
638 bottom = local_read(&t->bottom); in __rb_time_read()
639 msb = local_read(&t->msb); in __rb_time_read()
640 } while (c != local_read(&t->cnt)); in __rb_time_read()
687 cnt = local_inc_return(&t->cnt); in rb_time_set()
688 rb_time_val_set(&t->top, top, cnt); in rb_time_set()
689 rb_time_val_set(&t->bottom, bottom, cnt); in rb_time_set()
690 rb_time_val_set(&t->msb, val >> RB_TIME_MSB_SHIFT, cnt); in rb_time_set()
691 } while (cnt != local_read(&t->cnt)); in rb_time_set()
716 cnt = local_read(&t->cnt); in rb_time_cmpxchg()
730 if (!rb_time_read_cmpxchg(&t->cnt, cnt, cnt2)) in rb_time_cmpxchg()
732 if (!rb_time_read_cmpxchg(&t->msb, msb, msb2)) in rb_time_cmpxchg()
734 if (!rb_time_read_cmpxchg(&t->top, top, top2)) in rb_time_cmpxchg()
736 if (!rb_time_read_cmpxchg(&t->bottom, bottom, bottom2)) in rb_time_cmpxchg()
747 *ret = local64_read(&t->time); in rb_time_read()
752 local64_set(&t->time, val); in rb_time_set()
758 val = local64_cmpxchg(&t->time, expect, set); in rb_time_cmpxchg()
774 struct buffer_page *page = cpu_buffer->commit_page; in verify_event()
775 struct buffer_page *tail_page = READ_ONCE(cpu_buffer->tail_page); in verify_event()
786 commit = local_read(&page->page->commit); in verify_event()
787 write = local_read(&page->write); in verify_event()
788 if (addr >= (unsigned long)&page->page->data[commit] && in verify_event()
789 addr < (unsigned long)&page->page->data[write]) in verify_event()
792 next = rb_list_head(page->list.next); in verify_event()
826 * ring_buffer_event_time_stamp - return the event's current time stamp
845 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()]; in ring_buffer_event_time_stamp()
850 if (event->type_len == RINGBUF_TYPE_TIME_STAMP) { in ring_buffer_event_time_stamp()
852 return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp); in ring_buffer_event_time_stamp()
855 nest = local_read(&cpu_buffer->committing); in ring_buffer_event_time_stamp()
860 /* Read the current saved nesting level time stamp */ in ring_buffer_event_time_stamp()
861 if (likely(--nest < MAX_NEST)) in ring_buffer_event_time_stamp()
862 return cpu_buffer->event_stamp[nest]; in ring_buffer_event_time_stamp()
869 if (!rb_time_read(&cpu_buffer->write_stamp, &ts)) in ring_buffer_event_time_stamp()
870 /* Screw it, just read the current time */ in ring_buffer_event_time_stamp()
871 ts = rb_time_stamp(cpu_buffer->buffer); in ring_buffer_event_time_stamp()
877 * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
879 * @cpu: The cpu of the ring_buffer to get the number of pages from
883 size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu) in ring_buffer_nr_pages() argument
885 return buffer->buffers[cpu]->nr_pages; in ring_buffer_nr_pages()
889 * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
891 * @cpu: The cpu of the ring_buffer to get the number of pages from
895 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu) in ring_buffer_nr_dirty_pages() argument
897 size_t read; in ring_buffer_nr_dirty_pages() local
901 read = local_read(&buffer->buffers[cpu]->pages_read); in ring_buffer_nr_dirty_pages()
902 lost = local_read(&buffer->buffers[cpu]->pages_lost); in ring_buffer_nr_dirty_pages()
903 cnt = local_read(&buffer->buffers[cpu]->pages_touched); in ring_buffer_nr_dirty_pages()
908 cnt -= lost; in ring_buffer_nr_dirty_pages()
910 /* The reader can read an empty page, but not more than that */ in ring_buffer_nr_dirty_pages()
911 if (cnt < read) { in ring_buffer_nr_dirty_pages()
912 WARN_ON_ONCE(read > cnt + 1); in ring_buffer_nr_dirty_pages()
916 return cnt - read; in ring_buffer_nr_dirty_pages()
919 static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full) in full_hit() argument
921 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in full_hit()
925 nr_pages = cpu_buffer->nr_pages; in full_hit()
929 dirty = ring_buffer_nr_dirty_pages(buffer, cpu); in full_hit()
935 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
944 wake_up_all(&rbwork->waiters); in rb_wake_up_waiters()
945 if (rbwork->full_waiters_pending || rbwork->wakeup_full) { in rb_wake_up_waiters()
946 rbwork->wakeup_full = false; in rb_wake_up_waiters()
947 rbwork->full_waiters_pending = false; in rb_wake_up_waiters()
948 wake_up_all(&rbwork->full_waiters); in rb_wake_up_waiters()
953 * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
959 void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu) in ring_buffer_wake_waiters() argument
967 if (cpu == RING_BUFFER_ALL_CPUS) { in ring_buffer_wake_waiters()
970 for_each_buffer_cpu(buffer, cpu) in ring_buffer_wake_waiters()
971 ring_buffer_wake_waiters(buffer, cpu); in ring_buffer_wake_waiters()
973 rbwork = &buffer->irq_work; in ring_buffer_wake_waiters()
975 if (WARN_ON_ONCE(!buffer->buffers)) in ring_buffer_wake_waiters()
977 if (WARN_ON_ONCE(cpu >= nr_cpu_ids)) in ring_buffer_wake_waiters()
980 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wake_waiters()
981 /* The CPU buffer may not have been initialized yet */ in ring_buffer_wake_waiters()
984 rbwork = &cpu_buffer->irq_work; in ring_buffer_wake_waiters()
987 rbwork->wait_index++; in ring_buffer_wake_waiters()
991 rb_wake_up_waiters(&rbwork->work); in ring_buffer_wake_waiters()
995 * ring_buffer_wait - wait for input to the ring buffer
997 * @cpu: the cpu buffer to wait on
998 * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
1000 * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
1001 * as data is added to any of the @buffer's cpu buffers. Otherwise
1002 * it will wait for data to be added to a specific cpu buffer.
1004 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) in ring_buffer_wait() argument
1014 * data in any cpu buffer, or a specific buffer, put the in ring_buffer_wait()
1017 if (cpu == RING_BUFFER_ALL_CPUS) { in ring_buffer_wait()
1018 work = &buffer->irq_work; in ring_buffer_wait()
1019 /* Full only makes sense on per cpu reads */ in ring_buffer_wait()
1022 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_wait()
1023 return -ENODEV; in ring_buffer_wait()
1024 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wait()
1025 work = &cpu_buffer->irq_work; in ring_buffer_wait()
1028 wait_index = READ_ONCE(work->wait_index); in ring_buffer_wait()
1032 prepare_to_wait(&work->full_waiters, &wait, TASK_INTERRUPTIBLE); in ring_buffer_wait()
1034 prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE); in ring_buffer_wait()
1057 work->full_waiters_pending = true; in ring_buffer_wait()
1059 work->waiters_pending = true; in ring_buffer_wait()
1062 ret = -EINTR; in ring_buffer_wait()
1066 if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) in ring_buffer_wait()
1069 if (cpu != RING_BUFFER_ALL_CPUS && in ring_buffer_wait()
1070 !ring_buffer_empty_cpu(buffer, cpu)) { in ring_buffer_wait()
1078 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_wait()
1079 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page; in ring_buffer_wait()
1080 done = !pagebusy && full_hit(buffer, cpu, full); in ring_buffer_wait()
1082 if (!cpu_buffer->shortest_full || in ring_buffer_wait()
1083 cpu_buffer->shortest_full > full) in ring_buffer_wait()
1084 cpu_buffer->shortest_full = full; in ring_buffer_wait()
1085 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_wait()
1094 if (wait_index != work->wait_index) in ring_buffer_wait()
1099 finish_wait(&work->full_waiters, &wait); in ring_buffer_wait()
1101 finish_wait(&work->waiters, &wait); in ring_buffer_wait()
1107 * ring_buffer_poll_wait - poll on buffer input
1109 * @cpu: the cpu buffer to wait on
1112 * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
1114 * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
1115 * as data is added to any of the @buffer's cpu buffers. Otherwise
1116 * it will wait for data to be added to a specific cpu buffer.
1121 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, in ring_buffer_poll_wait() argument
1127 if (cpu == RING_BUFFER_ALL_CPUS) { in ring_buffer_poll_wait()
1128 work = &buffer->irq_work; in ring_buffer_poll_wait()
1131 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_poll_wait()
1132 return -EINVAL; in ring_buffer_poll_wait()
1134 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_poll_wait()
1135 work = &cpu_buffer->irq_work; in ring_buffer_poll_wait()
1139 poll_wait(filp, &work->full_waiters, poll_table); in ring_buffer_poll_wait()
1140 work->full_waiters_pending = true; in ring_buffer_poll_wait()
1142 poll_wait(filp, &work->waiters, poll_table); in ring_buffer_poll_wait()
1143 work->waiters_pending = true; in ring_buffer_poll_wait()
1162 return full_hit(buffer, cpu, full) ? EPOLLIN | EPOLLRDNORM : 0; in ring_buffer_poll_wait()
1164 if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) || in ring_buffer_poll_wait()
1165 (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu))) in ring_buffer_poll_wait()
1178 atomic_inc(&__b->buffer->record_disabled); \
1180 atomic_inc(&b->record_disabled); \
1193 /* Skip retpolines :-( */ in rb_time_stamp()
1194 if (IS_ENABLED(CONFIG_RETPOLINE) && likely(buffer->clock == trace_clock_local)) in rb_time_stamp()
1197 ts = buffer->clock(); in rb_time_stamp()
1216 int cpu, u64 *ts) in ring_buffer_normalize_time_stamp() argument
1225 * Although writes only happen on the CPU that they are on,
1227 * happen on any CPU.
1255 * head->list->prev->next bit 1 bit 0
1256 * ------- -------
1263 * +----+ +-----+ +-----+
1264 * | |------>| T |---X--->| N |
1265 * | |<------| | | |
1266 * +----+ +-----+ +-----+
1268 * | +-----+ | |
1269 * +----------| R |----------+ |
1270 * | |<-----------+
1271 * +-----+
1273 * Key: ---X--> HEAD flag set in pointer
1303 * rb_list_head - remove any bit
1313 * rb_is_head_page - test if the given page is the head page
1325 val = (unsigned long)list->next; in rb_is_head_page()
1327 if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list) in rb_is_head_page()
1342 struct list_head *list = page->list.prev; in rb_is_reader_page()
1344 return rb_list_head(list->next) != &page->list; in rb_is_reader_page()
1348 * rb_set_list_to_head - set a list_head to be pointing to head.
1354 ptr = (unsigned long *)&list->next; in rb_set_list_to_head()
1360 * rb_head_page_activate - sets up head page
1366 head = cpu_buffer->head_page; in rb_head_page_activate()
1373 rb_set_list_to_head(head->list.prev); in rb_head_page_activate()
1378 unsigned long *ptr = (unsigned long *)&list->next; in rb_list_head_clear()
1384 * rb_head_page_deactivate - clears head page ptr (for free list)
1392 rb_list_head_clear(cpu_buffer->pages); in rb_head_page_deactivate()
1394 list_for_each(hd, cpu_buffer->pages) in rb_head_page_deactivate()
1404 unsigned long val = (unsigned long)&head->list; in rb_head_page_set()
1407 list = &prev->list; in rb_head_page_set()
1411 ret = cmpxchg((unsigned long *)&list->next, in rb_head_page_set()
1450 struct list_head *p = rb_list_head((*bpage)->list.next); in rb_inc_page()
1463 if (RB_WARN_ON(cpu_buffer, !cpu_buffer->head_page)) in rb_set_head_page()
1467 list = cpu_buffer->pages; in rb_set_head_page()
1468 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list)) in rb_set_head_page()
1471 page = head = cpu_buffer->head_page; in rb_set_head_page()
1480 if (rb_is_head_page(page, page->list.prev)) { in rb_set_head_page()
1481 cpu_buffer->head_page = page; in rb_set_head_page()
1496 unsigned long *ptr = (unsigned long *)&old->list.prev->next; in rb_head_page_replace()
1503 ret = cmpxchg(ptr, val, (unsigned long)&new->list); in rb_head_page_replace()
1509 * rb_tail_page_update - move the tail page forward
1527 old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write); in rb_tail_page_update()
1528 old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries); in rb_tail_page_update()
1530 local_inc(&cpu_buffer->pages_touched); in rb_tail_page_update()
1542 if (tail_page == READ_ONCE(cpu_buffer->tail_page)) { in rb_tail_page_update()
1557 (void)local_cmpxchg(&next_page->write, old_write, val); in rb_tail_page_update()
1558 (void)local_cmpxchg(&next_page->entries, old_entries, eval); in rb_tail_page_update()
1565 local_set(&next_page->page->commit, 0); in rb_tail_page_update()
1568 (void)cmpxchg(&cpu_buffer->tail_page, tail_page, next_page); in rb_tail_page_update()
1584 * rb_check_list - make sure a pointer to a list has the last bits zero
1589 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev) != list->prev)) in rb_check_list()
1591 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->next) != list->next)) in rb_check_list()
1597 * rb_check_pages - integrity check of buffer pages
1598 * @cpu_buffer: CPU buffer with pages to test
1605 struct list_head *head = cpu_buffer->pages; in rb_check_pages()
1609 if (cpu_buffer->head_page) in rb_check_pages()
1614 if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) in rb_check_pages()
1615 return -1; in rb_check_pages()
1616 if (RB_WARN_ON(cpu_buffer, head->prev->next != head)) in rb_check_pages()
1617 return -1; in rb_check_pages()
1620 return -1; in rb_check_pages()
1624 bpage->list.next->prev != &bpage->list)) in rb_check_pages()
1625 return -1; in rb_check_pages()
1627 bpage->list.prev->next != &bpage->list)) in rb_check_pages()
1628 return -1; in rb_check_pages()
1629 if (rb_check_list(cpu_buffer, &bpage->list)) in rb_check_pages()
1630 return -1; in rb_check_pages()
1642 bool user_thread = current->mm != NULL; in __rb_allocate_pages()
1655 return -ENOMEM; in __rb_allocate_pages()
1659 * gracefully without invoking oom-killer and the system is not in __rb_allocate_pages()
1679 mflags, cpu_to_node(cpu_buffer->cpu)); in __rb_allocate_pages()
1685 list_add(&bpage->list, pages); in __rb_allocate_pages()
1687 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags, 0); in __rb_allocate_pages()
1690 bpage->page = page_address(page); in __rb_allocate_pages()
1691 rb_init_page(bpage->page); in __rb_allocate_pages()
1703 list_del_init(&bpage->list); in __rb_allocate_pages()
1709 return -ENOMEM; in __rb_allocate_pages()
1720 return -ENOMEM; in rb_allocate_pages()
1727 cpu_buffer->pages = pages.next; in rb_allocate_pages()
1730 cpu_buffer->nr_pages = nr_pages; in rb_allocate_pages()
1738 rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) in rb_allocate_cpu_buffer() argument
1746 GFP_KERNEL, cpu_to_node(cpu)); in rb_allocate_cpu_buffer()
1750 cpu_buffer->cpu = cpu; in rb_allocate_cpu_buffer()
1751 cpu_buffer->buffer = buffer; in rb_allocate_cpu_buffer()
1752 raw_spin_lock_init(&cpu_buffer->reader_lock); in rb_allocate_cpu_buffer()
1753 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); in rb_allocate_cpu_buffer()
1754 cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; in rb_allocate_cpu_buffer()
1755 INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler); in rb_allocate_cpu_buffer()
1756 init_completion(&cpu_buffer->update_done); in rb_allocate_cpu_buffer()
1757 init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters); in rb_allocate_cpu_buffer()
1758 init_waitqueue_head(&cpu_buffer->irq_work.waiters); in rb_allocate_cpu_buffer()
1759 init_waitqueue_head(&cpu_buffer->irq_work.full_waiters); in rb_allocate_cpu_buffer()
1762 GFP_KERNEL, cpu_to_node(cpu)); in rb_allocate_cpu_buffer()
1768 cpu_buffer->reader_page = bpage; in rb_allocate_cpu_buffer()
1769 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0); in rb_allocate_cpu_buffer()
1772 bpage->page = page_address(page); in rb_allocate_cpu_buffer()
1773 rb_init_page(bpage->page); in rb_allocate_cpu_buffer()
1775 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); in rb_allocate_cpu_buffer()
1776 INIT_LIST_HEAD(&cpu_buffer->new_pages); in rb_allocate_cpu_buffer()
1782 cpu_buffer->head_page in rb_allocate_cpu_buffer()
1783 = list_entry(cpu_buffer->pages, struct buffer_page, list); in rb_allocate_cpu_buffer()
1784 cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; in rb_allocate_cpu_buffer()
1791 free_buffer_page(cpu_buffer->reader_page); in rb_allocate_cpu_buffer()
1800 struct list_head *head = cpu_buffer->pages; in rb_free_cpu_buffer()
1803 free_buffer_page(cpu_buffer->reader_page); in rb_free_cpu_buffer()
1809 list_del_init(&bpage->list); in rb_free_cpu_buffer()
1820 * __ring_buffer_alloc - allocate a new ring_buffer
1821 * @size: the size in bytes per cpu that is needed.
1836 int cpu; in __ring_buffer_alloc() local
1845 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) in __ring_buffer_alloc()
1849 buffer->flags = flags; in __ring_buffer_alloc()
1850 buffer->clock = trace_clock_local; in __ring_buffer_alloc()
1851 buffer->reader_lock_key = key; in __ring_buffer_alloc()
1853 init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); in __ring_buffer_alloc()
1854 init_waitqueue_head(&buffer->irq_work.waiters); in __ring_buffer_alloc()
1860 buffer->cpus = nr_cpu_ids; in __ring_buffer_alloc()
1863 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), in __ring_buffer_alloc()
1865 if (!buffer->buffers) in __ring_buffer_alloc()
1868 cpu = raw_smp_processor_id(); in __ring_buffer_alloc()
1869 cpumask_set_cpu(cpu, buffer->cpumask); in __ring_buffer_alloc()
1870 buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in __ring_buffer_alloc()
1871 if (!buffer->buffers[cpu]) in __ring_buffer_alloc()
1874 ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in __ring_buffer_alloc()
1878 mutex_init(&buffer->mutex); in __ring_buffer_alloc()
1883 for_each_buffer_cpu(buffer, cpu) { in __ring_buffer_alloc()
1884 if (buffer->buffers[cpu]) in __ring_buffer_alloc()
1885 rb_free_cpu_buffer(buffer->buffers[cpu]); in __ring_buffer_alloc()
1887 kfree(buffer->buffers); in __ring_buffer_alloc()
1890 free_cpumask_var(buffer->cpumask); in __ring_buffer_alloc()
1899 * ring_buffer_free - free a ring buffer.
1905 int cpu; in ring_buffer_free() local
1907 cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in ring_buffer_free()
1909 for_each_buffer_cpu(buffer, cpu) in ring_buffer_free()
1910 rb_free_cpu_buffer(buffer->buffers[cpu]); in ring_buffer_free()
1912 kfree(buffer->buffers); in ring_buffer_free()
1913 free_cpumask_var(buffer->cpumask); in ring_buffer_free()
1922 buffer->clock = clock; in ring_buffer_set_clock()
1927 buffer->time_stamp_abs = abs; in ring_buffer_set_time_stamp_abs()
1932 return buffer->time_stamp_abs; in ring_buffer_time_stamp_abs()
1939 return local_read(&bpage->entries) & RB_WRITE_MASK; in rb_page_entries()
1944 return local_read(&bpage->write) & RB_WRITE_MASK; in rb_page_write()
1959 raw_spin_lock_irq(&cpu_buffer->reader_lock); in rb_remove_pages()
1960 atomic_inc(&cpu_buffer->record_disabled); in rb_remove_pages()
1970 tail_page = &cpu_buffer->tail_page->list; in rb_remove_pages()
1976 if (cpu_buffer->tail_page == cpu_buffer->reader_page) in rb_remove_pages()
1977 tail_page = rb_list_head(tail_page->next); in rb_remove_pages()
1981 first_page = list_entry(rb_list_head(to_remove->next), in rb_remove_pages()
1985 to_remove = rb_list_head(to_remove)->next; in rb_remove_pages()
1989 next_page = rb_list_head(to_remove)->next; in rb_remove_pages()
1996 tail_page->next = (struct list_head *)((unsigned long)next_page | in rb_remove_pages()
1999 next_page->prev = tail_page; in rb_remove_pages()
2002 cpu_buffer->pages = next_page; in rb_remove_pages()
2006 cpu_buffer->head_page = list_entry(next_page, in rb_remove_pages()
2010 * change read pointer to make sure any read iterators reset in rb_remove_pages()
2013 cpu_buffer->read = 0; in rb_remove_pages()
2016 atomic_dec(&cpu_buffer->record_disabled); in rb_remove_pages()
2017 raw_spin_unlock_irq(&cpu_buffer->reader_lock); in rb_remove_pages()
2019 RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)); in rb_remove_pages()
2041 local_add(page_entries, &cpu_buffer->overrun); in rb_remove_pages()
2042 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes); in rb_remove_pages()
2043 local_inc(&cpu_buffer->pages_lost); in rb_remove_pages()
2051 nr_removed--; in rb_remove_pages()
2063 struct list_head *pages = &cpu_buffer->new_pages; in rb_insert_pages()
2066 raw_spin_lock_irq(&cpu_buffer->reader_lock); in rb_insert_pages()
2074 * 2. We cmpxchg the prev_page->next to point from head page to the in rb_insert_pages()
2076 * 3. Finally, we update the head->prev to the end of new list. in rb_insert_pages()
2083 while (retries--) { in rb_insert_pages()
2088 head_page = &rb_set_head_page(cpu_buffer)->list; in rb_insert_pages()
2091 prev_page = head_page->prev; in rb_insert_pages()
2093 first_page = pages->next; in rb_insert_pages()
2094 last_page = pages->prev; in rb_insert_pages()
2099 last_page->next = head_page_with_bit; in rb_insert_pages()
2100 first_page->prev = prev_page; in rb_insert_pages()
2102 r = cmpxchg(&prev_page->next, head_page_with_bit, first_page); in rb_insert_pages()
2110 head_page->prev = last_page; in rb_insert_pages()
2123 raw_spin_unlock_irq(&cpu_buffer->reader_lock); in rb_insert_pages()
2128 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, in rb_insert_pages()
2130 list_del_init(&bpage->list); in rb_insert_pages()
2141 if (cpu_buffer->nr_pages_to_update > 0) in rb_update_pages()
2145 -cpu_buffer->nr_pages_to_update); in rb_update_pages()
2148 cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update; in rb_update_pages()
2156 complete(&cpu_buffer->update_done); in update_pages_handler()
2160 * ring_buffer_resize - resize the ring buffer
2163 * @cpu_id: the cpu buffer to resize
2174 int cpu, err; in ring_buffer_resize() local
2177 * Always succeed at resizing a non-existent buffer: in ring_buffer_resize()
2184 !cpumask_test_cpu(cpu_id, buffer->cpumask)) in ring_buffer_resize()
2194 mutex_lock(&buffer->mutex); in ring_buffer_resize()
2203 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2204 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2205 if (atomic_read(&cpu_buffer->resize_disabled)) { in ring_buffer_resize()
2206 err = -EBUSY; in ring_buffer_resize()
2212 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2213 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2215 cpu_buffer->nr_pages_to_update = nr_pages - in ring_buffer_resize()
2216 cpu_buffer->nr_pages; in ring_buffer_resize()
2220 if (cpu_buffer->nr_pages_to_update <= 0) in ring_buffer_resize()
2226 INIT_LIST_HEAD(&cpu_buffer->new_pages); in ring_buffer_resize()
2227 if (__rb_allocate_pages(cpu_buffer, cpu_buffer->nr_pages_to_update, in ring_buffer_resize()
2228 &cpu_buffer->new_pages)) { in ring_buffer_resize()
2230 err = -ENOMEM; in ring_buffer_resize()
2241 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2242 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2243 if (!cpu_buffer->nr_pages_to_update) in ring_buffer_resize()
2246 /* Can't run something on an offline CPU. */ in ring_buffer_resize()
2247 if (!cpu_online(cpu)) { in ring_buffer_resize()
2249 cpu_buffer->nr_pages_to_update = 0; in ring_buffer_resize()
2251 schedule_work_on(cpu, in ring_buffer_resize()
2252 &cpu_buffer->update_pages_work); in ring_buffer_resize()
2257 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2258 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2259 if (!cpu_buffer->nr_pages_to_update) in ring_buffer_resize()
2262 if (cpu_online(cpu)) in ring_buffer_resize()
2263 wait_for_completion(&cpu_buffer->update_done); in ring_buffer_resize()
2264 cpu_buffer->nr_pages_to_update = 0; in ring_buffer_resize()
2269 cpu_buffer = buffer->buffers[cpu_id]; in ring_buffer_resize()
2271 if (nr_pages == cpu_buffer->nr_pages) in ring_buffer_resize()
2279 if (atomic_read(&cpu_buffer->resize_disabled)) { in ring_buffer_resize()
2280 err = -EBUSY; in ring_buffer_resize()
2284 cpu_buffer->nr_pages_to_update = nr_pages - in ring_buffer_resize()
2285 cpu_buffer->nr_pages; in ring_buffer_resize()
2287 INIT_LIST_HEAD(&cpu_buffer->new_pages); in ring_buffer_resize()
2288 if (cpu_buffer->nr_pages_to_update > 0 && in ring_buffer_resize()
2289 __rb_allocate_pages(cpu_buffer, cpu_buffer->nr_pages_to_update, in ring_buffer_resize()
2290 &cpu_buffer->new_pages)) { in ring_buffer_resize()
2291 err = -ENOMEM; in ring_buffer_resize()
2297 /* Can't run something on an offline CPU. */ in ring_buffer_resize()
2302 &cpu_buffer->update_pages_work); in ring_buffer_resize()
2303 wait_for_completion(&cpu_buffer->update_done); in ring_buffer_resize()
2306 cpu_buffer->nr_pages_to_update = 0; in ring_buffer_resize()
2318 if (atomic_read(&buffer->record_disabled)) { in ring_buffer_resize()
2319 atomic_inc(&buffer->record_disabled); in ring_buffer_resize()
2327 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2328 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2331 atomic_dec(&buffer->record_disabled); in ring_buffer_resize()
2334 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
2338 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2341 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2342 cpu_buffer->nr_pages_to_update = 0; in ring_buffer_resize()
2344 if (list_empty(&cpu_buffer->new_pages)) in ring_buffer_resize()
2347 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, in ring_buffer_resize()
2349 list_del_init(&bpage->list); in ring_buffer_resize()
2354 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
2361 mutex_lock(&buffer->mutex); in ring_buffer_change_overwrite()
2363 buffer->flags |= RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
2365 buffer->flags &= ~RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
2366 mutex_unlock(&buffer->mutex); in ring_buffer_change_overwrite()
2372 return bpage->page->data + index; in __rb_page_index()
2378 return __rb_page_index(cpu_buffer->reader_page, in rb_reader_event()
2379 cpu_buffer->reader_page->read); in rb_reader_event()
2384 return local_read(&bpage->page->commit); in rb_page_commit()
2391 struct buffer_page *iter_head_page = iter->head_page; in rb_iter_head_event()
2395 if (iter->head != iter->next_event) in rb_iter_head_event()
2396 return iter->event; in rb_iter_head_event()
2405 event = __rb_page_index(iter_head_page, iter->head); in rb_iter_head_event()
2414 if ((iter->head + length) > commit || length > BUF_MAX_DATA_SIZE) in rb_iter_head_event()
2415 /* Writer corrupted the read? */ in rb_iter_head_event()
2418 memcpy(iter->event, event, length); in rb_iter_head_event()
2425 /* Make sure the page didn't change since we read this */ in rb_iter_head_event()
2426 if (iter->page_stamp != iter_head_page->page->time_stamp || in rb_iter_head_event()
2430 iter->next_event = iter->head + length; in rb_iter_head_event()
2431 return iter->event; in rb_iter_head_event()
2434 iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; in rb_iter_head_event()
2435 iter->head = 0; in rb_iter_head_event()
2436 iter->next_event = 0; in rb_iter_head_event()
2437 iter->missed_events = 1; in rb_iter_head_event()
2450 return rb_page_commit(cpu_buffer->commit_page); in rb_commit_index()
2458 return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE; in rb_event_index()
2463 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; in rb_inc_iter()
2471 if (iter->head_page == cpu_buffer->reader_page) in rb_inc_iter()
2472 iter->head_page = rb_set_head_page(cpu_buffer); in rb_inc_iter()
2474 rb_inc_page(&iter->head_page); in rb_inc_iter()
2476 iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; in rb_inc_iter()
2477 iter->head = 0; in rb_inc_iter()
2478 iter->next_event = 0; in rb_inc_iter()
2482 * rb_handle_head_page - writer hit the head page
2486 * -1 on error
2510 * NORMAL - an interrupt already moved it for us in rb_handle_head_page()
2511 * HEAD - we are the first to get here. in rb_handle_head_page()
2512 * UPDATE - we are the interrupt interrupting in rb_handle_head_page()
2514 * MOVED - a reader on another CPU moved the next in rb_handle_head_page()
2526 local_add(entries, &cpu_buffer->overrun); in rb_handle_head_page()
2527 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes); in rb_handle_head_page()
2528 local_inc(&cpu_buffer->pages_lost); in rb_handle_head_page()
2553 * The reader is on another CPU and just did in rb_handle_head_page()
2560 return -1; in rb_handle_head_page()
2567 * The reader (on another CPU) will spin till in rb_handle_head_page()
2585 * HEAD - an interrupt came in and already set it. in rb_handle_head_page()
2586 * NORMAL - One of two things: in rb_handle_head_page()
2598 return -1; in rb_handle_head_page()
2614 buffer_tail_page = READ_ONCE(cpu_buffer->tail_page); in rb_handle_head_page()
2637 return -1; in rb_handle_head_page()
2647 struct buffer_page *tail_page = info->tail_page; in rb_reset_tail()
2649 unsigned long length = info->length; in rb_reset_tail()
2662 tail_page->real_end = 0; in rb_reset_tail()
2664 local_sub(length, &tail_page->write); in rb_reset_tail()
2671 local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes); in rb_reset_tail()
2678 tail_page->real_end = tail; in rb_reset_tail()
2691 if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) { in rb_reset_tail()
2701 local_sub(length, &tail_page->write); in rb_reset_tail()
2706 event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE; in rb_reset_tail()
2707 event->type_len = RINGBUF_TYPE_PADDING; in rb_reset_tail()
2709 event->time_delta = 1; in rb_reset_tail()
2711 /* Make sure the padding is visible before the tail_page->write update */ in rb_reset_tail()
2715 length = (tail + length) - BUF_PAGE_SIZE; in rb_reset_tail()
2716 local_sub(length, &tail_page->write); in rb_reset_tail()
2728 struct buffer_page *tail_page = info->tail_page; in rb_move_tail()
2729 struct buffer_page *commit_page = cpu_buffer->commit_page; in rb_move_tail()
2730 struct trace_buffer *buffer = cpu_buffer->buffer; in rb_move_tail()
2744 local_inc(&cpu_buffer->commit_overrun); in rb_move_tail()
2752 * could be on another CPU trying to swap its reader in rb_move_tail()
2762 if (rb_is_head_page(next_page, &tail_page->list)) { in rb_move_tail()
2768 if (!rb_is_reader_page(cpu_buffer->commit_page)) { in rb_move_tail()
2773 if (!(buffer->flags & RB_FL_OVERWRITE)) { in rb_move_tail()
2774 local_inc(&cpu_buffer->dropped_events); in rb_move_tail()
2796 if (unlikely((cpu_buffer->commit_page != in rb_move_tail()
2797 cpu_buffer->tail_page) && in rb_move_tail()
2798 (cpu_buffer->commit_page == in rb_move_tail()
2799 cpu_buffer->reader_page))) { in rb_move_tail()
2800 local_inc(&cpu_buffer->commit_overrun); in rb_move_tail()
2815 local_inc(&cpu_buffer->committing); in rb_move_tail()
2818 return ERR_PTR(-EAGAIN); in rb_move_tail()
2832 event->type_len = RINGBUF_TYPE_TIME_STAMP; in rb_add_time_stamp()
2834 event->type_len = RINGBUF_TYPE_TIME_EXTEND; in rb_add_time_stamp()
2838 event->time_delta = delta & TS_MASK; in rb_add_time_stamp()
2839 event->array[0] = delta >> TS_SHIFT; in rb_add_time_stamp()
2842 event->time_delta = 0; in rb_add_time_stamp()
2843 event->array[0] = 0; in rb_add_time_stamp()
2863 (unsigned long long)info->delta, in rb_check_timestamp()
2864 (unsigned long long)info->ts, in rb_check_timestamp()
2865 (unsigned long long)info->before, in rb_check_timestamp()
2866 (unsigned long long)info->after, in rb_check_timestamp()
2867 (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0), in rb_check_timestamp()
2881 bool abs = info->add_timestamp & in rb_add_timestamp()
2884 if (unlikely(info->delta > (1ULL << 59))) { in rb_add_timestamp()
2889 if (abs && (info->ts & TS_MSB)) { in rb_add_timestamp()
2890 info->delta &= ABS_TS_MASK; in rb_add_timestamp()
2893 } else if (info->before == info->after && info->before > info->ts) { in rb_add_timestamp()
2903 pr_warn("Ring buffer clock went backwards: %llu -> %llu\n", in rb_add_timestamp()
2904 info->before, info->ts); in rb_add_timestamp()
2909 info->delta = 0; in rb_add_timestamp()
2911 *event = rb_add_time_stamp(*event, info->delta, abs); in rb_add_timestamp()
2912 *length -= RB_LEN_TIME_EXTEND; in rb_add_timestamp()
2917 * rb_update_event - update event type and data
2918 * @cpu_buffer: The per cpu buffer of the @event
2932 unsigned length = info->length; in rb_update_event()
2933 u64 delta = info->delta; in rb_update_event()
2934 unsigned int nest = local_read(&cpu_buffer->committing) - 1; in rb_update_event()
2937 cpu_buffer->event_stamp[nest] = info->ts; in rb_update_event()
2943 if (unlikely(info->add_timestamp)) in rb_update_event()
2946 event->time_delta = delta; in rb_update_event()
2947 length -= RB_EVNT_HDR_SIZE; in rb_update_event()
2949 event->type_len = 0; in rb_update_event()
2950 event->array[0] = length; in rb_update_event()
2952 event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); in rb_update_event()
2989 switch (event->type_len) { in rb_time_delta()
3000 return event->time_delta; in rb_time_delta()
3022 bpage = READ_ONCE(cpu_buffer->tail_page); in rb_try_to_discard()
3026 if (!rb_time_read(&cpu_buffer->write_stamp, &write_stamp)) in rb_try_to_discard()
3029 /* Make sure the write stamp is read before testing the location */ in rb_try_to_discard()
3032 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { in rb_try_to_discard()
3034 local_read(&bpage->write) & ~RB_WRITE_MASK; in rb_try_to_discard()
3038 if (!rb_time_cmpxchg(&cpu_buffer->write_stamp, in rb_try_to_discard()
3039 write_stamp, write_stamp - delta)) in rb_try_to_discard()
3051 rb_time_set(&cpu_buffer->before_stamp, 0); in rb_try_to_discard()
3069 index = local_cmpxchg(&bpage->write, old_index, new_index); in rb_try_to_discard()
3072 local_sub(event_length, &cpu_buffer->entries_bytes); in rb_try_to_discard()
3083 local_inc(&cpu_buffer->committing); in rb_start_commit()
3084 local_inc(&cpu_buffer->commits); in rb_start_commit()
3093 * We only race with interrupts and NMIs on this CPU. in rb_set_commit_to_write()
3101 max_count = cpu_buffer->nr_pages * 100; in rb_set_commit_to_write()
3103 while (cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)) { in rb_set_commit_to_write()
3104 if (RB_WARN_ON(cpu_buffer, !(--max_count))) in rb_set_commit_to_write()
3107 rb_is_reader_page(cpu_buffer->tail_page))) in rb_set_commit_to_write()
3109 local_set(&cpu_buffer->commit_page->page->commit, in rb_set_commit_to_write()
3110 rb_page_write(cpu_buffer->commit_page)); in rb_set_commit_to_write()
3111 rb_inc_page(&cpu_buffer->commit_page); in rb_set_commit_to_write()
3116 rb_page_write(cpu_buffer->commit_page)) { in rb_set_commit_to_write()
3118 local_set(&cpu_buffer->commit_page->page->commit, in rb_set_commit_to_write()
3119 rb_page_write(cpu_buffer->commit_page)); in rb_set_commit_to_write()
3121 local_read(&cpu_buffer->commit_page->page->commit) & in rb_set_commit_to_write()
3134 if (unlikely(cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page))) in rb_set_commit_to_write()
3143 !local_read(&cpu_buffer->committing))) in rb_end_commit()
3147 commits = local_read(&cpu_buffer->commits); in rb_end_commit()
3150 if (local_read(&cpu_buffer->committing) == 1) in rb_end_commit()
3153 local_dec(&cpu_buffer->committing); in rb_end_commit()
3163 if (unlikely(local_read(&cpu_buffer->commits) != commits) && in rb_end_commit()
3164 !local_read(&cpu_buffer->committing)) { in rb_end_commit()
3165 local_inc(&cpu_buffer->committing); in rb_end_commit()
3176 event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE; in rb_event_discard()
3177 event->type_len = RINGBUF_TYPE_PADDING; in rb_event_discard()
3179 if (!event->time_delta) in rb_event_discard()
3180 event->time_delta = 1; in rb_event_discard()
3186 local_inc(&cpu_buffer->entries); in rb_commit()
3193 if (buffer->irq_work.waiters_pending) { in rb_wakeups()
3194 buffer->irq_work.waiters_pending = false; in rb_wakeups()
3196 irq_work_queue(&buffer->irq_work.work); in rb_wakeups()
3199 if (cpu_buffer->irq_work.waiters_pending) { in rb_wakeups()
3200 cpu_buffer->irq_work.waiters_pending = false; in rb_wakeups()
3202 irq_work_queue(&cpu_buffer->irq_work.work); in rb_wakeups()
3205 if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched)) in rb_wakeups()
3208 if (cpu_buffer->reader_page == cpu_buffer->commit_page) in rb_wakeups()
3211 if (!cpu_buffer->irq_work.full_waiters_pending) in rb_wakeups()
3214 cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched); in rb_wakeups()
3216 if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full)) in rb_wakeups()
3219 cpu_buffer->irq_work.wakeup_full = true; in rb_wakeups()
3220 cpu_buffer->irq_work.full_waiters_pending = false; in rb_wakeups()
3222 irq_work_queue(&cpu_buffer->irq_work.work); in rb_wakeups()
3259 * 101 - 1 = 100
3262 * 1010 - 1 = 1001
3297 unsigned int val = cpu_buffer->current_context; in trace_recursive_lock()
3300 bit = RB_CTX_NORMAL - bit; in trace_recursive_lock()
3302 if (unlikely(val & (1 << (bit + cpu_buffer->nest)))) { in trace_recursive_lock()
3309 if (val & (1 << (bit + cpu_buffer->nest))) { in trace_recursive_lock()
3315 val |= (1 << (bit + cpu_buffer->nest)); in trace_recursive_lock()
3316 cpu_buffer->current_context = val; in trace_recursive_lock()
3324 cpu_buffer->current_context &= in trace_recursive_unlock()
3325 cpu_buffer->current_context - (1 << cpu_buffer->nest); in trace_recursive_unlock()
3332 * ring_buffer_nest_start - Allow to trace while nested
3347 int cpu; in ring_buffer_nest_start() local
3351 cpu = raw_smp_processor_id(); in ring_buffer_nest_start()
3352 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_start()
3354 cpu_buffer->nest += NESTED_BITS; in ring_buffer_nest_start()
3358 * ring_buffer_nest_end - Allow to trace while nested
3367 int cpu; in ring_buffer_nest_end() local
3370 cpu = raw_smp_processor_id(); in ring_buffer_nest_end()
3371 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_end()
3373 cpu_buffer->nest -= NESTED_BITS; in ring_buffer_nest_end()
3378 * ring_buffer_unlock_commit - commit a reserved
3390 int cpu = raw_smp_processor_id(); in ring_buffer_unlock_commit() local
3392 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_unlock_commit()
3418 ts = bpage->time_stamp; in dump_buffer_page()
3423 event = (struct ring_buffer_event *)(bpage->data + e); in dump_buffer_page()
3425 switch (event->type_len) { in dump_buffer_page()
3440 ts += event->time_delta; in dump_buffer_page()
3441 pr_warn(" [%lld] delta:%d PADDING\n", ts, event->time_delta); in dump_buffer_page()
3445 ts += event->time_delta; in dump_buffer_page()
3446 pr_warn(" [%lld] delta:%d\n", ts, event->time_delta); in dump_buffer_page()
3472 bpage = info->tail_page->page; in check_buffer()
3476 tail = local_read(&bpage->commit); in check_buffer()
3477 } else if (info->add_timestamp & in check_buffer()
3487 if (tail <= 8 || tail > local_read(&bpage->commit)) in check_buffer()
3496 ts = bpage->time_stamp; in check_buffer()
3500 event = (struct ring_buffer_event *)(bpage->data + e); in check_buffer()
3502 switch (event->type_len) { in check_buffer()
3515 if (event->time_delta == 1) in check_buffer()
3519 ts += event->time_delta; in check_buffer()
3526 if ((full && ts > info->ts) || in check_buffer()
3527 (!full && ts + info->delta != info->ts)) { in check_buffer()
3533 atomic_inc(&cpu_buffer->record_disabled); in check_buffer()
3536 …pr_warn("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%… in check_buffer()
3537 cpu_buffer->cpu, in check_buffer()
3538 ts + info->delta, info->ts, info->delta, in check_buffer()
3539 info->before, info->after, in check_buffer()
3543 /* Do not re-enable checking */ in check_buffer()
3567 /* Don't let the compiler play games with cpu_buffer->tail_page */ in __rb_reserve_next()
3568 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page); in __rb_reserve_next()
3570 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK; in __rb_reserve_next()
3572 b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before); in __rb_reserve_next()
3573 a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after); in __rb_reserve_next()
3575 info->ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
3577 if ((info->add_timestamp & RB_ADD_STAMP_ABSOLUTE)) { in __rb_reserve_next()
3578 info->delta = info->ts; in __rb_reserve_next()
3585 if (unlikely(!a_ok || !b_ok || (info->before != info->after && w))) { in __rb_reserve_next()
3586 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND; in __rb_reserve_next()
3587 info->length += RB_LEN_TIME_EXTEND; in __rb_reserve_next()
3589 info->delta = info->ts - info->after; in __rb_reserve_next()
3590 if (unlikely(test_time_stamp(info->delta))) { in __rb_reserve_next()
3591 info->add_timestamp |= RB_ADD_STAMP_EXTEND; in __rb_reserve_next()
3592 info->length += RB_LEN_TIME_EXTEND; in __rb_reserve_next()
3597 /*B*/ rb_time_set(&cpu_buffer->before_stamp, info->ts); in __rb_reserve_next()
3599 /*C*/ write = local_add_return(info->length, &tail_page->write); in __rb_reserve_next()
3604 tail = write - info->length; in __rb_reserve_next()
3609 b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before); in __rb_reserve_next()
3610 a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after); in __rb_reserve_next()
3611 if (a_ok && b_ok && info->before != info->after) in __rb_reserve_next()
3612 (void)rb_time_cmpxchg(&cpu_buffer->before_stamp, in __rb_reserve_next()
3613 info->before, info->after); in __rb_reserve_next()
3624 /*D*/ rb_time_set(&cpu_buffer->write_stamp, info->ts); in __rb_reserve_next()
3626 /*E*/ s_ok = rb_time_read(&cpu_buffer->before_stamp, &save_before); in __rb_reserve_next()
3628 if (likely(!(info->add_timestamp & in __rb_reserve_next()
3631 info->delta = info->ts - info->after; in __rb_reserve_next()
3634 info->delta = info->ts; in __rb_reserve_next()
3637 if (unlikely(info->ts != save_before)) { in __rb_reserve_next()
3638 /* SLOW PATH - Interrupted between C and E */ in __rb_reserve_next()
3640 a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after); in __rb_reserve_next()
3644 if (save_before > info->after) { in __rb_reserve_next()
3649 (void)rb_time_cmpxchg(&cpu_buffer->write_stamp, in __rb_reserve_next()
3650 info->after, save_before); in __rb_reserve_next()
3655 /* SLOW PATH - Interrupted between A and C */ in __rb_reserve_next()
3656 a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after); in __rb_reserve_next()
3659 ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
3661 /*E*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) && in __rb_reserve_next()
3662 info->after < ts && in __rb_reserve_next()
3663 rb_time_cmpxchg(&cpu_buffer->write_stamp, in __rb_reserve_next()
3664 info->after, ts)) { in __rb_reserve_next()
3666 info->delta = ts - info->after; in __rb_reserve_next()
3676 info->delta = 0; in __rb_reserve_next()
3678 info->ts = ts; in __rb_reserve_next()
3679 info->add_timestamp &= ~RB_ADD_STAMP_FORCE; in __rb_reserve_next()
3686 if (unlikely(!tail && !(info->add_timestamp & in __rb_reserve_next()
3688 info->delta = 0; in __rb_reserve_next()
3695 local_inc(&tail_page->entries); in __rb_reserve_next()
3702 tail_page->page->time_stamp = info->ts; in __rb_reserve_next()
3705 local_add(info->length, &cpu_buffer->entries_bytes); in __rb_reserve_next()
3725 * Due to the ability to swap a cpu buffer from a buffer in rb_reserve_next_event()
3731 if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) { in rb_reserve_next_event()
3732 local_dec(&cpu_buffer->committing); in rb_reserve_next_event()
3733 local_dec(&cpu_buffer->commits); in rb_reserve_next_event()
3740 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { in rb_reserve_next_event()
3765 if (unlikely(PTR_ERR(event) == -EAGAIN)) { in rb_reserve_next_event()
3767 info.length -= RB_LEN_TIME_EXTEND; in rb_reserve_next_event()
3779 * ring_buffer_lock_reserve - reserve a part of the buffer
3798 int cpu; in ring_buffer_lock_reserve() local
3803 if (unlikely(atomic_read(&buffer->record_disabled))) in ring_buffer_lock_reserve()
3806 cpu = raw_smp_processor_id(); in ring_buffer_lock_reserve()
3808 if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask))) in ring_buffer_lock_reserve()
3811 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_lock_reserve()
3813 if (unlikely(atomic_read(&cpu_buffer->record_disabled))) in ring_buffer_lock_reserve()
3847 struct buffer_page *bpage = cpu_buffer->commit_page; in rb_decrement_entry()
3853 if (likely(bpage->page == (void *)addr)) { in rb_decrement_entry()
3854 local_dec(&bpage->entries); in rb_decrement_entry()
3865 if (bpage->page == (void *)addr) { in rb_decrement_entry()
3866 local_dec(&bpage->entries); in rb_decrement_entry()
3877 * ring_buffer_discard_commit - discard an event that has not been committed
3883 * and then that event will not be read later.
3899 int cpu; in ring_buffer_discard_commit() local
3904 cpu = smp_processor_id(); in ring_buffer_discard_commit()
3905 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_discard_commit()
3912 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); in ring_buffer_discard_commit()
3929 * ring_buffer_write - write data to the buffer without reserving
3948 int ret = -EBUSY; in ring_buffer_write()
3949 int cpu; in ring_buffer_write() local
3953 if (atomic_read(&buffer->record_disabled)) in ring_buffer_write()
3956 cpu = raw_smp_processor_id(); in ring_buffer_write()
3958 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_write()
3961 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_write()
3963 if (atomic_read(&cpu_buffer->record_disabled)) in ring_buffer_write()
3998 struct buffer_page *reader = cpu_buffer->reader_page; in rb_per_cpu_empty()
4000 struct buffer_page *commit = cpu_buffer->commit_page; in rb_per_cpu_empty()
4007 if (reader->read != rb_page_commit(reader)) in rb_per_cpu_empty()
4012 * committed content has been read, the ring buffer is empty. in rb_per_cpu_empty()
4019 * and head page, there should always be content to read. in rb_per_cpu_empty()
4027 * swap reader page with head page when it is to read data. in rb_per_cpu_empty()
4033 * ring_buffer_record_disable - stop all writes into the buffer
4043 atomic_inc(&buffer->record_disabled); in ring_buffer_record_disable()
4048 * ring_buffer_record_enable - enable writes to the buffer
4056 atomic_dec(&buffer->record_disabled); in ring_buffer_record_enable()
4061 * ring_buffer_record_off - stop all writes into the buffer
4077 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_off()
4079 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd); in ring_buffer_record_off()
4084 * ring_buffer_record_on - restart writes into the buffer
4100 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_on()
4102 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd); in ring_buffer_record_on()
4107 * ring_buffer_record_is_on - return true if the ring buffer can write
4114 return !atomic_read(&buffer->record_disabled); in ring_buffer_record_is_on()
4118 * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
4130 return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF); in ring_buffer_record_is_set_on()
4134 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
4136 * @cpu: The CPU buffer to stop
4143 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_disable_cpu() argument
4147 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_disable_cpu()
4150 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_disable_cpu()
4151 atomic_inc(&cpu_buffer->record_disabled); in ring_buffer_record_disable_cpu()
4156 * ring_buffer_record_enable_cpu - enable writes to the buffer
4158 * @cpu: The CPU to enable.
4163 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_enable_cpu() argument
4167 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_enable_cpu()
4170 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_enable_cpu()
4171 atomic_dec(&cpu_buffer->record_disabled); in ring_buffer_record_enable_cpu()
4178 * the entries read from the ring buffer and the number of
4184 return local_read(&cpu_buffer->entries) - in rb_num_of_entries()
4185 (local_read(&cpu_buffer->overrun) + cpu_buffer->read); in rb_num_of_entries()
4189 * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
4191 * @cpu: The per CPU buffer to read from.
4193 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu) in ring_buffer_oldest_event_ts() argument
4200 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_oldest_event_ts()
4203 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_oldest_event_ts()
4204 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_oldest_event_ts()
4209 if (cpu_buffer->tail_page == cpu_buffer->reader_page) in ring_buffer_oldest_event_ts()
4210 bpage = cpu_buffer->reader_page; in ring_buffer_oldest_event_ts()
4214 ret = bpage->page->time_stamp; in ring_buffer_oldest_event_ts()
4215 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_oldest_event_ts()
4222 * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer
4224 * @cpu: The per CPU buffer to read from.
4226 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_bytes_cpu() argument
4231 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_bytes_cpu()
4234 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_bytes_cpu()
4235 ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes; in ring_buffer_bytes_cpu()
4242 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
4244 * @cpu: The per CPU buffer to get the entries from.
4246 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_entries_cpu() argument
4250 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_entries_cpu()
4253 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries_cpu()
4260 * ring_buffer_overrun_cpu - get the number of overruns caused by the ring
4263 * @cpu: The per CPU buffer to get the number of overruns from
4265 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_overrun_cpu() argument
4270 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_overrun_cpu()
4273 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overrun_cpu()
4274 ret = local_read(&cpu_buffer->overrun); in ring_buffer_overrun_cpu()
4281 * ring_buffer_commit_overrun_cpu - get the number of overruns caused by
4285 * @cpu: The per CPU buffer to get the number of overruns from
4288 ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_commit_overrun_cpu() argument
4293 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_commit_overrun_cpu()
4296 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_commit_overrun_cpu()
4297 ret = local_read(&cpu_buffer->commit_overrun); in ring_buffer_commit_overrun_cpu()
4304 * ring_buffer_dropped_events_cpu - get the number of dropped events caused by
4307 * @cpu: The per CPU buffer to get the number of overruns from
4310 ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_dropped_events_cpu() argument
4315 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_dropped_events_cpu()
4318 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_dropped_events_cpu()
4319 ret = local_read(&cpu_buffer->dropped_events); in ring_buffer_dropped_events_cpu()
4326 * ring_buffer_read_events_cpu - get the number of events successfully read
4328 * @cpu: The per CPU buffer to get the number of events read
4331 ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_read_events_cpu() argument
4335 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_events_cpu()
4338 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_events_cpu()
4339 return cpu_buffer->read; in ring_buffer_read_events_cpu()
4344 * ring_buffer_entries - get the number of entries in a buffer
4348 * (all CPU entries)
4354 int cpu; in ring_buffer_entries() local
4357 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_entries()
4358 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries()
4367 * ring_buffer_overruns - get the number of overruns in buffer
4371 * (all CPU entries)
4377 int cpu; in ring_buffer_overruns() local
4380 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_overruns()
4381 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overruns()
4382 overruns += local_read(&cpu_buffer->overrun); in ring_buffer_overruns()
4391 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; in rb_iter_reset()
4394 iter->head_page = cpu_buffer->reader_page; in rb_iter_reset()
4395 iter->head = cpu_buffer->reader_page->read; in rb_iter_reset()
4396 iter->next_event = iter->head; in rb_iter_reset()
4398 iter->cache_reader_page = iter->head_page; in rb_iter_reset()
4399 iter->cache_read = cpu_buffer->read; in rb_iter_reset()
4401 if (iter->head) { in rb_iter_reset()
4402 iter->read_stamp = cpu_buffer->read_stamp; in rb_iter_reset()
4403 iter->page_stamp = cpu_buffer->reader_page->page->time_stamp; in rb_iter_reset()
4405 iter->read_stamp = iter->head_page->page->time_stamp; in rb_iter_reset()
4406 iter->page_stamp = iter->read_stamp; in rb_iter_reset()
4411 * ring_buffer_iter_reset - reset an iterator
4425 cpu_buffer = iter->cpu_buffer; in ring_buffer_iter_reset()
4427 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_iter_reset()
4429 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_iter_reset()
4434 * ring_buffer_iter_empty - check if an iterator has no more to read
4448 cpu_buffer = iter->cpu_buffer; in ring_buffer_iter_empty()
4449 reader = cpu_buffer->reader_page; in ring_buffer_iter_empty()
4450 head_page = cpu_buffer->head_page; in ring_buffer_iter_empty()
4451 commit_page = cpu_buffer->commit_page; in ring_buffer_iter_empty()
4452 commit_ts = commit_page->page->time_stamp; in ring_buffer_iter_empty()
4465 curr_commit_page = READ_ONCE(cpu_buffer->commit_page); in ring_buffer_iter_empty()
4466 curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp); in ring_buffer_iter_empty()
4474 return ((iter->head_page == commit_page && iter->head >= commit) || in ring_buffer_iter_empty()
4475 (iter->head_page == reader && commit_page == head_page && in ring_buffer_iter_empty()
4476 head_page->read == commit && in ring_buffer_iter_empty()
4477 iter->head == rb_page_commit(cpu_buffer->reader_page))); in ring_buffer_iter_empty()
4487 switch (event->type_len) { in rb_update_read_stamp()
4493 cpu_buffer->read_stamp += delta; in rb_update_read_stamp()
4498 delta = rb_fix_abs_ts(delta, cpu_buffer->read_stamp); in rb_update_read_stamp()
4499 cpu_buffer->read_stamp = delta; in rb_update_read_stamp()
4503 cpu_buffer->read_stamp += event->time_delta; in rb_update_read_stamp()
4518 switch (event->type_len) { in rb_update_iter_read_stamp()
4524 iter->read_stamp += delta; in rb_update_iter_read_stamp()
4529 delta = rb_fix_abs_ts(delta, iter->read_stamp); in rb_update_iter_read_stamp()
4530 iter->read_stamp = delta; in rb_update_iter_read_stamp()
4534 iter->read_stamp += event->time_delta; in rb_update_iter_read_stamp()
4538 RB_WARN_ON(iter->cpu_buffer, 1); in rb_update_iter_read_stamp()
4553 arch_spin_lock(&cpu_buffer->lock); in rb_get_reader_page()
4567 reader = cpu_buffer->reader_page; in rb_get_reader_page()
4569 /* If there's more to read, return this page */ in rb_get_reader_page()
4570 if (cpu_buffer->reader_page->read < rb_page_size(reader)) in rb_get_reader_page()
4575 cpu_buffer->reader_page->read > rb_page_size(reader))) in rb_get_reader_page()
4580 if (cpu_buffer->commit_page == cpu_buffer->reader_page) in rb_get_reader_page()
4590 local_set(&cpu_buffer->reader_page->write, 0); in rb_get_reader_page()
4591 local_set(&cpu_buffer->reader_page->entries, 0); in rb_get_reader_page()
4592 local_set(&cpu_buffer->reader_page->page->commit, 0); in rb_get_reader_page()
4593 cpu_buffer->reader_page->real_end = 0; in rb_get_reader_page()
4602 cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next); in rb_get_reader_page()
4603 cpu_buffer->reader_page->list.prev = reader->list.prev; in rb_get_reader_page()
4606 * cpu_buffer->pages just needs to point to the buffer, it in rb_get_reader_page()
4610 cpu_buffer->pages = reader->list.prev; in rb_get_reader_page()
4613 rb_set_list_to_head(&cpu_buffer->reader_page->list); in rb_get_reader_page()
4616 * We want to make sure we read the overruns after we set up our in rb_get_reader_page()
4625 overwrite = local_read(&(cpu_buffer->overrun)); in rb_get_reader_page()
4638 ret = rb_head_page_replace(reader, cpu_buffer->reader_page); in rb_get_reader_page()
4651 rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list; in rb_get_reader_page()
4652 rb_inc_page(&cpu_buffer->head_page); in rb_get_reader_page()
4654 local_inc(&cpu_buffer->pages_read); in rb_get_reader_page()
4657 cpu_buffer->reader_page = reader; in rb_get_reader_page()
4658 cpu_buffer->reader_page->read = 0; in rb_get_reader_page()
4660 if (overwrite != cpu_buffer->last_overrun) { in rb_get_reader_page()
4661 cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun; in rb_get_reader_page()
4662 cpu_buffer->last_overrun = overwrite; in rb_get_reader_page()
4669 if (reader && reader->read == 0) in rb_get_reader_page()
4670 cpu_buffer->read_stamp = reader->page->time_stamp; in rb_get_reader_page()
4672 arch_spin_unlock(&cpu_buffer->lock); in rb_get_reader_page()
4719 if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX) in rb_advance_reader()
4720 cpu_buffer->read++; in rb_advance_reader()
4725 cpu_buffer->reader_page->read += length; in rb_advance_reader()
4732 cpu_buffer = iter->cpu_buffer; in rb_advance_iter()
4735 if (iter->head == iter->next_event) { in rb_advance_iter()
4741 iter->head = iter->next_event; in rb_advance_iter()
4746 if (iter->next_event >= rb_page_size(iter->head_page)) { in rb_advance_iter()
4748 if (iter->head_page == cpu_buffer->commit_page) in rb_advance_iter()
4754 rb_update_iter_read_stamp(iter, iter->event); in rb_advance_iter()
4759 return cpu_buffer->lost_events; in rb_lost_events()
4788 switch (event->type_len) { in rb_buffer_peek()
4810 *ts = rb_fix_abs_ts(*ts, reader->page->time_stamp); in rb_buffer_peek()
4811 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
4812 cpu_buffer->cpu, ts); in rb_buffer_peek()
4820 *ts = cpu_buffer->read_stamp + event->time_delta; in rb_buffer_peek()
4821 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
4822 cpu_buffer->cpu, ts); in rb_buffer_peek()
4847 cpu_buffer = iter->cpu_buffer; in rb_iter_peek()
4848 buffer = cpu_buffer->buffer; in rb_iter_peek()
4851 * Check if someone performed a consuming read to in rb_iter_peek()
4852 * the buffer. A consuming read invalidates the iterator in rb_iter_peek()
4855 if (unlikely(iter->cache_read != cpu_buffer->read || in rb_iter_peek()
4856 iter->cache_reader_page != cpu_buffer->reader_page)) in rb_iter_peek()
4865 * to read, just give up if we fail to get an event after in rb_iter_peek()
4876 if (iter->head >= rb_page_size(iter->head_page)) { in rb_iter_peek()
4885 switch (event->type_len) { in rb_iter_peek()
4902 *ts = rb_fix_abs_ts(*ts, iter->head_page->page->time_stamp); in rb_iter_peek()
4903 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_iter_peek()
4904 cpu_buffer->cpu, ts); in rb_iter_peek()
4912 *ts = iter->read_stamp + event->time_delta; in rb_iter_peek()
4914 cpu_buffer->cpu, ts); in rb_iter_peek()
4929 raw_spin_lock(&cpu_buffer->reader_lock); in rb_reader_lock()
4938 * to do the read, but this can corrupt the ring buffer, in rb_reader_lock()
4942 if (raw_spin_trylock(&cpu_buffer->reader_lock)) in rb_reader_lock()
4946 atomic_inc(&cpu_buffer->record_disabled); in rb_reader_lock()
4954 raw_spin_unlock(&cpu_buffer->reader_lock); in rb_reader_unlock()
4959 * ring_buffer_peek - peek at the next event to be read
4960 * @buffer: The ring buffer to read
4961 * @cpu: The cpu to peak at
4965 * This will return the event that will be read next, but does
4969 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_peek() argument
4972 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_peek()
4977 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_peek()
4984 if (event && event->type_len == RINGBUF_TYPE_PADDING) in ring_buffer_peek()
4989 if (event && event->type_len == RINGBUF_TYPE_PADDING) in ring_buffer_peek()
4995 /** ring_buffer_iter_dropped - report if there are dropped events
5002 bool ret = iter->missed_events != 0; in ring_buffer_iter_dropped()
5004 iter->missed_events = 0; in ring_buffer_iter_dropped()
5010 * ring_buffer_iter_peek - peek at the next event to be read
5014 * This will return the event that will be read next, but does
5020 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; in ring_buffer_iter_peek()
5025 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_iter_peek()
5027 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_iter_peek()
5029 if (event && event->type_len == RINGBUF_TYPE_PADDING) in ring_buffer_iter_peek()
5036 * ring_buffer_consume - return an event and consume it
5038 * @cpu: the cpu to read the buffer from
5047 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_consume() argument
5059 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_consume()
5062 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_consume()
5068 cpu_buffer->lost_events = 0; in ring_buffer_consume()
5078 if (event && event->type_len == RINGBUF_TYPE_PADDING) in ring_buffer_consume()
5086 * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
5087 * @buffer: The ring buffer to read from
5088 * @cpu: The cpu buffer to iterate over
5096 * corrupted. This is not a consuming read, so a producer is not
5107 ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags) in ring_buffer_read_prepare() argument
5112 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_prepare()
5119 iter->event = kmalloc(BUF_MAX_DATA_SIZE, flags); in ring_buffer_read_prepare()
5120 if (!iter->event) { in ring_buffer_read_prepare()
5125 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_prepare()
5127 iter->cpu_buffer = cpu_buffer; in ring_buffer_read_prepare()
5129 atomic_inc(&cpu_buffer->resize_disabled); in ring_buffer_read_prepare()
5136 * ring_buffer_read_prepare_sync - Synchronize a set of prepare calls
5150 * ring_buffer_read_start - start a non consuming read of the buffer
5169 cpu_buffer = iter->cpu_buffer; in ring_buffer_read_start()
5171 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_read_start()
5172 arch_spin_lock(&cpu_buffer->lock); in ring_buffer_read_start()
5174 arch_spin_unlock(&cpu_buffer->lock); in ring_buffer_read_start()
5175 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_read_start()
5180 * ring_buffer_read_finish - finish reading the iterator of the buffer
5183 * This re-enables the recording to the buffer, and frees the
5189 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; in ring_buffer_read_finish()
5195 * Must prevent readers from trying to read, as the check in ring_buffer_read_finish()
5198 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_read_finish()
5200 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_read_finish()
5202 atomic_dec(&cpu_buffer->resize_disabled); in ring_buffer_read_finish()
5203 kfree(iter->event); in ring_buffer_read_finish()
5209 * ring_buffer_iter_advance - advance the iterator to the next location
5212 * Move the location of the iterator such that the next read will
5217 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; in ring_buffer_iter_advance()
5220 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_iter_advance()
5224 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_iter_advance()
5229 * ring_buffer_size - return the size of the ring buffer (in bytes)
5231 * @cpu: The CPU to get ring buffer size from.
5233 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) in ring_buffer_size() argument
5237 * BUF_PAGE_SIZE * buffer->nr_pages in ring_buffer_size()
5239 * return the per cpu buffer value. in ring_buffer_size()
5241 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_size()
5244 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages; in ring_buffer_size()
5253 cpu_buffer->head_page in rb_reset_cpu()
5254 = list_entry(cpu_buffer->pages, struct buffer_page, list); in rb_reset_cpu()
5255 local_set(&cpu_buffer->head_page->write, 0); in rb_reset_cpu()
5256 local_set(&cpu_buffer->head_page->entries, 0); in rb_reset_cpu()
5257 local_set(&cpu_buffer->head_page->page->commit, 0); in rb_reset_cpu()
5259 cpu_buffer->head_page->read = 0; in rb_reset_cpu()
5261 cpu_buffer->tail_page = cpu_buffer->head_page; in rb_reset_cpu()
5262 cpu_buffer->commit_page = cpu_buffer->head_page; in rb_reset_cpu()
5264 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); in rb_reset_cpu()
5265 INIT_LIST_HEAD(&cpu_buffer->new_pages); in rb_reset_cpu()
5266 local_set(&cpu_buffer->reader_page->write, 0); in rb_reset_cpu()
5267 local_set(&cpu_buffer->reader_page->entries, 0); in rb_reset_cpu()
5268 local_set(&cpu_buffer->reader_page->page->commit, 0); in rb_reset_cpu()
5269 cpu_buffer->reader_page->read = 0; in rb_reset_cpu()
5271 local_set(&cpu_buffer->entries_bytes, 0); in rb_reset_cpu()
5272 local_set(&cpu_buffer->overrun, 0); in rb_reset_cpu()
5273 local_set(&cpu_buffer->commit_overrun, 0); in rb_reset_cpu()
5274 local_set(&cpu_buffer->dropped_events, 0); in rb_reset_cpu()
5275 local_set(&cpu_buffer->entries, 0); in rb_reset_cpu()
5276 local_set(&cpu_buffer->committing, 0); in rb_reset_cpu()
5277 local_set(&cpu_buffer->commits, 0); in rb_reset_cpu()
5278 local_set(&cpu_buffer->pages_touched, 0); in rb_reset_cpu()
5279 local_set(&cpu_buffer->pages_lost, 0); in rb_reset_cpu()
5280 local_set(&cpu_buffer->pages_read, 0); in rb_reset_cpu()
5281 cpu_buffer->last_pages_touch = 0; in rb_reset_cpu()
5282 cpu_buffer->shortest_full = 0; in rb_reset_cpu()
5283 cpu_buffer->read = 0; in rb_reset_cpu()
5284 cpu_buffer->read_bytes = 0; in rb_reset_cpu()
5286 rb_time_set(&cpu_buffer->write_stamp, 0); in rb_reset_cpu()
5287 rb_time_set(&cpu_buffer->before_stamp, 0); in rb_reset_cpu()
5289 memset(cpu_buffer->event_stamp, 0, sizeof(cpu_buffer->event_stamp)); in rb_reset_cpu()
5291 cpu_buffer->lost_events = 0; in rb_reset_cpu()
5292 cpu_buffer->last_overrun = 0; in rb_reset_cpu()
5297 /* Must have disabled the cpu buffer then done a synchronize_rcu */
5302 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in reset_disabled_cpu_buffer()
5304 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) in reset_disabled_cpu_buffer()
5307 arch_spin_lock(&cpu_buffer->lock); in reset_disabled_cpu_buffer()
5311 arch_spin_unlock(&cpu_buffer->lock); in reset_disabled_cpu_buffer()
5314 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in reset_disabled_cpu_buffer()
5318 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
5319 * @buffer: The ring buffer to reset a per cpu buffer of
5320 * @cpu: The CPU buffer to be reset
5322 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_reset_cpu() argument
5324 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_cpu()
5326 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_reset_cpu()
5330 mutex_lock(&buffer->mutex); in ring_buffer_reset_cpu()
5332 atomic_inc(&cpu_buffer->resize_disabled); in ring_buffer_reset_cpu()
5333 atomic_inc(&cpu_buffer->record_disabled); in ring_buffer_reset_cpu()
5340 atomic_dec(&cpu_buffer->record_disabled); in ring_buffer_reset_cpu()
5341 atomic_dec(&cpu_buffer->resize_disabled); in ring_buffer_reset_cpu()
5343 mutex_unlock(&buffer->mutex); in ring_buffer_reset_cpu()
5348 * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
5349 * @buffer: The ring buffer to reset a per cpu buffer of
5350 * @cpu: The CPU buffer to be reset
5355 int cpu; in ring_buffer_reset_online_cpus() local
5358 mutex_lock(&buffer->mutex); in ring_buffer_reset_online_cpus()
5360 for_each_online_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
5361 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
5363 atomic_inc(&cpu_buffer->resize_disabled); in ring_buffer_reset_online_cpus()
5364 atomic_inc(&cpu_buffer->record_disabled); in ring_buffer_reset_online_cpus()
5370 for_each_online_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
5371 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
5375 atomic_dec(&cpu_buffer->record_disabled); in ring_buffer_reset_online_cpus()
5376 atomic_dec(&cpu_buffer->resize_disabled); in ring_buffer_reset_online_cpus()
5379 mutex_unlock(&buffer->mutex); in ring_buffer_reset_online_cpus()
5383 * ring_buffer_reset - reset a ring buffer
5384 * @buffer: The ring buffer to reset all cpu buffers
5389 int cpu; in ring_buffer_reset() local
5392 mutex_lock(&buffer->mutex); in ring_buffer_reset()
5394 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
5395 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
5397 atomic_inc(&cpu_buffer->resize_disabled); in ring_buffer_reset()
5398 atomic_inc(&cpu_buffer->record_disabled); in ring_buffer_reset()
5404 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
5405 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
5409 atomic_dec(&cpu_buffer->record_disabled); in ring_buffer_reset()
5410 atomic_dec(&cpu_buffer->resize_disabled); in ring_buffer_reset()
5413 mutex_unlock(&buffer->mutex); in ring_buffer_reset()
5418 * ring_buffer_empty - is the ring buffer empty?
5426 int cpu; in ring_buffer_empty() local
5430 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_empty()
5431 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty()
5447 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
5449 * @cpu: The CPU buffer to test
5451 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_empty_cpu() argument
5458 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_empty_cpu()
5461 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty_cpu()
5474 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
5477 * @cpu: the CPU of the buffers to swap
5480 * of a CPU buffer and has another back up buffer lying around.
5481 * it is expected that the tracer handles the cpu buffer not being
5485 struct trace_buffer *buffer_b, int cpu) in ring_buffer_swap_cpu() argument
5489 int ret = -EINVAL; in ring_buffer_swap_cpu()
5491 if (!cpumask_test_cpu(cpu, buffer_a->cpumask) || in ring_buffer_swap_cpu()
5492 !cpumask_test_cpu(cpu, buffer_b->cpumask)) in ring_buffer_swap_cpu()
5495 cpu_buffer_a = buffer_a->buffers[cpu]; in ring_buffer_swap_cpu()
5496 cpu_buffer_b = buffer_b->buffers[cpu]; in ring_buffer_swap_cpu()
5499 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages) in ring_buffer_swap_cpu()
5502 ret = -EAGAIN; in ring_buffer_swap_cpu()
5504 if (atomic_read(&buffer_a->record_disabled)) in ring_buffer_swap_cpu()
5507 if (atomic_read(&buffer_b->record_disabled)) in ring_buffer_swap_cpu()
5510 if (atomic_read(&cpu_buffer_a->record_disabled)) in ring_buffer_swap_cpu()
5513 if (atomic_read(&cpu_buffer_b->record_disabled)) in ring_buffer_swap_cpu()
5519 * Normally this will be called from the same CPU as cpu. in ring_buffer_swap_cpu()
5522 atomic_inc(&cpu_buffer_a->record_disabled); in ring_buffer_swap_cpu()
5523 atomic_inc(&cpu_buffer_b->record_disabled); in ring_buffer_swap_cpu()
5525 ret = -EBUSY; in ring_buffer_swap_cpu()
5526 if (local_read(&cpu_buffer_a->committing)) in ring_buffer_swap_cpu()
5528 if (local_read(&cpu_buffer_b->committing)) in ring_buffer_swap_cpu()
5531 buffer_a->buffers[cpu] = cpu_buffer_b; in ring_buffer_swap_cpu()
5532 buffer_b->buffers[cpu] = cpu_buffer_a; in ring_buffer_swap_cpu()
5534 cpu_buffer_b->buffer = buffer_a; in ring_buffer_swap_cpu()
5535 cpu_buffer_a->buffer = buffer_b; in ring_buffer_swap_cpu()
5540 atomic_dec(&cpu_buffer_a->record_disabled); in ring_buffer_swap_cpu()
5541 atomic_dec(&cpu_buffer_b->record_disabled); in ring_buffer_swap_cpu()
5549 * ring_buffer_alloc_read_page - allocate a page to read from buffer
5551 * @cpu: the cpu buffer to allocate.
5559 * the page that was allocated, with the read page of the buffer.
5564 void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) in ring_buffer_alloc_read_page() argument
5571 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_alloc_read_page()
5572 return ERR_PTR(-ENODEV); in ring_buffer_alloc_read_page()
5574 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_alloc_read_page()
5576 arch_spin_lock(&cpu_buffer->lock); in ring_buffer_alloc_read_page()
5578 if (cpu_buffer->free_page) { in ring_buffer_alloc_read_page()
5579 bpage = cpu_buffer->free_page; in ring_buffer_alloc_read_page()
5580 cpu_buffer->free_page = NULL; in ring_buffer_alloc_read_page()
5583 arch_spin_unlock(&cpu_buffer->lock); in ring_buffer_alloc_read_page()
5589 page = alloc_pages_node(cpu_to_node(cpu), in ring_buffer_alloc_read_page()
5592 return ERR_PTR(-ENOMEM); in ring_buffer_alloc_read_page()
5604 * ring_buffer_free_read_page - free an allocated read page
5606 * @cpu: the cpu buffer the page came from
5611 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data) in ring_buffer_free_read_page() argument
5613 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_free_read_page()
5623 arch_spin_lock(&cpu_buffer->lock); in ring_buffer_free_read_page()
5625 if (!cpu_buffer->free_page) { in ring_buffer_free_read_page()
5626 cpu_buffer->free_page = bpage; in ring_buffer_free_read_page()
5630 arch_spin_unlock(&cpu_buffer->lock); in ring_buffer_free_read_page()
5639 * ring_buffer_read_page - extract a page from the ring buffer
5643 * @cpu: the cpu of the buffer to extract
5652 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
5655 * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
5672 void **data_page, size_t len, int cpu, int full) in ring_buffer_read_page() argument
5674 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_page()
5681 unsigned int read; in ring_buffer_read_page() local
5683 int ret = -1; in ring_buffer_read_page()
5685 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_page()
5695 len -= BUF_PAGE_HDR_SIZE; in ring_buffer_read_page()
5704 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); in ring_buffer_read_page()
5712 read = reader->read; in ring_buffer_read_page()
5716 missed_events = cpu_buffer->lost_events; in ring_buffer_read_page()
5719 * If this page has been partially read or in ring_buffer_read_page()
5720 * if len is not big enough to read the rest of the page or in ring_buffer_read_page()
5725 if (read || (len < (commit - read)) || in ring_buffer_read_page()
5726 cpu_buffer->reader_page == cpu_buffer->commit_page) { in ring_buffer_read_page()
5727 struct buffer_data_page *rpage = cpu_buffer->reader_page->page; in ring_buffer_read_page()
5728 unsigned int rpos = read; in ring_buffer_read_page()
5734 * if there's been a previous partial read and the in ring_buffer_read_page()
5735 * rest of the page can be read and the commit page is off in ring_buffer_read_page()
5739 (!read || (len < (commit - read)) || in ring_buffer_read_page()
5740 cpu_buffer->reader_page == cpu_buffer->commit_page)) in ring_buffer_read_page()
5743 if (len > (commit - read)) in ring_buffer_read_page()
5744 len = (commit - read); in ring_buffer_read_page()
5753 save_timestamp = cpu_buffer->read_stamp; in ring_buffer_read_page()
5764 memcpy(bpage->data + pos, rpage->data + rpos, size); in ring_buffer_read_page()
5766 len -= size; in ring_buffer_read_page()
5769 rpos = reader->read; in ring_buffer_read_page()
5781 local_set(&bpage->commit, pos); in ring_buffer_read_page()
5782 bpage->time_stamp = save_timestamp; in ring_buffer_read_page()
5785 read = 0; in ring_buffer_read_page()
5788 cpu_buffer->read += rb_page_entries(reader); in ring_buffer_read_page()
5789 cpu_buffer->read_bytes += BUF_PAGE_SIZE; in ring_buffer_read_page()
5793 bpage = reader->page; in ring_buffer_read_page()
5794 reader->page = *data_page; in ring_buffer_read_page()
5795 local_set(&reader->write, 0); in ring_buffer_read_page()
5796 local_set(&reader->entries, 0); in ring_buffer_read_page()
5797 reader->read = 0; in ring_buffer_read_page()
5805 if (reader->real_end) in ring_buffer_read_page()
5806 local_set(&bpage->commit, reader->real_end); in ring_buffer_read_page()
5808 ret = read; in ring_buffer_read_page()
5810 cpu_buffer->lost_events = 0; in ring_buffer_read_page()
5812 commit = local_read(&bpage->commit); in ring_buffer_read_page()
5820 if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) { in ring_buffer_read_page()
5821 memcpy(&bpage->data[commit], &missed_events, in ring_buffer_read_page()
5823 local_add(RB_MISSED_STORED, &bpage->commit); in ring_buffer_read_page()
5826 local_add(RB_MISSED_EVENTS, &bpage->commit); in ring_buffer_read_page()
5833 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); in ring_buffer_read_page()
5836 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); in ring_buffer_read_page()
5844 * We only allocate new buffers, never free them if the CPU goes down.
5848 int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) in trace_rb_cpu_prepare() argument
5856 if (cpumask_test_cpu(cpu, buffer->cpumask)) in trace_rb_cpu_prepare()
5861 /* check if all cpu sizes are same */ in trace_rb_cpu_prepare()
5863 /* fill in the size from first enabled cpu */ in trace_rb_cpu_prepare()
5865 nr_pages = buffer->buffers[cpu_i]->nr_pages; in trace_rb_cpu_prepare()
5866 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { in trace_rb_cpu_prepare()
5874 buffer->buffers[cpu] = in trace_rb_cpu_prepare()
5875 rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in trace_rb_cpu_prepare()
5876 if (!buffer->buffers[cpu]) { in trace_rb_cpu_prepare()
5877 WARN(1, "failed to allocate ring buffer on CPU %u\n", in trace_rb_cpu_prepare()
5878 cpu); in trace_rb_cpu_prepare()
5879 return -ENOMEM; in trace_rb_cpu_prepare()
5882 cpumask_set_cpu(cpu, buffer->cpumask); in trace_rb_cpu_prepare()
5890 * It will kick off a thread per CPU that will go into a loop
5891 * writing to the per cpu ring buffer various sizes of data.
5918 int cpu; member
5924 /* 1 meg per cpu */
5950 cnt = data->cnt + (nested ? 27 : 0); in rb_write_something()
5953 size = (cnt * 68 / 25) % (sizeof(rb_string) - 1); in rb_write_something()
5958 /* read rb_test_started before checking buffer enabled */ in rb_write_something()
5961 event = ring_buffer_lock_reserve(data->buffer, len); in rb_write_something()
5966 data->bytes_dropped += len; in rb_write_something()
5968 data->bytes_dropped_nested += len; in rb_write_something()
5975 if (RB_WARN_ON(data->buffer, event_len < len)) in rb_write_something()
5979 item->size = size; in rb_write_something()
5980 memcpy(item->str, rb_string, size); in rb_write_something()
5983 data->bytes_alloc_nested += event_len; in rb_write_something()
5984 data->bytes_written_nested += len; in rb_write_something()
5985 data->events_nested++; in rb_write_something()
5986 if (!data->min_size_nested || len < data->min_size_nested) in rb_write_something()
5987 data->min_size_nested = len; in rb_write_something()
5988 if (len > data->max_size_nested) in rb_write_something()
5989 data->max_size_nested = len; in rb_write_something()
5991 data->bytes_alloc += event_len; in rb_write_something()
5992 data->bytes_written += len; in rb_write_something()
5993 data->events++; in rb_write_something()
5994 if (!data->min_size || len < data->min_size) in rb_write_something()
5995 data->max_size = len; in rb_write_something()
5996 if (len > data->max_size) in rb_write_something()
5997 data->max_size = len; in rb_write_something()
6001 ring_buffer_unlock_commit(data->buffer, event); in rb_write_something()
6012 data->cnt++; in rb_test()
6015 /* Now sleep between a min of 100-300us and a max of 1ms */ in rb_test()
6016 usleep_range(((data->cnt % 3) + 1) * 100, 1000); in rb_test()
6025 int cpu = smp_processor_id(); in rb_ipi() local
6027 data = &rb_data[cpu]; in rb_ipi()
6048 int cpu; in test_ringbuffer() local
6065 for_each_online_cpu(cpu) { in test_ringbuffer()
6066 rb_data[cpu].buffer = buffer; in test_ringbuffer()
6067 rb_data[cpu].cpu = cpu; in test_ringbuffer()
6068 rb_data[cpu].cnt = cpu; in test_ringbuffer()
6069 rb_threads[cpu] = kthread_run_on_cpu(rb_test, &rb_data[cpu], in test_ringbuffer()
6070 cpu, "rbtester/%u"); in test_ringbuffer()
6071 if (WARN_ON(IS_ERR(rb_threads[cpu]))) { in test_ringbuffer()
6073 ret = PTR_ERR(rb_threads[cpu]); in test_ringbuffer()
6106 for_each_online_cpu(cpu) { in test_ringbuffer()
6107 if (!rb_threads[cpu]) in test_ringbuffer()
6109 kthread_stop(rb_threads[cpu]); in test_ringbuffer()
6118 for_each_online_cpu(cpu) { in test_ringbuffer()
6120 struct rb_test_data *data = &rb_data[cpu]; in test_ringbuffer()
6134 ret = -1; in test_ringbuffer()
6136 total_events = data->events + data->events_nested; in test_ringbuffer()
6137 total_written = data->bytes_written + data->bytes_written_nested; in test_ringbuffer()
6138 total_alloc = data->bytes_alloc + data->bytes_alloc_nested; in test_ringbuffer()
6139 total_dropped = data->bytes_dropped + data->bytes_dropped_nested; in test_ringbuffer()
6141 big_event_size = data->max_size + data->max_size_nested; in test_ringbuffer()
6142 small_event_size = data->min_size + data->min_size_nested; in test_ringbuffer()
6144 pr_info("CPU %d:\n", cpu); in test_ringbuffer()
6157 while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) { in test_ringbuffer()
6161 total_size += item->size + sizeof(struct rb_item); in test_ringbuffer()
6162 if (memcmp(&item->str[0], rb_string, item->size) != 0) { in test_ringbuffer()
6164 pr_info("buffer had: %.*s\n", item->size, item->str); in test_ringbuffer()
6165 pr_info("expected: %.*s\n", item->size, rb_string); in test_ringbuffer()
6167 ret = -1; in test_ringbuffer()
6175 ret = -1; in test_ringbuffer()
6177 pr_info(" read events: %ld\n", total_read); in test_ringbuffer()