20 static void perf_output_wakeup(struct perf_output_handle *handle)  in perf_output_wakeup()  argument
22 	atomic_set(&handle->rb->poll, EPOLLIN);  in perf_output_wakeup()
24 	handle->event->pending_wakeup = 1;  in perf_output_wakeup()
25 	irq_work_queue(&handle->event->pending);  in perf_output_wakeup()
36 static void perf_output_get_handle(struct perf_output_handle *handle)  in perf_output_get_handle()  argument
38 	struct ring_buffer *rb = handle->rb;  in perf_output_get_handle()
47 	handle->wakeup = local_read(&rb->wakeup);  in perf_output_get_handle()
50 static void perf_output_put_handle(struct perf_output_handle *handle)  in perf_output_put_handle()  argument
52 	struct ring_buffer *rb = handle->rb;  in perf_output_put_handle()
130 	if (handle->wakeup != local_read(&rb->wakeup))  in perf_output_put_handle()
131 		perf_output_wakeup(handle);  in perf_output_put_handle()
149 __perf_output_begin(struct perf_output_handle *handle,  in __perf_output_begin()  argument
179 	handle->rb    = rb;  in __perf_output_begin()
180 	handle->event = event;  in __perf_output_begin()
189 	perf_output_get_handle(handle);  in __perf_output_begin()
234 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);  in __perf_output_begin()
236 	handle->addr = rb->data_pages[handle->page] + offset;  in __perf_output_begin()
237 	handle->size = (1UL << page_shift) - offset;  in __perf_output_begin()
250 		perf_output_put(handle, lost_event);  in __perf_output_begin()
251 		perf_event__output_id_sample(event, handle, &sample_data);  in __perf_output_begin()
258 	perf_output_put_handle(handle);  in __perf_output_begin()
265 int perf_output_begin_forward(struct perf_output_handle *handle,  in perf_output_begin_forward()  argument
268 	return __perf_output_begin(handle, event, size, false);  in perf_output_begin_forward()
271 int perf_output_begin_backward(struct perf_output_handle *handle,  in perf_output_begin_backward()  argument
274 	return __perf_output_begin(handle, event, size, true);  in perf_output_begin_backward()
277 int perf_output_begin(struct perf_output_handle *handle,  in perf_output_begin()  argument
281 	return __perf_output_begin(handle, event, size,  in perf_output_begin()
285 unsigned int perf_output_copy(struct perf_output_handle *handle,  in perf_output_copy()  argument
288 	return __output_copy(handle, buf, len);  in perf_output_copy()
291 unsigned int perf_output_skip(struct perf_output_handle *handle,  in perf_output_skip()  argument
294 	return __output_skip(handle, NULL, len);  in perf_output_skip()
297 void perf_output_end(struct perf_output_handle *handle)  in perf_output_end()  argument
299 	perf_output_put_handle(handle);  in perf_output_end()
332 void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)  in perf_aux_output_flag()  argument
341 	handle->aux_flags |= flags;  in perf_aux_output_flag()
359 void *perf_aux_output_begin(struct perf_output_handle *handle,  in perf_aux_output_begin()  argument
408 	handle->rb = rb;  in perf_aux_output_begin()
409 	handle->event = event;  in perf_aux_output_begin()
410 	handle->head = aux_head;  in perf_aux_output_begin()
411 	handle->size = 0;  in perf_aux_output_begin()
412 	handle->aux_flags = 0;  in perf_aux_output_begin()
421 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_begin()
423 			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));  in perf_aux_output_begin()
430 		if (!handle->size) { /* A, matches D */  in perf_aux_output_begin()
432 			perf_output_wakeup(handle);  in perf_aux_output_begin()
438 	return handle->rb->aux_priv;  in perf_aux_output_begin()
446 	handle->event = NULL;  in perf_aux_output_begin()
475 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_end()  argument
477 	bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);  in perf_aux_output_end()
478 	struct ring_buffer *rb = handle->rb;  in perf_aux_output_end()
483 		handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
485 		aux_head = handle->head;  in perf_aux_output_end()
488 		handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
506 	if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))  in perf_aux_output_end()
507 		perf_event_aux_event(handle->event, aux_head, size,  in perf_aux_output_end()
508 				     handle->aux_flags);  in perf_aux_output_end()
515 		if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)  in perf_aux_output_end()
516 			handle->event->pending_disable = smp_processor_id();  in perf_aux_output_end()
517 		perf_output_wakeup(handle);  in perf_aux_output_end()
520 	handle->event = NULL;  in perf_aux_output_end()
533 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_skip()  argument
535 	struct ring_buffer *rb = handle->rb;  in perf_aux_output_skip()
537 	if (size > handle->size)  in perf_aux_output_skip()
544 		perf_output_wakeup(handle);  in perf_aux_output_skip()
545 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_skip()
548 	handle->head = rb->aux_head;  in perf_aux_output_skip()
549 	handle->size -= size;  in perf_aux_output_skip()
555 void *perf_get_aux(struct perf_output_handle *handle)  in perf_get_aux()  argument
558 	if (!handle->event)  in perf_get_aux()
561 	return handle->rb->aux_priv;  in perf_get_aux()