21 static void perf_output_wakeup(struct perf_output_handle *handle)  in perf_output_wakeup()  argument
23 	atomic_set(&handle->rb->poll, EPOLLIN);  in perf_output_wakeup()
25 	handle->event->pending_wakeup = 1;  in perf_output_wakeup()
26 	irq_work_queue(&handle->event->pending);  in perf_output_wakeup()
37 static void perf_output_get_handle(struct perf_output_handle *handle)  in perf_output_get_handle()  argument
39 	struct ring_buffer *rb = handle->rb;  in perf_output_get_handle()
43 	handle->wakeup = local_read(&rb->wakeup);  in perf_output_get_handle()
46 static void perf_output_put_handle(struct perf_output_handle *handle)  in perf_output_put_handle()  argument
48 	struct ring_buffer *rb = handle->rb;  in perf_output_put_handle()
99 	if (handle->wakeup != local_read(&rb->wakeup))  in perf_output_put_handle()
100 		perf_output_wakeup(handle);  in perf_output_put_handle()
118 __perf_output_begin(struct perf_output_handle *handle,  in __perf_output_begin()  argument
148 	handle->rb    = rb;  in __perf_output_begin()
149 	handle->event = event;  in __perf_output_begin()
158 	perf_output_get_handle(handle);  in __perf_output_begin()
203 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);  in __perf_output_begin()
205 	handle->addr = rb->data_pages[handle->page] + offset;  in __perf_output_begin()
206 	handle->size = (1UL << page_shift) - offset;  in __perf_output_begin()
219 		perf_output_put(handle, lost_event);  in __perf_output_begin()
220 		perf_event__output_id_sample(event, handle, &sample_data);  in __perf_output_begin()
227 	perf_output_put_handle(handle);  in __perf_output_begin()
234 int perf_output_begin_forward(struct perf_output_handle *handle,  in perf_output_begin_forward()  argument
237 	return __perf_output_begin(handle, event, size, false);  in perf_output_begin_forward()
240 int perf_output_begin_backward(struct perf_output_handle *handle,  in perf_output_begin_backward()  argument
243 	return __perf_output_begin(handle, event, size, true);  in perf_output_begin_backward()
246 int perf_output_begin(struct perf_output_handle *handle,  in perf_output_begin()  argument
250 	return __perf_output_begin(handle, event, size,  in perf_output_begin()
254 unsigned int perf_output_copy(struct perf_output_handle *handle,  in perf_output_copy()  argument
257 	return __output_copy(handle, buf, len);  in perf_output_copy()
260 unsigned int perf_output_skip(struct perf_output_handle *handle,  in perf_output_skip()  argument
263 	return __output_skip(handle, NULL, len);  in perf_output_skip()
266 void perf_output_end(struct perf_output_handle *handle)  in perf_output_end()  argument
268 	perf_output_put_handle(handle);  in perf_output_end()
301 void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)  in perf_aux_output_flag()  argument
310 	handle->aux_flags |= flags;  in perf_aux_output_flag()
328 void *perf_aux_output_begin(struct perf_output_handle *handle,  in perf_aux_output_begin()  argument
373 	handle->rb = rb;  in perf_aux_output_begin()
374 	handle->event = event;  in perf_aux_output_begin()
375 	handle->head = aux_head;  in perf_aux_output_begin()
376 	handle->size = 0;  in perf_aux_output_begin()
377 	handle->aux_flags = 0;  in perf_aux_output_begin()
386 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_begin()
388 			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));  in perf_aux_output_begin()
395 		if (!handle->size) { /* A, matches D */  in perf_aux_output_begin()
397 			perf_output_wakeup(handle);  in perf_aux_output_begin()
403 	return handle->rb->aux_priv;  in perf_aux_output_begin()
411 	handle->event = NULL;  in perf_aux_output_begin()
440 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_end()  argument
442 	bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);  in perf_aux_output_end()
443 	struct ring_buffer *rb = handle->rb;  in perf_aux_output_end()
448 		handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
450 		aux_head = handle->head;  in perf_aux_output_end()
453 		handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
459 	if (size || handle->aux_flags) {  in perf_aux_output_end()
464 		perf_event_aux_event(handle->event, aux_head, size,  in perf_aux_output_end()
465 		                     handle->aux_flags);  in perf_aux_output_end()
473 		if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)  in perf_aux_output_end()
474 			handle->event->pending_disable = 1;  in perf_aux_output_end()
475 		perf_output_wakeup(handle);  in perf_aux_output_end()
478 	handle->event = NULL;  in perf_aux_output_end()
491 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_skip()  argument
493 	struct ring_buffer *rb = handle->rb;  in perf_aux_output_skip()
495 	if (size > handle->size)  in perf_aux_output_skip()
502 		perf_output_wakeup(handle);  in perf_aux_output_skip()
503 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_skip()
506 	handle->head = rb->aux_head;  in perf_aux_output_skip()
507 	handle->size -= size;  in perf_aux_output_skip()
513 void *perf_get_aux(struct perf_output_handle *handle)  in perf_get_aux()  argument
516 	if (!handle->event)  in perf_get_aux()
519 	return handle->rb->aux_priv;  in perf_get_aux()