20 static void perf_output_wakeup(struct perf_output_handle *handle)  in perf_output_wakeup()  argument
22 	atomic_set(&handle->rb->poll, EPOLLIN);  in perf_output_wakeup()
24 	handle->event->pending_wakeup = 1;  in perf_output_wakeup()
25 	irq_work_queue(&handle->event->pending_irq);  in perf_output_wakeup()
36 static void perf_output_get_handle(struct perf_output_handle *handle)  in perf_output_get_handle()  argument
38 	struct perf_buffer *rb = handle->rb;  in perf_output_get_handle()
47 	handle->wakeup = local_read(&rb->wakeup);  in perf_output_get_handle()
50 static void perf_output_put_handle(struct perf_output_handle *handle)  in perf_output_put_handle()  argument
52 	struct perf_buffer *rb = handle->rb;  in perf_output_put_handle()
130 	if (handle->wakeup != local_read(&rb->wakeup))  in perf_output_put_handle()
131 		perf_output_wakeup(handle);  in perf_output_put_handle()
149 __perf_output_begin(struct perf_output_handle *handle,  in __perf_output_begin()  argument
182 	handle->rb    = rb;  in __perf_output_begin()
183 	handle->event = event;  in __perf_output_begin()
192 	perf_output_get_handle(handle);  in __perf_output_begin()
238 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);  in __perf_output_begin()
240 	handle->addr = rb->data_pages[handle->page] + offset;  in __perf_output_begin()
241 	handle->size = (1UL << page_shift) - offset;  in __perf_output_begin()
252 		perf_output_put(handle, lost_event);  in __perf_output_begin()
253 		perf_event__output_id_sample(event, handle, data);  in __perf_output_begin()
261 	perf_output_put_handle(handle);  in __perf_output_begin()
268 int perf_output_begin_forward(struct perf_output_handle *handle,  in perf_output_begin_forward()  argument
272 	return __perf_output_begin(handle, data, event, size, false);  in perf_output_begin_forward()
275 int perf_output_begin_backward(struct perf_output_handle *handle,  in perf_output_begin_backward()  argument
279 	return __perf_output_begin(handle, data, event, size, true);  in perf_output_begin_backward()
282 int perf_output_begin(struct perf_output_handle *handle,  in perf_output_begin()  argument
287 	return __perf_output_begin(handle, data, event, size,  in perf_output_begin()
291 unsigned int perf_output_copy(struct perf_output_handle *handle,  in perf_output_copy()  argument
294 	return __output_copy(handle, buf, len);  in perf_output_copy()
297 unsigned int perf_output_skip(struct perf_output_handle *handle,  in perf_output_skip()  argument
300 	return __output_skip(handle, NULL, len);  in perf_output_skip()
303 void perf_output_end(struct perf_output_handle *handle)  in perf_output_end()  argument
305 	perf_output_put_handle(handle);  in perf_output_end()
338 void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)  in perf_aux_output_flag()  argument
347 	handle->aux_flags |= flags;  in perf_aux_output_flag()
365 void *perf_aux_output_begin(struct perf_output_handle *handle,  in perf_aux_output_begin()  argument
414 	handle->rb = rb;  in perf_aux_output_begin()
415 	handle->event = event;  in perf_aux_output_begin()
416 	handle->head = aux_head;  in perf_aux_output_begin()
417 	handle->size = 0;  in perf_aux_output_begin()
418 	handle->aux_flags = 0;  in perf_aux_output_begin()
427 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_begin()
429 			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));  in perf_aux_output_begin()
436 		if (!handle->size) { /* A, matches D */  in perf_aux_output_begin()
438 			perf_output_wakeup(handle);  in perf_aux_output_begin()
444 	return handle->rb->aux_priv;  in perf_aux_output_begin()
452 	handle->event = NULL;  in perf_aux_output_begin()
481 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_end()  argument
483 	bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);  in perf_aux_output_end()
484 	struct perf_buffer *rb = handle->rb;  in perf_aux_output_end()
489 		handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
491 		aux_head = handle->head;  in perf_aux_output_end()
494 		handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
512 	if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))  in perf_aux_output_end()
513 		perf_event_aux_event(handle->event, aux_head, size,  in perf_aux_output_end()
514 				     handle->aux_flags);  in perf_aux_output_end()
521 		if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)  in perf_aux_output_end()
522 			handle->event->pending_disable = smp_processor_id();  in perf_aux_output_end()
523 		perf_output_wakeup(handle);  in perf_aux_output_end()
526 	handle->event = NULL;  in perf_aux_output_end()
539 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_skip()  argument
541 	struct perf_buffer *rb = handle->rb;  in perf_aux_output_skip()
543 	if (size > handle->size)  in perf_aux_output_skip()
550 		perf_output_wakeup(handle);  in perf_aux_output_skip()
551 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_skip()
554 	handle->head = rb->aux_head;  in perf_aux_output_skip()
555 	handle->size -= size;  in perf_aux_output_skip()
561 void *perf_get_aux(struct perf_output_handle *handle)  in perf_get_aux()  argument
564 	if (!handle->event)  in perf_get_aux()
567 	return handle->rb->aux_priv;  in perf_get_aux()
575 			  struct perf_output_handle *handle,  in perf_output_copy_aux()  argument
595 		remainder = perf_output_copy(handle, addr, tocopy);  in perf_output_copy_aux()