60 static void ringbuf_unmap_ring(struct ring_buffer *rb, struct ring *r)  in ringbuf_unmap_ring()  argument
63 		munmap(r->consumer_pos, rb->page_size);  in ringbuf_unmap_ring()
67 		munmap(r->producer_pos, rb->page_size + 2 * (r->mask + 1));  in ringbuf_unmap_ring()
73 int ring_buffer__add(struct ring_buffer *rb, int map_fd,  in ring_buffer__add()  argument
100 	tmp = libbpf_reallocarray(rb->rings, rb->ring_cnt + 1, sizeof(*rb->rings));  in ring_buffer__add()
103 	rb->rings = tmp;  in ring_buffer__add()
105 	tmp = libbpf_reallocarray(rb->events, rb->ring_cnt + 1, sizeof(*rb->events));  in ring_buffer__add()
108 	rb->events = tmp;  in ring_buffer__add()
110 	r = &rb->rings[rb->ring_cnt];  in ring_buffer__add()
119 	tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0);  in ring_buffer__add()
132 	mmap_sz = rb->page_size + 2 * (__u64)info.max_entries;  in ring_buffer__add()
137 	tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size);  in ring_buffer__add()
140 		ringbuf_unmap_ring(rb, r);  in ring_buffer__add()
146 	r->data = tmp + rb->page_size;  in ring_buffer__add()
148 	e = &rb->events[rb->ring_cnt];  in ring_buffer__add()
152 	e->data.fd = rb->ring_cnt;  in ring_buffer__add()
153 	if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) {  in ring_buffer__add()
155 		ringbuf_unmap_ring(rb, r);  in ring_buffer__add()
161 	rb->ring_cnt++;  in ring_buffer__add()
165 void ring_buffer__free(struct ring_buffer *rb)  in ring_buffer__free()  argument
169 	if (!rb)  in ring_buffer__free()
172 	for (i = 0; i < rb->ring_cnt; ++i)  in ring_buffer__free()
173 		ringbuf_unmap_ring(rb, &rb->rings[i]);  in ring_buffer__free()
174 	if (rb->epoll_fd >= 0)  in ring_buffer__free()
175 		close(rb->epoll_fd);  in ring_buffer__free()
177 	free(rb->events);  in ring_buffer__free()
178 	free(rb->rings);  in ring_buffer__free()
179 	free(rb);  in ring_buffer__free()
186 	struct ring_buffer *rb;  in ring_buffer__new()  local
192 	rb = calloc(1, sizeof(*rb));  in ring_buffer__new()
193 	if (!rb)  in ring_buffer__new()
196 	rb->page_size = getpagesize();  in ring_buffer__new()
198 	rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in ring_buffer__new()
199 	if (rb->epoll_fd < 0) {  in ring_buffer__new()
205 	err = ring_buffer__add(rb, map_fd, sample_cb, ctx);  in ring_buffer__new()
209 	return rb;  in ring_buffer__new()
212 	ring_buffer__free(rb);  in ring_buffer__new()
275 int ring_buffer__consume(struct ring_buffer *rb)  in ring_buffer__consume()  argument
280 	for (i = 0; i < rb->ring_cnt; i++) {  in ring_buffer__consume()
281 		struct ring *ring = &rb->rings[i];  in ring_buffer__consume()
297 int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms)  in ring_buffer__poll()  argument
302 	cnt = epoll_wait(rb->epoll_fd, rb->events, rb->ring_cnt, timeout_ms);  in ring_buffer__poll()
307 		__u32 ring_id = rb->events[i].data.fd;  in ring_buffer__poll()
308 		struct ring *ring = &rb->rings[ring_id];  in ring_buffer__poll()
321 int ring_buffer__epoll_fd(const struct ring_buffer *rb)  in ring_buffer__epoll_fd()  argument
323 	return rb->epoll_fd;  in ring_buffer__epoll_fd()
326 static void user_ringbuf_unmap_ring(struct user_ring_buffer *rb)  in user_ringbuf_unmap_ring()  argument
328 	if (rb->consumer_pos) {  in user_ringbuf_unmap_ring()
329 		munmap(rb->consumer_pos, rb->page_size);  in user_ringbuf_unmap_ring()
330 		rb->consumer_pos = NULL;  in user_ringbuf_unmap_ring()
332 	if (rb->producer_pos) {  in user_ringbuf_unmap_ring()
333 		munmap(rb->producer_pos, rb->page_size + 2 * (rb->mask + 1));  in user_ringbuf_unmap_ring()
334 		rb->producer_pos = NULL;  in user_ringbuf_unmap_ring()
338 void user_ring_buffer__free(struct user_ring_buffer *rb)  in user_ring_buffer__free()  argument
340 	if (!rb)  in user_ring_buffer__free()
343 	user_ringbuf_unmap_ring(rb);  in user_ring_buffer__free()
345 	if (rb->epoll_fd >= 0)  in user_ring_buffer__free()
346 		close(rb->epoll_fd);  in user_ring_buffer__free()
348 	free(rb);  in user_ring_buffer__free()
351 static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd)  in user_ringbuf_map()  argument
374 	rb->map_fd = map_fd;  in user_ringbuf_map()
375 	rb->mask = info.max_entries - 1;  in user_ringbuf_map()
378 	tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);  in user_ringbuf_map()
385 	rb->consumer_pos = tmp;  in user_ringbuf_map()
392 	mmap_sz = rb->page_size + 2 * (__u64)info.max_entries;  in user_ringbuf_map()
398 		   map_fd, rb->page_size);  in user_ringbuf_map()
406 	rb->producer_pos = tmp;  in user_ringbuf_map()
407 	rb->data = tmp + rb->page_size;  in user_ringbuf_map()
409 	rb_epoll = &rb->event;  in user_ringbuf_map()
411 	if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {  in user_ringbuf_map()
423 	struct user_ring_buffer *rb;  in user_ring_buffer__new()  local
429 	rb = calloc(1, sizeof(*rb));  in user_ring_buffer__new()
430 	if (!rb)  in user_ring_buffer__new()
433 	rb->page_size = getpagesize();  in user_ring_buffer__new()
435 	rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in user_ring_buffer__new()
436 	if (rb->epoll_fd < 0) {  in user_ring_buffer__new()
442 	err = user_ringbuf_map(rb, map_fd);  in user_ring_buffer__new()
446 	return rb;  in user_ring_buffer__new()
449 	user_ring_buffer__free(rb);  in user_ring_buffer__new()
453 static void user_ringbuf_commit(struct user_ring_buffer *rb, void *sample, bool discard)  in user_ringbuf_commit()  argument
459 	hdr_offset = rb->mask + 1 + (sample - rb->data) - BPF_RINGBUF_HDR_SZ;  in user_ringbuf_commit()
460 	hdr = rb->data + (hdr_offset & rb->mask);  in user_ringbuf_commit()
472 void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample)  in user_ring_buffer__discard()  argument
474 	user_ringbuf_commit(rb, sample, true);  in user_ring_buffer__discard()
477 void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample)  in user_ring_buffer__submit()  argument
479 	user_ringbuf_commit(rb, sample, false);  in user_ring_buffer__submit()
482 void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size)  in user_ring_buffer__reserve()  argument
496 	cons_pos = smp_load_acquire(rb->consumer_pos);  in user_ring_buffer__reserve()
498 	prod_pos = smp_load_acquire(rb->producer_pos);  in user_ring_buffer__reserve()
500 	max_size = rb->mask + 1;  in user_ring_buffer__reserve()
511 	hdr = rb->data + (prod_pos & rb->mask);  in user_ring_buffer__reserve()
518 	smp_store_release(rb->producer_pos, prod_pos + total_size);  in user_ring_buffer__reserve()
520 	return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask);  in user_ring_buffer__reserve()
533 void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, __u32 size, int timeout_ms)  in user_ring_buffer__reserve_blocking()  argument
553 		sample = user_ring_buffer__reserve(rb, size);  in user_ring_buffer__reserve_blocking()
570 		cnt = epoll_wait(rb->epoll_fd, &rb->event, 1, ms_remaining);  in user_ring_buffer__reserve_blocking()
586 	return user_ring_buffer__reserve(rb, size);  in user_ring_buffer__reserve_blocking()