52 	struct bpf_ringbuf *rb;  member
69 	struct bpf_ringbuf *rb;  in bpf_ringbuf_area_alloc()  local
109 	rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,  in bpf_ringbuf_area_alloc()
111 	if (rb) {  in bpf_ringbuf_area_alloc()
112 		rb->pages = pages;  in bpf_ringbuf_area_alloc()
113 		rb->nr_pages = nr_pages;  in bpf_ringbuf_area_alloc()
114 		return rb;  in bpf_ringbuf_area_alloc()
126 	struct bpf_ringbuf *rb = container_of(work, struct bpf_ringbuf, work);  in bpf_ringbuf_notify()  local
128 	wake_up_all(&rb->waitq);  in bpf_ringbuf_notify()
133 	struct bpf_ringbuf *rb;  in bpf_ringbuf_alloc()  local
135 	rb = bpf_ringbuf_area_alloc(data_sz, numa_node);  in bpf_ringbuf_alloc()
136 	if (!rb)  in bpf_ringbuf_alloc()
139 	spin_lock_init(&rb->spinlock);  in bpf_ringbuf_alloc()
140 	init_waitqueue_head(&rb->waitq);  in bpf_ringbuf_alloc()
141 	init_irq_work(&rb->work, bpf_ringbuf_notify);  in bpf_ringbuf_alloc()
143 	rb->mask = data_sz - 1;  in bpf_ringbuf_alloc()
144 	rb->consumer_pos = 0;  in bpf_ringbuf_alloc()
145 	rb->producer_pos = 0;  in bpf_ringbuf_alloc()
147 	return rb;  in bpf_ringbuf_alloc()
183 	rb_map->rb = bpf_ringbuf_alloc(attr->max_entries, rb_map->map.numa_node);  in ringbuf_map_alloc()
184 	if (IS_ERR(rb_map->rb)) {  in ringbuf_map_alloc()
185 		err = PTR_ERR(rb_map->rb);  in ringbuf_map_alloc()
198 static void bpf_ringbuf_free(struct bpf_ringbuf *rb)  in bpf_ringbuf_free()  argument
201 	 * to unmap rb itself with vunmap() below  in bpf_ringbuf_free()
203 	struct page **pages = rb->pages;  in bpf_ringbuf_free()
204 	int i, nr_pages = rb->nr_pages;  in bpf_ringbuf_free()
206 	vunmap(rb);  in bpf_ringbuf_free()
217 	bpf_ringbuf_free(rb_map->rb);  in ringbuf_map_free()
243 static size_t bpf_ringbuf_mmap_page_cnt(const struct bpf_ringbuf *rb)  in bpf_ringbuf_mmap_page_cnt()  argument
245 	size_t data_pages = (rb->mask + 1) >> PAGE_SHIFT;  in bpf_ringbuf_mmap_page_cnt()
257 	mmap_sz = bpf_ringbuf_mmap_page_cnt(rb_map->rb) << PAGE_SHIFT;  in ringbuf_map_mmap()
262 	return remap_vmalloc_range(vma, rb_map->rb,  in ringbuf_map_mmap()
266 static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb)  in ringbuf_avail_data_sz()  argument
270 	cons_pos = smp_load_acquire(&rb->consumer_pos);  in ringbuf_avail_data_sz()
271 	prod_pos = smp_load_acquire(&rb->producer_pos);  in ringbuf_avail_data_sz()
281 	poll_wait(filp, &rb_map->rb->waitq, pts);  in ringbuf_map_poll()
283 	if (ringbuf_avail_data_sz(rb_map->rb))  in ringbuf_map_poll()
309 static size_t bpf_ringbuf_rec_pg_off(struct bpf_ringbuf *rb,  in bpf_ringbuf_rec_pg_off()  argument
312 	return ((void *)hdr - (void *)rb) >> PAGE_SHIFT;  in bpf_ringbuf_rec_pg_off()
327 static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size)  in __bpf_ringbuf_reserve()  argument
337 	cons_pos = smp_load_acquire(&rb->consumer_pos);  in __bpf_ringbuf_reserve()
340 		if (!spin_trylock_irqsave(&rb->spinlock, flags))  in __bpf_ringbuf_reserve()
343 		spin_lock_irqsave(&rb->spinlock, flags);  in __bpf_ringbuf_reserve()
346 	prod_pos = rb->producer_pos;  in __bpf_ringbuf_reserve()
352 	if (new_prod_pos - cons_pos > rb->mask) {  in __bpf_ringbuf_reserve()
353 		spin_unlock_irqrestore(&rb->spinlock, flags);  in __bpf_ringbuf_reserve()
357 	hdr = (void *)rb->data + (prod_pos & rb->mask);  in __bpf_ringbuf_reserve()
358 	pg_off = bpf_ringbuf_rec_pg_off(rb, hdr);  in __bpf_ringbuf_reserve()
363 	smp_store_release(&rb->producer_pos, new_prod_pos);  in __bpf_ringbuf_reserve()
365 	spin_unlock_irqrestore(&rb->spinlock, flags);  in __bpf_ringbuf_reserve()
378 	return (unsigned long)__bpf_ringbuf_reserve(rb_map->rb, size);  in BPF_CALL_3()
393 	struct bpf_ringbuf *rb;  in bpf_ringbuf_commit()  local
397 	rb = bpf_ringbuf_restore_from_rec(hdr);  in bpf_ringbuf_commit()
408 	rec_pos = (void *)hdr - (void *)rb->data;  in bpf_ringbuf_commit()
409 	cons_pos = smp_load_acquire(&rb->consumer_pos) & rb->mask;  in bpf_ringbuf_commit()
412 		irq_work_queue(&rb->work);  in bpf_ringbuf_commit()
414 		irq_work_queue(&rb->work);  in bpf_ringbuf_commit()
453 	rec = __bpf_ringbuf_reserve(rb_map->rb, size);  in BPF_CALL_4()
473 	struct bpf_ringbuf *rb;  in BPF_CALL_2()  local
475 	rb = container_of(map, struct bpf_ringbuf_map, map)->rb;  in BPF_CALL_2()
479 		return ringbuf_avail_data_sz(rb);  in BPF_CALL_2()
481 		return rb->mask + 1;  in BPF_CALL_2()
483 		return smp_load_acquire(&rb->consumer_pos);  in BPF_CALL_2()
485 		return smp_load_acquire(&rb->producer_pos);  in BPF_CALL_2()