25 int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,  in wg_packet_queue_init()  argument
30 	memset(queue, 0, sizeof(*queue));  in wg_packet_queue_init()
31 	queue->last_cpu = -1;  in wg_packet_queue_init()
32 	ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL);  in wg_packet_queue_init()
35 	queue->worker = wg_packet_percpu_multicore_worker_alloc(function, queue);  in wg_packet_queue_init()
36 	if (!queue->worker) {  in wg_packet_queue_init()
37 		ptr_ring_cleanup(&queue->ring, NULL);  in wg_packet_queue_init()
43 void wg_packet_queue_free(struct crypt_queue *queue, bool purge)  in wg_packet_queue_free()  argument
45 	free_percpu(queue->worker);  in wg_packet_queue_free()
46 	WARN_ON(!purge && !__ptr_ring_empty(&queue->ring));  in wg_packet_queue_free()
47 	ptr_ring_cleanup(&queue->ring, purge ? __skb_array_destroy_skb : NULL);  in wg_packet_queue_free()
51 #define STUB(queue) ((struct sk_buff *)&queue->empty)  argument
53 void wg_prev_queue_init(struct prev_queue *queue)  in wg_prev_queue_init()  argument
55 	NEXT(STUB(queue)) = NULL;  in wg_prev_queue_init()
56 	queue->head = queue->tail = STUB(queue);  in wg_prev_queue_init()
57 	queue->peeked = NULL;  in wg_prev_queue_init()
58 	atomic_set(&queue->count, 0);  in wg_prev_queue_init()
66 static void __wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb)  in __wg_prev_queue_enqueue()  argument
69 	WRITE_ONCE(NEXT(xchg_release(&queue->head, skb)), skb);  in __wg_prev_queue_enqueue()
72 bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb)  in wg_prev_queue_enqueue()  argument
74 	if (!atomic_add_unless(&queue->count, 1, MAX_QUEUED_PACKETS))  in wg_prev_queue_enqueue()
76 	__wg_prev_queue_enqueue(queue, skb);  in wg_prev_queue_enqueue()
80 struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue)  in wg_prev_queue_dequeue()  argument
82 	struct sk_buff *tail = queue->tail, *next = smp_load_acquire(&NEXT(tail));  in wg_prev_queue_dequeue()
84 	if (tail == STUB(queue)) {  in wg_prev_queue_dequeue()
87 		queue->tail = next;  in wg_prev_queue_dequeue()
92 		queue->tail = next;  in wg_prev_queue_dequeue()
93 		atomic_dec(&queue->count);  in wg_prev_queue_dequeue()
96 	if (tail != READ_ONCE(queue->head))  in wg_prev_queue_dequeue()
98 	__wg_prev_queue_enqueue(queue, STUB(queue));  in wg_prev_queue_dequeue()
101 		queue->tail = next;  in wg_prev_queue_dequeue()
102 		atomic_dec(&queue->count);  in wg_prev_queue_dequeue()