47 static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)  in xskq_nb_invalid_descs()  argument
49 	return q ? q->invalid_descs : 0;  in xskq_nb_invalid_descs()
52 static inline u32 xskq_nb_avail(struct xsk_queue *q, u32 dcnt)  in xskq_nb_avail()  argument
54 	u32 entries = q->prod_tail - q->cons_tail;  in xskq_nb_avail()
58 		q->prod_tail = READ_ONCE(q->ring->producer);  in xskq_nb_avail()
59 		entries = q->prod_tail - q->cons_tail;  in xskq_nb_avail()
65 static inline u32 xskq_nb_free(struct xsk_queue *q, u32 producer, u32 dcnt)  in xskq_nb_free()  argument
67 	u32 free_entries = q->nentries - (producer - q->cons_tail);  in xskq_nb_free()
73 	q->cons_tail = READ_ONCE(q->ring->consumer);  in xskq_nb_free()
74 	return q->nentries - (producer - q->cons_tail);  in xskq_nb_free()
79 static inline bool xskq_is_valid_addr(struct xsk_queue *q, u64 addr)  in xskq_is_valid_addr()  argument
81 	if (addr >= q->umem_props.size) {  in xskq_is_valid_addr()
82 		q->invalid_descs++;  in xskq_is_valid_addr()
89 static inline u64 *xskq_validate_addr(struct xsk_queue *q, u64 *addr)  in xskq_validate_addr()  argument
91 	while (q->cons_tail != q->cons_head) {  in xskq_validate_addr()
92 		struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;  in xskq_validate_addr()
93 		unsigned int idx = q->cons_tail & q->ring_mask;  in xskq_validate_addr()
95 		*addr = READ_ONCE(ring->desc[idx]) & q->umem_props.chunk_mask;  in xskq_validate_addr()
96 		if (xskq_is_valid_addr(q, *addr))  in xskq_validate_addr()
99 		q->cons_tail++;  in xskq_validate_addr()
105 static inline u64 *xskq_peek_addr(struct xsk_queue *q, u64 *addr)  in xskq_peek_addr()  argument
107 	if (q->cons_tail == q->cons_head) {  in xskq_peek_addr()
108 		WRITE_ONCE(q->ring->consumer, q->cons_tail);  in xskq_peek_addr()
109 		q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);  in xskq_peek_addr()
115 	return xskq_validate_addr(q, addr);  in xskq_peek_addr()
118 static inline void xskq_discard_addr(struct xsk_queue *q)  in xskq_discard_addr()  argument
120 	q->cons_tail++;  in xskq_discard_addr()
123 static inline int xskq_produce_addr(struct xsk_queue *q, u64 addr)  in xskq_produce_addr()  argument
125 	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;  in xskq_produce_addr()
127 	if (xskq_nb_free(q, q->prod_tail, 1) == 0)  in xskq_produce_addr()
130 	ring->desc[q->prod_tail++ & q->ring_mask] = addr;  in xskq_produce_addr()
135 	WRITE_ONCE(q->ring->producer, q->prod_tail);  in xskq_produce_addr()
139 static inline int xskq_produce_addr_lazy(struct xsk_queue *q, u64 addr)  in xskq_produce_addr_lazy()  argument
141 	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;  in xskq_produce_addr_lazy()
143 	if (xskq_nb_free(q, q->prod_head, LAZY_UPDATE_THRESHOLD) == 0)  in xskq_produce_addr_lazy()
146 	ring->desc[q->prod_head++ & q->ring_mask] = addr;  in xskq_produce_addr_lazy()
150 static inline void xskq_produce_flush_addr_n(struct xsk_queue *q,  in xskq_produce_flush_addr_n()  argument
156 	q->prod_tail += nb_entries;  in xskq_produce_flush_addr_n()
157 	WRITE_ONCE(q->ring->producer, q->prod_tail);  in xskq_produce_flush_addr_n()
160 static inline int xskq_reserve_addr(struct xsk_queue *q)  in xskq_reserve_addr()  argument
162 	if (xskq_nb_free(q, q->prod_head, 1) == 0)  in xskq_reserve_addr()
165 	q->prod_head++;  in xskq_reserve_addr()
171 static inline bool xskq_is_valid_desc(struct xsk_queue *q, struct xdp_desc *d)  in xskq_is_valid_desc()  argument
173 	if (!xskq_is_valid_addr(q, d->addr))  in xskq_is_valid_desc()
176 	if (((d->addr + d->len) & q->umem_props.chunk_mask) !=  in xskq_is_valid_desc()
177 	    (d->addr & q->umem_props.chunk_mask)) {  in xskq_is_valid_desc()
178 		q->invalid_descs++;  in xskq_is_valid_desc()
185 static inline struct xdp_desc *xskq_validate_desc(struct xsk_queue *q,  in xskq_validate_desc()  argument
188 	while (q->cons_tail != q->cons_head) {  in xskq_validate_desc()
189 		struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;  in xskq_validate_desc()
190 		unsigned int idx = q->cons_tail & q->ring_mask;  in xskq_validate_desc()
193 		if (xskq_is_valid_desc(q, desc))  in xskq_validate_desc()
196 		q->cons_tail++;  in xskq_validate_desc()
202 static inline struct xdp_desc *xskq_peek_desc(struct xsk_queue *q,  in xskq_peek_desc()  argument
205 	if (q->cons_tail == q->cons_head) {  in xskq_peek_desc()
206 		WRITE_ONCE(q->ring->consumer, q->cons_tail);  in xskq_peek_desc()
207 		q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);  in xskq_peek_desc()
213 	return xskq_validate_desc(q, desc);  in xskq_peek_desc()
216 static inline void xskq_discard_desc(struct xsk_queue *q)  in xskq_discard_desc()  argument
218 	q->cons_tail++;  in xskq_discard_desc()
221 static inline int xskq_produce_batch_desc(struct xsk_queue *q,  in xskq_produce_batch_desc()  argument
224 	struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;  in xskq_produce_batch_desc()
227 	if (xskq_nb_free(q, q->prod_head, 1) == 0)  in xskq_produce_batch_desc()
230 	idx = (q->prod_head++) & q->ring_mask;  in xskq_produce_batch_desc()
237 static inline void xskq_produce_flush_desc(struct xsk_queue *q)  in xskq_produce_flush_desc()  argument
242 	q->prod_tail = q->prod_head,  in xskq_produce_flush_desc()
243 	WRITE_ONCE(q->ring->producer, q->prod_tail);  in xskq_produce_flush_desc()
246 static inline bool xskq_full_desc(struct xsk_queue *q)  in xskq_full_desc()  argument
248 	return xskq_nb_avail(q, q->nentries) == q->nentries;  in xskq_full_desc()
251 static inline bool xskq_empty_desc(struct xsk_queue *q)  in xskq_empty_desc()  argument
253 	return xskq_nb_free(q, q->prod_tail, q->nentries) == q->nentries;  in xskq_empty_desc()
256 void xskq_set_umem(struct xsk_queue *q, struct xdp_umem_props *umem_props);