1 /* SPDX-License-Identifier: GPL-2.0 */
9 #include <linux/dma-mapping.h>
24 	dma_addr_t dma;  member
26 	struct xsk_buff_pool *pool;  member
58 	/* For performance reasons, each buff pool has its own array of dma_pages
80 	 * sockets share a single cq when the same netdev and queue id is shared.
87  * The low 12-bits of the addr will be 0 since this is the page address, so we
96 int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,
98 int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs,
100 int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs);
101 void xp_destroy(struct xsk_buff_pool *pool);
102 void xp_get_pool(struct xsk_buff_pool *pool);
103 bool xp_put_pool(struct xsk_buff_pool *pool);
104 void xp_clear_dev(struct xsk_buff_pool *pool);
105 void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
106 void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
111 static inline void xp_init_xskb_addr(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool,  in xp_init_xskb_addr()  argument
114 	xskb->orig_addr = addr;  in xp_init_xskb_addr()
115 	xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom;  in xp_init_xskb_addr()
118 static inline void xp_init_xskb_dma(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool,  in xp_init_xskb_dma()  argument
121 	xskb->frame_dma = (dma_pages[addr >> PAGE_SHIFT] & ~XSK_NEXT_PG_CONTIG_MASK) +  in xp_init_xskb_dma()
123 	xskb->dma = xskb->frame_dma + pool->headroom + XDP_PACKET_HEADROOM;  in xp_init_xskb_dma()
127 void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq);
128 int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
130 void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs);
131 struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
132 u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max);
133 bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
134 void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
135 dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
138 	return xskb->dma;  in xp_get_dma()
143 	return xskb->frame_dma;  in xp_get_frame_dma()
152 void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
154 static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool,  in xp_dma_sync_for_device()  argument
155 					  dma_addr_t dma, size_t size)  in xp_dma_sync_for_device()  argument
157 	if (!pool->dma_need_sync)  in xp_dma_sync_for_device()
160 	xp_dma_sync_for_device_slow(pool, dma, size);  in xp_dma_sync_for_device()
164  * The low 12-bits of the addr will be 0 since this is the page address, so we
170 static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,  in xp_desc_crosses_non_contig_pg()  argument
173 	bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE;  in xp_desc_crosses_non_contig_pg()
178 	if (pool->dma_pages_cnt) {  in xp_desc_crosses_non_contig_pg()
179 		return !(pool->dma_pages[addr >> PAGE_SHIFT] &  in xp_desc_crosses_non_contig_pg()
184 	return addr + len > pool->addrs_cnt;  in xp_desc_crosses_non_contig_pg()
187 static inline u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)  in xp_aligned_extract_addr()  argument
189 	return addr & pool->chunk_mask;  in xp_aligned_extract_addr()
208 static inline u32 xp_aligned_extract_idx(struct xsk_buff_pool *pool, u64 addr)  in xp_aligned_extract_idx()  argument
210 	return xp_aligned_extract_addr(pool, addr) >> pool->chunk_shift;  in xp_aligned_extract_idx()
215 	if (xskb->pool->unaligned)  in xp_release()
216 		xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;  in xp_release()
221 	u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;  in xp_get_handle()
223 	offset += xskb->pool->headroom;  in xp_get_handle()
224 	if (!xskb->pool->unaligned)  in xp_get_handle()
225 		return xskb->orig_addr + offset;  in xp_get_handle()
226 	return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);  in xp_get_handle()