88 static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp)  in xdp_buff_has_frags()  argument
90 	return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS);  in xdp_buff_has_frags()
93 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp)  in xdp_buff_set_frags_flag()  argument
95 	xdp->flags |= XDP_FLAGS_HAS_FRAGS;  in xdp_buff_set_frags_flag()
98 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp)  in xdp_buff_clear_frags_flag()  argument
100 	xdp->flags &= ~XDP_FLAGS_HAS_FRAGS;  in xdp_buff_clear_frags_flag()
103 static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp)  in xdp_buff_is_frag_pfmemalloc()  argument
105 	return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);  in xdp_buff_is_frag_pfmemalloc()
108 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp)  in xdp_buff_set_frag_pfmemalloc()  argument
110 	xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC;  in xdp_buff_set_frag_pfmemalloc()
114 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq)  in xdp_init_buff()  argument
116 	xdp->frame_sz = frame_sz;  in xdp_init_buff()
117 	xdp->rxq = rxq;  in xdp_init_buff()
118 	xdp->flags = 0;  in xdp_init_buff()
122 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start,  in xdp_prepare_buff()  argument
127 	xdp->data_hard_start = hard_start;  in xdp_prepare_buff()
128 	xdp->data = data;  in xdp_prepare_buff()
129 	xdp->data_end = data + data_len;  in xdp_prepare_buff()
130 	xdp->data_meta = meta_valid ? data : data + 1;  in xdp_prepare_buff()
139 #define xdp_data_hard_end(xdp)				\  argument
140 	((xdp)->data_hard_start + (xdp)->frame_sz -	\
144 xdp_get_shared_info_from_buff(struct xdp_buff *xdp)  in xdp_get_shared_info_from_buff()  argument
146 	return (struct skb_shared_info *)xdp_data_hard_end(xdp);  in xdp_get_shared_info_from_buff()
149 static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp)  in xdp_get_buff_len()  argument
151 	unsigned int len = xdp->data_end - xdp->data;  in xdp_get_buff_len()
154 	if (likely(!xdp_buff_has_frags(xdp)))  in xdp_get_buff_len()
157 	sinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_get_buff_len()
239 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
249 void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp)  in xdp_convert_frame_to_buff()  argument
251 	xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame);  in xdp_convert_frame_to_buff()
252 	xdp->data = frame->data;  in xdp_convert_frame_to_buff()
253 	xdp->data_end = frame->data + frame->len;  in xdp_convert_frame_to_buff()
254 	xdp->data_meta = frame->data - frame->metasize;  in xdp_convert_frame_to_buff()
255 	xdp->frame_sz = frame->frame_sz;  in xdp_convert_frame_to_buff()
256 	xdp->flags = frame->flags;  in xdp_convert_frame_to_buff()
260 int xdp_update_frame_from_buff(struct xdp_buff *xdp,  in xdp_update_frame_from_buff()  argument
266 	headroom = xdp->data - xdp->data_hard_start;  in xdp_update_frame_from_buff()
267 	metasize = xdp->data - xdp->data_meta;  in xdp_update_frame_from_buff()
273 	if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {  in xdp_update_frame_from_buff()
278 	xdp_frame->data = xdp->data;  in xdp_update_frame_from_buff()
279 	xdp_frame->len  = xdp->data_end - xdp->data;  in xdp_update_frame_from_buff()
282 	xdp_frame->frame_sz = xdp->frame_sz;  in xdp_update_frame_from_buff()
283 	xdp_frame->flags = xdp->flags;  in xdp_update_frame_from_buff()
290 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)  in xdp_convert_buff_to_frame()  argument
294 	if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)  in xdp_convert_buff_to_frame()
295 		return xdp_convert_zc_to_xdp_frame(xdp);  in xdp_convert_buff_to_frame()
298 	xdp_frame = xdp->data_hard_start;  in xdp_convert_buff_to_frame()
299 	if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0))  in xdp_convert_buff_to_frame()
303 	xdp_frame->mem = xdp->rxq->mem;  in xdp_convert_buff_to_frame()
309 		  struct xdp_buff *xdp);
312 void xdp_return_buff(struct xdp_buff *xdp);
385 xdp_set_data_meta_invalid(struct xdp_buff *xdp)  in xdp_set_data_meta_invalid()  argument
387 	xdp->data_meta = xdp->data + 1;  in xdp_set_data_meta_invalid()
391 xdp_data_meta_unsupported(const struct xdp_buff *xdp)  in xdp_data_meta_unsupported()  argument
393 	return unlikely(xdp->data_meta > xdp->data);  in xdp_data_meta_unsupported()