108 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
112 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
113 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
114 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
115 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
116 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
119 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
120 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
123 			      struct sk_msg *msg, u32 bytes);
125 			     struct sk_msg *msg, u32 bytes);
127 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)  in sk_msg_check_to_free()  argument
129 	WARN_ON(i == msg->sg.end && bytes);  in sk_msg_check_to_free()
162 #define sk_msg_iter_prev(msg, which)			\  argument
163 	sk_msg_iter_var_prev(msg->sg.which)
165 #define sk_msg_iter_next(msg, which)			\  argument
166 	sk_msg_iter_var_next(msg->sg.which)
168 static inline void sk_msg_clear_meta(struct sk_msg *msg)  in sk_msg_clear_meta()  argument
170 	memset(&msg->sg, 0, offsetofend(struct sk_msg_sg, copy));  in sk_msg_clear_meta()
173 static inline void sk_msg_init(struct sk_msg *msg)  in sk_msg_init()  argument
175 	BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != MAX_MSG_FRAGS);  in sk_msg_init()
176 	memset(msg, 0, sizeof(*msg));  in sk_msg_init()
177 	sg_init_marker(msg->sg.data, MAX_MSG_FRAGS);  in sk_msg_init()
196 static inline bool sk_msg_full(const struct sk_msg *msg)  in sk_msg_full()  argument
198 	return (msg->sg.end == msg->sg.start) && msg->sg.size;  in sk_msg_full()
201 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)  in sk_msg_elem_used()  argument
203 	if (sk_msg_full(msg))  in sk_msg_elem_used()
206 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end);  in sk_msg_elem_used()
209 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)  in sk_msg_elem()  argument
211 	return &msg->sg.data[which];  in sk_msg_elem()
214 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)  in sk_msg_elem_cpy()  argument
216 	return msg->sg.data[which];  in sk_msg_elem_cpy()
219 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)  in sk_msg_page()  argument
221 	return sg_page(sk_msg_elem(msg, which));  in sk_msg_page()
224 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)  in sk_msg_to_ingress()  argument
226 	return msg->flags & BPF_F_INGRESS;  in sk_msg_to_ingress()
229 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)  in sk_msg_compute_data_pointers()  argument
231 	struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);  in sk_msg_compute_data_pointers()
233 	if (msg->sg.copy[msg->sg.start]) {  in sk_msg_compute_data_pointers()
234 		msg->data = NULL;  in sk_msg_compute_data_pointers()
235 		msg->data_end = NULL;  in sk_msg_compute_data_pointers()
237 		msg->data = sg_virt(sge);  in sk_msg_compute_data_pointers()
238 		msg->data_end = msg->data + sge->length;  in sk_msg_compute_data_pointers()
242 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,  in sk_msg_page_add()  argument
248 	sge = sk_msg_elem(msg, msg->sg.end);  in sk_msg_page_add()
252 	msg->sg.copy[msg->sg.end] = true;  in sk_msg_page_add()
253 	msg->sg.size += len;  in sk_msg_page_add()
254 	sk_msg_iter_next(msg, end);  in sk_msg_page_add()
257 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)  in sk_msg_sg_copy()  argument
260 		msg->sg.copy[i] = copy_state;  in sk_msg_sg_copy()
262 		if (i == msg->sg.end)  in sk_msg_sg_copy()
267 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_set()  argument
269 	sk_msg_sg_copy(msg, start, true);  in sk_msg_sg_copy_set()
272 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_clear()  argument
274 	sk_msg_sg_copy(msg, start, false);  in sk_msg_sg_copy_clear()
283 				      struct sk_msg *msg)  in sk_psock_queue_msg()  argument
285 	list_add_tail(&msg->list, &psock->ingress_msg);  in sk_psock_queue_msg()
308 			 struct sk_msg *msg);