111 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
115 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
116 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
117 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
118 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
119 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
122 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
123 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
126 			      struct sk_msg *msg, u32 bytes);
128 			     struct sk_msg *msg, u32 bytes);
129 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
133 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)  in sk_msg_check_to_free()  argument
135 	WARN_ON(i == msg->sg.end && bytes);  in sk_msg_check_to_free()
168 #define sk_msg_iter_prev(msg, which)			\  argument
169 	sk_msg_iter_var_prev(msg->sg.which)
171 #define sk_msg_iter_next(msg, which)			\  argument
172 	sk_msg_iter_var_next(msg->sg.which)
174 static inline void sk_msg_init(struct sk_msg *msg)  in sk_msg_init()  argument
176 	BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);  in sk_msg_init()
177 	memset(msg, 0, sizeof(*msg));  in sk_msg_init()
178 	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);  in sk_msg_init()
198 static inline bool sk_msg_full(const struct sk_msg *msg)  in sk_msg_full()  argument
200 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;  in sk_msg_full()
203 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)  in sk_msg_elem_used()  argument
205 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end);  in sk_msg_elem_used()
208 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)  in sk_msg_elem()  argument
210 	return &msg->sg.data[which];  in sk_msg_elem()
213 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)  in sk_msg_elem_cpy()  argument
215 	return msg->sg.data[which];  in sk_msg_elem_cpy()
218 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)  in sk_msg_page()  argument
220 	return sg_page(sk_msg_elem(msg, which));  in sk_msg_page()
223 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)  in sk_msg_to_ingress()  argument
225 	return msg->flags & BPF_F_INGRESS;  in sk_msg_to_ingress()
228 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)  in sk_msg_compute_data_pointers()  argument
230 	struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);  in sk_msg_compute_data_pointers()
232 	if (test_bit(msg->sg.start, msg->sg.copy)) {  in sk_msg_compute_data_pointers()
233 		msg->data = NULL;  in sk_msg_compute_data_pointers()
234 		msg->data_end = NULL;  in sk_msg_compute_data_pointers()
236 		msg->data = sg_virt(sge);  in sk_msg_compute_data_pointers()
237 		msg->data_end = msg->data + sge->length;  in sk_msg_compute_data_pointers()
241 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,  in sk_msg_page_add()  argument
247 	sge = sk_msg_elem(msg, msg->sg.end);  in sk_msg_page_add()
251 	__set_bit(msg->sg.end, msg->sg.copy);  in sk_msg_page_add()
252 	msg->sg.size += len;  in sk_msg_page_add()
253 	sk_msg_iter_next(msg, end);  in sk_msg_page_add()
256 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)  in sk_msg_sg_copy()  argument
260 			__set_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
262 			__clear_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
264 		if (i == msg->sg.end)  in sk_msg_sg_copy()
269 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_set()  argument
271 	sk_msg_sg_copy(msg, start, true);  in sk_msg_sg_copy_set()
274 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_clear()  argument
276 	sk_msg_sg_copy(msg, start, false);  in sk_msg_sg_copy_clear()
310 				      struct sk_msg *msg)  in sk_psock_queue_msg()  argument
314 		list_add_tail(&msg->list, &psock->ingress_msg);  in sk_psock_queue_msg()
316 		sk_msg_free(psock->sk, msg);  in sk_psock_queue_msg()
317 		kfree(msg);  in sk_psock_queue_msg()
324 	struct sk_msg *msg;  in sk_psock_dequeue_msg()  local
327 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_dequeue_msg()
328 	if (msg)  in sk_psock_dequeue_msg()
329 		list_del(&msg->list);  in sk_psock_dequeue_msg()
331 	return msg;  in sk_psock_dequeue_msg()
336 	struct sk_msg *msg;  in sk_psock_peek_msg()  local
339 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_peek_msg()
341 	return msg;  in sk_psock_peek_msg()
345 					       struct sk_msg *msg)  in sk_psock_next_msg()  argument
350 	if (list_is_last(&msg->list, &psock->ingress_msg))  in sk_psock_next_msg()
353 		ret = list_next_entry(msg, list);  in sk_psock_next_msg()
363 static inline void kfree_sk_msg(struct sk_msg *msg)  in kfree_sk_msg()  argument
365 	if (msg->skb)  in kfree_sk_msg()
366 		consume_skb(msg->skb);  in kfree_sk_msg()
367 	kfree(msg);  in kfree_sk_msg()
404 			 struct sk_msg *msg);