113 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
117 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
118 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
119 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
120 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
121 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
124 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
125 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
128 			      struct sk_msg *msg, u32 bytes);
130 			     struct sk_msg *msg, u32 bytes);
131 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
135 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)  in sk_msg_check_to_free()  argument
137 	WARN_ON(i == msg->sg.end && bytes);  in sk_msg_check_to_free()
170 #define sk_msg_iter_prev(msg, which)			\  argument
171 	sk_msg_iter_var_prev(msg->sg.which)
173 #define sk_msg_iter_next(msg, which)			\  argument
174 	sk_msg_iter_var_next(msg->sg.which)
176 static inline void sk_msg_init(struct sk_msg *msg)  in sk_msg_init()  argument
178 	BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);  in sk_msg_init()
179 	memset(msg, 0, sizeof(*msg));  in sk_msg_init()
180 	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);  in sk_msg_init()
200 static inline bool sk_msg_full(const struct sk_msg *msg)  in sk_msg_full()  argument
202 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;  in sk_msg_full()
205 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)  in sk_msg_elem_used()  argument
207 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end);  in sk_msg_elem_used()
210 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)  in sk_msg_elem()  argument
212 	return &msg->sg.data[which];  in sk_msg_elem()
215 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)  in sk_msg_elem_cpy()  argument
217 	return msg->sg.data[which];  in sk_msg_elem_cpy()
220 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)  in sk_msg_page()  argument
222 	return sg_page(sk_msg_elem(msg, which));  in sk_msg_page()
225 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)  in sk_msg_to_ingress()  argument
227 	return msg->flags & BPF_F_INGRESS;  in sk_msg_to_ingress()
230 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)  in sk_msg_compute_data_pointers()  argument
232 	struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);  in sk_msg_compute_data_pointers()
234 	if (test_bit(msg->sg.start, msg->sg.copy)) {  in sk_msg_compute_data_pointers()
235 		msg->data = NULL;  in sk_msg_compute_data_pointers()
236 		msg->data_end = NULL;  in sk_msg_compute_data_pointers()
238 		msg->data = sg_virt(sge);  in sk_msg_compute_data_pointers()
239 		msg->data_end = msg->data + sge->length;  in sk_msg_compute_data_pointers()
243 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,  in sk_msg_page_add()  argument
249 	sge = sk_msg_elem(msg, msg->sg.end);  in sk_msg_page_add()
253 	__set_bit(msg->sg.end, msg->sg.copy);  in sk_msg_page_add()
254 	msg->sg.size += len;  in sk_msg_page_add()
255 	sk_msg_iter_next(msg, end);  in sk_msg_page_add()
258 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)  in sk_msg_sg_copy()  argument
262 			__set_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
264 			__clear_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
266 		if (i == msg->sg.end)  in sk_msg_sg_copy()
271 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_set()  argument
273 	sk_msg_sg_copy(msg, start, true);  in sk_msg_sg_copy_set()
276 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_clear()  argument
278 	sk_msg_sg_copy(msg, start, false);  in sk_msg_sg_copy_clear()
312 				      struct sk_msg *msg)  in sk_psock_queue_msg()  argument
316 		list_add_tail(&msg->list, &psock->ingress_msg);  in sk_psock_queue_msg()
318 		sk_msg_free(psock->sk, msg);  in sk_psock_queue_msg()
319 		kfree(msg);  in sk_psock_queue_msg()
326 	struct sk_msg *msg;  in sk_psock_dequeue_msg()  local
329 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_dequeue_msg()
330 	if (msg)  in sk_psock_dequeue_msg()
331 		list_del(&msg->list);  in sk_psock_dequeue_msg()
333 	return msg;  in sk_psock_dequeue_msg()
338 	struct sk_msg *msg;  in sk_psock_peek_msg()  local
341 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_peek_msg()
343 	return msg;  in sk_psock_peek_msg()
347 					       struct sk_msg *msg)  in sk_psock_next_msg()  argument
352 	if (list_is_last(&msg->list, &psock->ingress_msg))  in sk_psock_next_msg()
355 		ret = list_next_entry(msg, list);  in sk_psock_next_msg()
365 static inline void kfree_sk_msg(struct sk_msg *msg)  in kfree_sk_msg()  argument
367 	if (msg->skb)  in kfree_sk_msg()
368 		consume_skb(msg->skb);  in kfree_sk_msg()
369 	kfree(msg);  in kfree_sk_msg()
406 			 struct sk_msg *msg);