1097 	struct ubuf_info *uarg;  in sock_zerocopy_alloc()  local
1106 	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));  in sock_zerocopy_alloc()
1107 	uarg = (void *)skb->cb;  in sock_zerocopy_alloc()
1108 	uarg->mmp.user = NULL;  in sock_zerocopy_alloc()
1110 	if (mm_account_pinned_pages(&uarg->mmp, size)) {  in sock_zerocopy_alloc()
1115 	uarg->callback = sock_zerocopy_callback;  in sock_zerocopy_alloc()
1116 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;  in sock_zerocopy_alloc()
1117 	uarg->len = 1;  in sock_zerocopy_alloc()
1118 	uarg->bytelen = size;  in sock_zerocopy_alloc()
1119 	uarg->zerocopy = 1;  in sock_zerocopy_alloc()
1120 	refcount_set(&uarg->refcnt, 1);  in sock_zerocopy_alloc()
1123 	return uarg;  in sock_zerocopy_alloc()
1127 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)  in skb_from_uarg()  argument
1129 	return container_of((void *)uarg, struct sk_buff, cb);  in skb_from_uarg()
1133 					struct ubuf_info *uarg)  in sock_zerocopy_realloc()  argument
1135 	if (uarg) {  in sock_zerocopy_realloc()
1147 		bytelen = uarg->bytelen + size;  in sock_zerocopy_realloc()
1148 		if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {  in sock_zerocopy_realloc()
1156 		if ((u32)(uarg->id + uarg->len) == next) {  in sock_zerocopy_realloc()
1157 			if (mm_account_pinned_pages(&uarg->mmp, size))  in sock_zerocopy_realloc()
1159 			uarg->len++;  in sock_zerocopy_realloc()
1160 			uarg->bytelen = bytelen;  in sock_zerocopy_realloc()
1165 				sock_zerocopy_get(uarg);  in sock_zerocopy_realloc()
1167 			return uarg;  in sock_zerocopy_realloc()
1196 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)  in sock_zerocopy_callback()  argument
1198 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);  in sock_zerocopy_callback()
1206 	mm_unaccount_pinned_pages(&uarg->mmp);  in sock_zerocopy_callback()
1211 	if (!uarg->len || sock_flag(sk, SOCK_DEAD))  in sock_zerocopy_callback()
1214 	len = uarg->len;  in sock_zerocopy_callback()
1215 	lo = uarg->id;  in sock_zerocopy_callback()
1216 	hi = uarg->id + len - 1;  in sock_zerocopy_callback()
1245 void sock_zerocopy_put(struct ubuf_info *uarg)  in sock_zerocopy_put()  argument
1247 	if (uarg && refcount_dec_and_test(&uarg->refcnt)) {  in sock_zerocopy_put()
1248 		if (uarg->callback)  in sock_zerocopy_put()
1249 			uarg->callback(uarg, uarg->zerocopy);  in sock_zerocopy_put()
1251 			consume_skb(skb_from_uarg(uarg));  in sock_zerocopy_put()
1256 void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)  in sock_zerocopy_put_abort()  argument
1258 	if (uarg) {  in sock_zerocopy_put_abort()
1259 		struct sock *sk = skb_from_uarg(uarg)->sk;  in sock_zerocopy_put_abort()
1262 		uarg->len--;  in sock_zerocopy_put_abort()
1265 			sock_zerocopy_put(uarg);  in sock_zerocopy_put_abort()
1278 			     struct ubuf_info *uarg)  in skb_zerocopy_iter_stream()  argument
1287 	if (orig_uarg && uarg != orig_uarg)  in skb_zerocopy_iter_stream()
1302 	skb_zcopy_set(skb, uarg, NULL);  in skb_zerocopy_iter_stream()