Lines Matching refs:skb
97 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, in skb_panic() argument
101 msg, addr, skb->len, sz, skb->head, skb->data, in skb_panic()
102 (unsigned long)skb->tail, (unsigned long)skb->end, in skb_panic()
103 skb->dev ? skb->dev->name : "<NULL>"); in skb_panic()
107 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) in skb_over_panic() argument
109 skb_panic(skb, sz, addr, __func__); in skb_over_panic()
112 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) in skb_under_panic() argument
114 skb_panic(skb, sz, addr, __func__); in skb_under_panic()
182 struct sk_buff *skb; in __alloc_skb() local
193 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); in __alloc_skb()
194 if (!skb) in __alloc_skb()
196 prefetchw(skb); in __alloc_skb()
220 memset(skb, 0, offsetof(struct sk_buff, tail)); in __alloc_skb()
222 skb->truesize = SKB_TRUESIZE(size); in __alloc_skb()
223 skb->pfmemalloc = pfmemalloc; in __alloc_skb()
224 refcount_set(&skb->users, 1); in __alloc_skb()
225 skb->head = data; in __alloc_skb()
226 skb->data = data; in __alloc_skb()
227 skb_reset_tail_pointer(skb); in __alloc_skb()
228 skb->end = skb->tail + size; in __alloc_skb()
229 skb->mac_header = (typeof(skb->mac_header))~0U; in __alloc_skb()
230 skb->transport_header = (typeof(skb->transport_header))~0U; in __alloc_skb()
233 shinfo = skb_shinfo(skb); in __alloc_skb()
240 fclones = container_of(skb, struct sk_buff_fclones, skb1); in __alloc_skb()
242 skb->fclone = SKB_FCLONE_ORIG; in __alloc_skb()
248 return skb; in __alloc_skb()
250 kmem_cache_free(cache, skb); in __alloc_skb()
251 skb = NULL; in __alloc_skb()
278 struct sk_buff *skb; in __build_skb() local
281 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); in __build_skb()
282 if (!skb) in __build_skb()
287 memset(skb, 0, offsetof(struct sk_buff, tail)); in __build_skb()
288 skb->truesize = SKB_TRUESIZE(size); in __build_skb()
289 refcount_set(&skb->users, 1); in __build_skb()
290 skb->head = data; in __build_skb()
291 skb->data = data; in __build_skb()
292 skb_reset_tail_pointer(skb); in __build_skb()
293 skb->end = skb->tail + size; in __build_skb()
294 skb->mac_header = (typeof(skb->mac_header))~0U; in __build_skb()
295 skb->transport_header = (typeof(skb->transport_header))~0U; in __build_skb()
298 shinfo = skb_shinfo(skb); in __build_skb()
302 return skb; in __build_skb()
312 struct sk_buff *skb = __build_skb(data, frag_size); in build_skb() local
314 if (skb && frag_size) { in build_skb()
315 skb->head_frag = 1; in build_skb()
317 skb->pfmemalloc = 1; in build_skb()
319 return skb; in build_skb()
391 struct sk_buff *skb; in __netdev_alloc_skb() local
399 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); in __netdev_alloc_skb()
400 if (!skb) in __netdev_alloc_skb()
422 skb = __build_skb(data, len); in __netdev_alloc_skb()
423 if (unlikely(!skb)) { in __netdev_alloc_skb()
430 skb->pfmemalloc = 1; in __netdev_alloc_skb()
431 skb->head_frag = 1; in __netdev_alloc_skb()
434 skb_reserve(skb, NET_SKB_PAD); in __netdev_alloc_skb()
435 skb->dev = dev; in __netdev_alloc_skb()
438 return skb; in __netdev_alloc_skb()
459 struct sk_buff *skb; in __napi_alloc_skb() local
466 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); in __napi_alloc_skb()
467 if (!skb) in __napi_alloc_skb()
482 skb = __build_skb(data, len); in __napi_alloc_skb()
483 if (unlikely(!skb)) { in __napi_alloc_skb()
490 skb->pfmemalloc = 1; in __napi_alloc_skb()
491 skb->head_frag = 1; in __napi_alloc_skb()
494 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); in __napi_alloc_skb()
495 skb->dev = napi->dev; in __napi_alloc_skb()
498 return skb; in __napi_alloc_skb()
502 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, in skb_add_rx_frag() argument
505 skb_fill_page_desc(skb, i, page, off, size); in skb_add_rx_frag()
506 skb->len += size; in skb_add_rx_frag()
507 skb->data_len += size; in skb_add_rx_frag()
508 skb->truesize += truesize; in skb_add_rx_frag()
512 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, in skb_coalesce_rx_frag() argument
515 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in skb_coalesce_rx_frag()
518 skb->len += size; in skb_coalesce_rx_frag()
519 skb->data_len += size; in skb_coalesce_rx_frag()
520 skb->truesize += truesize; in skb_coalesce_rx_frag()
530 static inline void skb_drop_fraglist(struct sk_buff *skb) in skb_drop_fraglist() argument
532 skb_drop_list(&skb_shinfo(skb)->frag_list); in skb_drop_fraglist()
535 static void skb_clone_fraglist(struct sk_buff *skb) in skb_clone_fraglist() argument
539 skb_walk_frags(skb, list) in skb_clone_fraglist()
543 static void skb_free_head(struct sk_buff *skb) in skb_free_head() argument
545 unsigned char *head = skb->head; in skb_free_head()
547 if (skb->head_frag) in skb_free_head()
553 static void skb_release_data(struct sk_buff *skb) in skb_release_data() argument
555 struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_release_data()
558 if (skb->cloned && in skb_release_data()
559 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, in skb_release_data()
569 skb_zcopy_clear(skb, true); in skb_release_data()
570 skb_free_head(skb); in skb_release_data()
576 static void kfree_skbmem(struct sk_buff *skb) in kfree_skbmem() argument
580 switch (skb->fclone) { in kfree_skbmem()
582 kmem_cache_free(skbuff_head_cache, skb); in kfree_skbmem()
586 fclones = container_of(skb, struct sk_buff_fclones, skb1); in kfree_skbmem()
597 fclones = container_of(skb, struct sk_buff_fclones, skb2); in kfree_skbmem()
606 void skb_release_head_state(struct sk_buff *skb) in skb_release_head_state() argument
608 skb_dst_drop(skb); in skb_release_head_state()
609 secpath_reset(skb); in skb_release_head_state()
610 if (skb->destructor) { in skb_release_head_state()
612 skb->destructor(skb); in skb_release_head_state()
615 nf_conntrack_put(skb_nfct(skb)); in skb_release_head_state()
618 nf_bridge_put(skb->nf_bridge); in skb_release_head_state()
623 static void skb_release_all(struct sk_buff *skb) in skb_release_all() argument
625 skb_release_head_state(skb); in skb_release_all()
626 if (likely(skb->head)) in skb_release_all()
627 skb_release_data(skb); in skb_release_all()
639 void __kfree_skb(struct sk_buff *skb) in __kfree_skb() argument
641 skb_release_all(skb); in __kfree_skb()
642 kfree_skbmem(skb); in __kfree_skb()
653 void kfree_skb(struct sk_buff *skb) in kfree_skb() argument
655 if (!skb_unref(skb)) in kfree_skb()
658 trace_kfree_skb(skb, __builtin_return_address(0)); in kfree_skb()
659 __kfree_skb(skb); in kfree_skb()
681 void skb_tx_error(struct sk_buff *skb) in skb_tx_error() argument
683 skb_zcopy_clear(skb, true); in skb_tx_error()
695 void consume_skb(struct sk_buff *skb) in consume_skb() argument
697 if (!skb_unref(skb)) in consume_skb()
700 trace_consume_skb(skb); in consume_skb()
701 __kfree_skb(skb); in consume_skb()
712 void __consume_stateless_skb(struct sk_buff *skb) in __consume_stateless_skb() argument
714 trace_consume_skb(skb); in __consume_stateless_skb()
715 skb_release_data(skb); in __consume_stateless_skb()
716 kfree_skbmem(skb); in __consume_stateless_skb()
731 static inline void _kfree_skb_defer(struct sk_buff *skb) in _kfree_skb_defer() argument
736 skb_release_all(skb); in _kfree_skb_defer()
739 nc->skb_cache[nc->skb_count++] = skb; in _kfree_skb_defer()
743 prefetchw(skb); in _kfree_skb_defer()
753 void __kfree_skb_defer(struct sk_buff *skb) in __kfree_skb_defer() argument
755 _kfree_skb_defer(skb); in __kfree_skb_defer()
758 void napi_consume_skb(struct sk_buff *skb, int budget) in napi_consume_skb() argument
760 if (unlikely(!skb)) in napi_consume_skb()
765 dev_consume_skb_any(skb); in napi_consume_skb()
769 if (!skb_unref(skb)) in napi_consume_skb()
773 trace_consume_skb(skb); in napi_consume_skb()
776 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { in napi_consume_skb()
777 __kfree_skb(skb); in napi_consume_skb()
781 _kfree_skb_defer(skb); in napi_consume_skb()
846 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) in __skb_clone() argument
848 #define C(x) n->x = skb->x in __skb_clone()
852 __copy_skb_header(n, skb); in __skb_clone()
857 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; in __skb_clone()
871 atomic_inc(&(skb_shinfo(skb)->dataref)); in __skb_clone()
872 skb->cloned = 1; in __skb_clone()
938 struct sk_buff *skb; in sock_zerocopy_alloc() local
942 skb = sock_omalloc(sk, 0, GFP_KERNEL); in sock_zerocopy_alloc()
943 if (!skb) in sock_zerocopy_alloc()
946 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); in sock_zerocopy_alloc()
947 uarg = (void *)skb->cb; in sock_zerocopy_alloc()
951 kfree_skb(skb); in sock_zerocopy_alloc()
1012 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) in skb_zerocopy_notify_extend() argument
1014 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); in skb_zerocopy_notify_extend()
1034 struct sk_buff *tail, *skb = skb_from_uarg(uarg); in sock_zerocopy_callback() local
1036 struct sock *sk = skb->sk; in sock_zerocopy_callback()
1054 serr = SKB_EXT_ERR(skb); in sock_zerocopy_callback()
1068 __skb_queue_tail(q, skb); in sock_zerocopy_callback()
1069 skb = NULL; in sock_zerocopy_callback()
1076 consume_skb(skb); in sock_zerocopy_callback()
1105 extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
1108 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, in skb_zerocopy_iter_stream() argument
1112 struct ubuf_info *orig_uarg = skb_zcopy(skb); in skb_zerocopy_iter_stream()
1114 int err, orig_len = skb->len; in skb_zerocopy_iter_stream()
1122 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); in skb_zerocopy_iter_stream()
1123 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { in skb_zerocopy_iter_stream()
1124 struct sock *save_sk = skb->sk; in skb_zerocopy_iter_stream()
1128 skb->sk = sk; in skb_zerocopy_iter_stream()
1129 ___pskb_trim(skb, orig_len); in skb_zerocopy_iter_stream()
1130 skb->sk = save_sk; in skb_zerocopy_iter_stream()
1134 skb_zcopy_set(skb, uarg); in skb_zerocopy_iter_stream()
1135 return skb->len - orig_len; in skb_zerocopy_iter_stream()
1174 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) in skb_copy_ubufs() argument
1176 int num_frags = skb_shinfo(skb)->nr_frags; in skb_copy_ubufs()
1181 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) in skb_copy_ubufs()
1187 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; in skb_copy_ubufs()
1205 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; in skb_copy_ubufs()
1232 skb_frag_unref(skb, i); in skb_copy_ubufs()
1236 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); in skb_copy_ubufs()
1239 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); in skb_copy_ubufs()
1240 skb_shinfo(skb)->nr_frags = new_frags; in skb_copy_ubufs()
1243 skb_zcopy_clear(skb, false); in skb_copy_ubufs()
1262 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) in skb_clone() argument
1264 struct sk_buff_fclones *fclones = container_of(skb, in skb_clone()
1269 if (skb_orphan_frags(skb, gfp_mask)) in skb_clone()
1272 if (skb->fclone == SKB_FCLONE_ORIG && in skb_clone()
1277 if (skb_pfmemalloc(skb)) in skb_clone()
1287 return __skb_clone(n, skb); in skb_clone()
1291 void skb_headers_offset_update(struct sk_buff *skb, int off) in skb_headers_offset_update() argument
1294 if (skb->ip_summed == CHECKSUM_PARTIAL) in skb_headers_offset_update()
1295 skb->csum_start += off; in skb_headers_offset_update()
1297 skb->transport_header += off; in skb_headers_offset_update()
1298 skb->network_header += off; in skb_headers_offset_update()
1299 if (skb_mac_header_was_set(skb)) in skb_headers_offset_update()
1300 skb->mac_header += off; in skb_headers_offset_update()
1301 skb->inner_transport_header += off; in skb_headers_offset_update()
1302 skb->inner_network_header += off; in skb_headers_offset_update()
1303 skb->inner_mac_header += off; in skb_headers_offset_update()
1317 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) in skb_alloc_rx_flag() argument
1319 if (skb_pfmemalloc(skb)) in skb_alloc_rx_flag()
1341 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) in skb_copy() argument
1343 int headerlen = skb_headroom(skb); in skb_copy()
1344 unsigned int size = skb_end_offset(skb) + skb->data_len; in skb_copy()
1346 skb_alloc_rx_flag(skb), NUMA_NO_NODE); in skb_copy()
1354 skb_put(n, skb->len); in skb_copy()
1356 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); in skb_copy()
1358 skb_copy_header(n, skb); in skb_copy()
1380 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, in __pskb_copy_fclone() argument
1383 unsigned int size = skb_headlen(skb) + headroom; in __pskb_copy_fclone()
1384 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); in __pskb_copy_fclone()
1393 skb_put(n, skb_headlen(skb)); in __pskb_copy_fclone()
1395 skb_copy_from_linear_data(skb, n->data, n->len); in __pskb_copy_fclone()
1397 n->truesize += skb->data_len; in __pskb_copy_fclone()
1398 n->data_len = skb->data_len; in __pskb_copy_fclone()
1399 n->len = skb->len; in __pskb_copy_fclone()
1401 if (skb_shinfo(skb)->nr_frags) { in __pskb_copy_fclone()
1404 if (skb_orphan_frags(skb, gfp_mask) || in __pskb_copy_fclone()
1405 skb_zerocopy_clone(n, skb, gfp_mask)) { in __pskb_copy_fclone()
1410 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in __pskb_copy_fclone()
1411 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; in __pskb_copy_fclone()
1412 skb_frag_ref(skb, i); in __pskb_copy_fclone()
1417 if (skb_has_frag_list(skb)) { in __pskb_copy_fclone()
1418 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; in __pskb_copy_fclone()
1422 skb_copy_header(n, skb); in __pskb_copy_fclone()
1444 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, in pskb_expand_head() argument
1447 int i, osize = skb_end_offset(skb); in pskb_expand_head()
1454 BUG_ON(skb_shared(skb)); in pskb_expand_head()
1458 if (skb_pfmemalloc(skb)) in pskb_expand_head()
1469 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); in pskb_expand_head()
1472 skb_shinfo(skb), in pskb_expand_head()
1473 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); in pskb_expand_head()
1480 if (skb_cloned(skb)) { in pskb_expand_head()
1481 if (skb_orphan_frags(skb, gfp_mask)) in pskb_expand_head()
1483 if (skb_zcopy(skb)) in pskb_expand_head()
1484 refcount_inc(&skb_uarg(skb)->refcnt); in pskb_expand_head()
1485 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) in pskb_expand_head()
1486 skb_frag_ref(skb, i); in pskb_expand_head()
1488 if (skb_has_frag_list(skb)) in pskb_expand_head()
1489 skb_clone_fraglist(skb); in pskb_expand_head()
1491 skb_release_data(skb); in pskb_expand_head()
1493 skb_free_head(skb); in pskb_expand_head()
1495 off = (data + nhead) - skb->head; in pskb_expand_head()
1497 skb->head = data; in pskb_expand_head()
1498 skb->head_frag = 0; in pskb_expand_head()
1499 skb->data += off; in pskb_expand_head()
1501 skb->end = size; in pskb_expand_head()
1504 skb->end = skb->head + size; in pskb_expand_head()
1506 skb->tail += off; in pskb_expand_head()
1507 skb_headers_offset_update(skb, nhead); in pskb_expand_head()
1508 skb->cloned = 0; in pskb_expand_head()
1509 skb->hdr_len = 0; in pskb_expand_head()
1510 skb->nohdr = 0; in pskb_expand_head()
1511 atomic_set(&skb_shinfo(skb)->dataref, 1); in pskb_expand_head()
1513 skb_metadata_clear(skb); in pskb_expand_head()
1519 if (!skb->sk || skb->destructor == sock_edemux) in pskb_expand_head()
1520 skb->truesize += size - osize; in pskb_expand_head()
1533 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) in skb_realloc_headroom() argument
1536 int delta = headroom - skb_headroom(skb); in skb_realloc_headroom()
1539 skb2 = pskb_copy(skb, GFP_ATOMIC); in skb_realloc_headroom()
1541 skb2 = skb_clone(skb, GFP_ATOMIC); in skb_realloc_headroom()
1570 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, in skb_copy_expand() argument
1577 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, in skb_copy_expand()
1578 gfp_mask, skb_alloc_rx_flag(skb), in skb_copy_expand()
1580 int oldheadroom = skb_headroom(skb); in skb_copy_expand()
1589 skb_put(n, skb->len); in skb_copy_expand()
1599 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, in skb_copy_expand()
1600 skb->len + head_copy_len)); in skb_copy_expand()
1602 skb_copy_header(n, skb); in skb_copy_expand()
1624 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) in __skb_pad() argument
1630 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { in __skb_pad()
1631 memset(skb->data+skb->len, 0, pad); in __skb_pad()
1635 ntail = skb->data_len + pad - (skb->end - skb->tail); in __skb_pad()
1636 if (likely(skb_cloned(skb) || ntail > 0)) { in __skb_pad()
1637 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); in __skb_pad()
1645 err = skb_linearize(skb); in __skb_pad()
1649 memset(skb->data + skb->len, 0, pad); in __skb_pad()
1654 kfree_skb(skb); in __skb_pad()
1672 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) in pskb_put() argument
1674 if (tail != skb) { in pskb_put()
1675 skb->data_len += len; in pskb_put()
1676 skb->len += len; in pskb_put()
1691 void *skb_put(struct sk_buff *skb, unsigned int len) in skb_put() argument
1693 void *tmp = skb_tail_pointer(skb); in skb_put()
1694 SKB_LINEAR_ASSERT(skb); in skb_put()
1695 skb->tail += len; in skb_put()
1696 skb->len += len; in skb_put()
1697 if (unlikely(skb->tail > skb->end)) in skb_put()
1698 skb_over_panic(skb, len, __builtin_return_address(0)); in skb_put()
1712 void *skb_push(struct sk_buff *skb, unsigned int len) in skb_push() argument
1714 skb->data -= len; in skb_push()
1715 skb->len += len; in skb_push()
1716 if (unlikely(skb->data < skb->head)) in skb_push()
1717 skb_under_panic(skb, len, __builtin_return_address(0)); in skb_push()
1718 return skb->data; in skb_push()
1732 void *skb_pull(struct sk_buff *skb, unsigned int len) in skb_pull() argument
1734 return skb_pull_inline(skb, len); in skb_pull()
1747 void skb_trim(struct sk_buff *skb, unsigned int len) in skb_trim() argument
1749 if (skb->len > len) in skb_trim()
1750 __skb_trim(skb, len); in skb_trim()
1757 int ___pskb_trim(struct sk_buff *skb, unsigned int len) in ___pskb_trim() argument
1761 int offset = skb_headlen(skb); in ___pskb_trim()
1762 int nfrags = skb_shinfo(skb)->nr_frags; in ___pskb_trim()
1766 if (skb_cloned(skb) && in ___pskb_trim()
1767 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) in ___pskb_trim()
1775 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); in ___pskb_trim()
1782 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); in ___pskb_trim()
1785 skb_shinfo(skb)->nr_frags = i; in ___pskb_trim()
1788 skb_frag_unref(skb, i); in ___pskb_trim()
1790 if (skb_has_frag_list(skb)) in ___pskb_trim()
1791 skb_drop_fraglist(skb); in ___pskb_trim()
1795 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); in ___pskb_trim()
1827 if (len > skb_headlen(skb)) { in ___pskb_trim()
1828 skb->data_len -= skb->len - len; in ___pskb_trim()
1829 skb->len = len; in ___pskb_trim()
1831 skb->len = len; in ___pskb_trim()
1832 skb->data_len = 0; in ___pskb_trim()
1833 skb_set_tail_pointer(skb, len); in ___pskb_trim()
1836 if (!skb->sk || skb->destructor == sock_edemux) in ___pskb_trim()
1837 skb_condense(skb); in ___pskb_trim()
1844 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) in pskb_trim_rcsum_slow() argument
1846 if (skb->ip_summed == CHECKSUM_COMPLETE) { in pskb_trim_rcsum_slow()
1847 int delta = skb->len - len; in pskb_trim_rcsum_slow()
1849 skb->csum = csum_block_sub(skb->csum, in pskb_trim_rcsum_slow()
1850 skb_checksum(skb, len, delta, 0), in pskb_trim_rcsum_slow()
1853 return __pskb_trim(skb, len); in pskb_trim_rcsum_slow()
1882 void *__pskb_pull_tail(struct sk_buff *skb, int delta) in __pskb_pull_tail() argument
1888 int i, k, eat = (skb->tail + delta) - skb->end; in __pskb_pull_tail()
1890 if (eat > 0 || skb_cloned(skb)) { in __pskb_pull_tail()
1891 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, in __pskb_pull_tail()
1896 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), in __pskb_pull_tail()
1897 skb_tail_pointer(skb), delta)); in __pskb_pull_tail()
1902 if (!skb_has_frag_list(skb)) in __pskb_pull_tail()
1907 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in __pskb_pull_tail()
1908 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); in __pskb_pull_tail()
1923 struct sk_buff *list = skb_shinfo(skb)->frag_list; in __pskb_pull_tail()
1959 while ((list = skb_shinfo(skb)->frag_list) != insp) { in __pskb_pull_tail()
1960 skb_shinfo(skb)->frag_list = list->next; in __pskb_pull_tail()
1966 skb_shinfo(skb)->frag_list = clone; in __pskb_pull_tail()
1974 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in __pskb_pull_tail()
1975 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); in __pskb_pull_tail()
1978 skb_frag_unref(skb, i); in __pskb_pull_tail()
1981 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; in __pskb_pull_tail()
1983 skb_shinfo(skb)->frags[k].page_offset += eat; in __pskb_pull_tail()
1984 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); in __pskb_pull_tail()
1992 skb_shinfo(skb)->nr_frags = k; in __pskb_pull_tail()
1995 skb->tail += delta; in __pskb_pull_tail()
1996 skb->data_len -= delta; in __pskb_pull_tail()
1998 if (!skb->data_len) in __pskb_pull_tail()
1999 skb_zcopy_clear(skb, false); in __pskb_pull_tail()
2001 return skb_tail_pointer(skb); in __pskb_pull_tail()
2020 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) in skb_copy_bits() argument
2022 int start = skb_headlen(skb); in skb_copy_bits()
2026 if (offset > (int)skb->len - len) in skb_copy_bits()
2033 skb_copy_from_linear_data_offset(skb, offset, to, copy); in skb_copy_bits()
2040 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in skb_copy_bits()
2042 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; in skb_copy_bits()
2071 skb_walk_frags(skb, frag_iter) { in skb_copy_bits()
2205 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, in __skb_splice_bits() argument
2217 if (__splice_segment(virt_to_page(skb->data), in __skb_splice_bits()
2218 (unsigned long) skb->data & (PAGE_SIZE - 1), in __skb_splice_bits()
2219 skb_headlen(skb), in __skb_splice_bits()
2221 skb_head_is_locked(skb), in __skb_splice_bits()
2228 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { in __skb_splice_bits()
2229 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; in __skb_splice_bits()
2237 skb_walk_frags(skb, iter) { in __skb_splice_bits()
2257 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, in skb_splice_bits() argument
2272 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); in skb_splice_bits()
2282 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, in skb_send_sock_locked() argument
2286 struct sk_buff *head = skb; in skb_send_sock_locked()
2293 while (offset < skb_headlen(skb) && len) { in skb_send_sock_locked()
2297 slen = min_t(int, len, skb_headlen(skb) - offset); in skb_send_sock_locked()
2298 kv.iov_base = skb->data + offset; in skb_send_sock_locked()
2315 offset -= skb_headlen(skb); in skb_send_sock_locked()
2318 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { in skb_send_sock_locked()
2319 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; in skb_send_sock_locked()
2327 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { in skb_send_sock_locked()
2328 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; in skb_send_sock_locked()
2350 if (skb == head) { in skb_send_sock_locked()
2351 if (skb_has_frag_list(skb)) { in skb_send_sock_locked()
2352 skb = skb_shinfo(skb)->frag_list; in skb_send_sock_locked()
2355 } else if (skb->next) { in skb_send_sock_locked()
2356 skb = skb->next; in skb_send_sock_locked()
2370 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) in skb_send_sock() argument
2375 ret = skb_send_sock_locked(sk, skb, offset, len); in skb_send_sock()
2394 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) in skb_store_bits() argument
2396 int start = skb_headlen(skb); in skb_store_bits()
2400 if (offset > (int)skb->len - len) in skb_store_bits()
2406 skb_copy_to_linear_data_offset(skb, offset, from, copy); in skb_store_bits()
2413 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in skb_store_bits()
2414 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in skb_store_bits()
2444 skb_walk_frags(skb, frag_iter) { in skb_store_bits()
2472 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, in __skb_checksum() argument
2475 int start = skb_headlen(skb); in __skb_checksum()
2484 csum = ops->update(skb->data + offset, copy, csum); in __skb_checksum()
2491 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in __skb_checksum()
2493 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in __skb_checksum()
2524 skb_walk_frags(skb, frag_iter) { in __skb_checksum()
2550 __wsum skb_checksum(const struct sk_buff *skb, int offset, in skb_checksum() argument
2558 return __skb_checksum(skb, offset, len, csum, &ops); in skb_checksum()
2564 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, in skb_copy_and_csum_bits() argument
2567 int start = skb_headlen(skb); in skb_copy_and_csum_bits()
2576 csum = csum_partial_copy_nocheck(skb->data + offset, to, in skb_copy_and_csum_bits()
2585 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in skb_copy_and_csum_bits()
2590 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); in skb_copy_and_csum_bits()
2592 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in skb_copy_and_csum_bits()
2621 skb_walk_frags(skb, frag_iter) { in skb_copy_and_csum_bits()
2773 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) in skb_copy_and_csum_dev() argument
2778 if (skb->ip_summed == CHECKSUM_PARTIAL) in skb_copy_and_csum_dev()
2779 csstart = skb_checksum_start_offset(skb); in skb_copy_and_csum_dev()
2781 csstart = skb_headlen(skb); in skb_copy_and_csum_dev()
2783 BUG_ON(csstart > skb_headlen(skb)); in skb_copy_and_csum_dev()
2785 skb_copy_from_linear_data(skb, to, csstart); in skb_copy_and_csum_dev()
2788 if (csstart != skb->len) in skb_copy_and_csum_dev()
2789 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, in skb_copy_and_csum_dev()
2790 skb->len - csstart, 0); in skb_copy_and_csum_dev()
2792 if (skb->ip_summed == CHECKSUM_PARTIAL) { in skb_copy_and_csum_dev()
2793 long csstuff = csstart + skb->csum_offset; in skb_copy_and_csum_dev()
2851 struct sk_buff *skb; in skb_queue_purge() local
2852 while ((skb = skb_dequeue(list)) != NULL) in skb_queue_purge()
2853 kfree_skb(skb); in skb_queue_purge()
2873 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); in skb_rbtree_purge() local
2876 rb_erase(&skb->rbnode, root); in skb_rbtree_purge()
2877 sum += skb->truesize; in skb_rbtree_purge()
2878 kfree_skb(skb); in skb_rbtree_purge()
2935 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) in skb_unlink() argument
2940 __skb_unlink(skb, list); in skb_unlink()
2987 static inline void skb_split_inside_header(struct sk_buff *skb, in skb_split_inside_header() argument
2993 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), in skb_split_inside_header()
2996 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) in skb_split_inside_header()
2997 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; in skb_split_inside_header()
2999 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; in skb_split_inside_header()
3000 skb_shinfo(skb)->nr_frags = 0; in skb_split_inside_header()
3001 skb1->data_len = skb->data_len; in skb_split_inside_header()
3003 skb->data_len = 0; in skb_split_inside_header()
3004 skb->len = len; in skb_split_inside_header()
3005 skb_set_tail_pointer(skb, len); in skb_split_inside_header()
3008 static inline void skb_split_no_header(struct sk_buff *skb, in skb_split_no_header() argument
3013 const int nfrags = skb_shinfo(skb)->nr_frags; in skb_split_no_header()
3015 skb_shinfo(skb)->nr_frags = 0; in skb_split_no_header()
3016 skb1->len = skb1->data_len = skb->len - len; in skb_split_no_header()
3017 skb->len = len; in skb_split_no_header()
3018 skb->data_len = len - pos; in skb_split_no_header()
3021 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); in skb_split_no_header()
3024 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; in skb_split_no_header()
3035 skb_frag_ref(skb, i); in skb_split_no_header()
3038 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); in skb_split_no_header()
3039 skb_shinfo(skb)->nr_frags++; in skb_split_no_header()
3043 skb_shinfo(skb)->nr_frags++; in skb_split_no_header()
3055 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) in skb_split() argument
3057 int pos = skb_headlen(skb); in skb_split()
3059 skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags & in skb_split()
3061 skb_zerocopy_clone(skb1, skb, 0); in skb_split()
3063 skb_split_inside_header(skb, skb1, len, pos); in skb_split()
3065 skb_split_no_header(skb, skb1, len, pos); in skb_split()
3073 static int skb_prepare_for_shift(struct sk_buff *skb) in skb_prepare_for_shift() argument
3075 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); in skb_prepare_for_shift()
3096 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) in skb_shift() argument
3101 BUG_ON(shiftlen > skb->len); in skb_shift()
3103 if (skb_headlen(skb)) in skb_shift()
3105 if (skb_zcopy(tgt) || skb_zcopy(skb)) in skb_shift()
3111 fragfrom = &skb_shinfo(skb)->frags[from]; in skb_shift()
3125 if (skb_prepare_for_shift(skb) || in skb_shift()
3130 fragfrom = &skb_shinfo(skb)->frags[from]; in skb_shift()
3144 if ((shiftlen == skb->len) && in skb_shift()
3145 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) in skb_shift()
3148 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) in skb_shift()
3151 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { in skb_shift()
3155 fragfrom = &skb_shinfo(skb)->frags[from]; in skb_shift()
3183 fragfrom = &skb_shinfo(skb)->frags[0]; in skb_shift()
3192 while (from < skb_shinfo(skb)->nr_frags) in skb_shift()
3193 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; in skb_shift()
3194 skb_shinfo(skb)->nr_frags = to; in skb_shift()
3196 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); in skb_shift()
3203 skb->ip_summed = CHECKSUM_PARTIAL; in skb_shift()
3206 skb->len -= shiftlen; in skb_shift()
3207 skb->data_len -= shiftlen; in skb_shift()
3208 skb->truesize -= shiftlen; in skb_shift()
3226 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, in skb_prepare_seq_read() argument
3231 st->root_skb = st->cur_skb = skb; in skb_prepare_seq_read()
3369 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, in skb_find_text() argument
3378 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); in skb_find_text()
3396 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, in skb_append_datato_frags() argument
3398 int len, int odd, struct sk_buff *skb), in skb_append_datato_frags() argument
3401 int frg_cnt = skb_shinfo(skb)->nr_frags; in skb_append_datato_frags()
3419 offset, copy, 0, skb); in skb_append_datato_frags()
3424 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset, in skb_append_datato_frags()
3430 skb->truesize += copy; in skb_append_datato_frags()
3432 skb->len += copy; in skb_append_datato_frags()
3433 skb->data_len += copy; in skb_append_datato_frags()
3443 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, in skb_append_pagefrags() argument
3446 int i = skb_shinfo(skb)->nr_frags; in skb_append_pagefrags()
3448 if (skb_can_coalesce(skb, i, page, offset)) { in skb_append_pagefrags()
3449 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); in skb_append_pagefrags()
3452 skb_fill_page_desc(skb, i, page, offset, size); in skb_append_pagefrags()
3472 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) in skb_pull_rcsum() argument
3474 unsigned char *data = skb->data; in skb_pull_rcsum()
3476 BUG_ON(len > skb->len); in skb_pull_rcsum()
3477 __skb_pull(skb, len); in skb_pull_rcsum()
3478 skb_postpull_rcsum(skb, data, len); in skb_pull_rcsum()
3479 return skb->data; in skb_pull_rcsum()
3822 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) in skb_gro_receive() argument
3824 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); in skb_gro_receive()
3825 unsigned int offset = skb_gro_offset(skb); in skb_gro_receive()
3826 unsigned int headlen = skb_headlen(skb); in skb_gro_receive()
3827 unsigned int len = skb_gro_len(skb); in skb_gro_receive()
3860 delta_truesize = skb->truesize - in skb_gro_receive()
3861 SKB_TRUESIZE(skb_end_offset(skb)); in skb_gro_receive()
3863 skb->truesize -= skb->data_len; in skb_gro_receive()
3864 skb->len -= skb->data_len; in skb_gro_receive()
3865 skb->data_len = 0; in skb_gro_receive()
3867 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; in skb_gro_receive()
3869 } else if (skb->head_frag) { in skb_gro_receive()
3872 struct page *page = virt_to_head_page(skb->head); in skb_gro_receive()
3879 first_offset = skb->data - in skb_gro_receive()
3892 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); in skb_gro_receive()
3893 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; in skb_gro_receive()
3898 delta_truesize = skb->truesize; in skb_gro_receive()
3904 skb->data_len -= eat; in skb_gro_receive()
3905 skb->len -= eat; in skb_gro_receive()
3909 __skb_pull(skb, offset); in skb_gro_receive()
3912 skb_shinfo(p)->frag_list = skb; in skb_gro_receive()
3914 NAPI_GRO_CB(p)->last->next = skb; in skb_gro_receive()
3915 NAPI_GRO_CB(p)->last = skb; in skb_gro_receive()
3916 __skb_header_release(skb); in skb_gro_receive()
3929 NAPI_GRO_CB(skb)->same_flow = 1; in skb_gro_receive()
3951 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, in __skb_to_sgvec() argument
3954 int start = skb_headlen(skb); in __skb_to_sgvec()
3965 sg_set_buf(sg, skb->data + offset, copy); in __skb_to_sgvec()
3972 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in __skb_to_sgvec()
3977 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); in __skb_to_sgvec()
3979 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in __skb_to_sgvec()
3995 skb_walk_frags(skb, frag_iter) { in __skb_to_sgvec()
4034 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) in skb_to_sgvec() argument
4036 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); in skb_to_sgvec()
4066 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, in skb_to_sgvec_nomark() argument
4069 return __skb_to_sgvec(skb, sg, offset, len, 0); in skb_to_sgvec_nomark()
4092 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) in skb_cow_data() argument
4102 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && in skb_cow_data()
4103 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) in skb_cow_data()
4107 if (!skb_has_frag_list(skb)) { in skb_cow_data()
4113 if (skb_tailroom(skb) < tailbits && in skb_cow_data()
4114 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) in skb_cow_data()
4118 *trailer = skb; in skb_cow_data()
4125 skb_p = &skb_shinfo(skb)->frag_list; in skb_cow_data()
4185 static void sock_rmem_free(struct sk_buff *skb) in sock_rmem_free() argument
4187 struct sock *sk = skb->sk; in sock_rmem_free()
4189 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); in sock_rmem_free()
4192 static void skb_set_err_queue(struct sk_buff *skb) in skb_set_err_queue() argument
4197 skb->pkt_type = PACKET_OUTGOING; in skb_set_err_queue()
4204 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) in sock_queue_err_skb() argument
4206 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= in sock_queue_err_skb()
4210 skb_orphan(skb); in sock_queue_err_skb()
4211 skb->sk = sk; in sock_queue_err_skb()
4212 skb->destructor = sock_rmem_free; in sock_queue_err_skb()
4213 atomic_add(skb->truesize, &sk->sk_rmem_alloc); in sock_queue_err_skb()
4214 skb_set_err_queue(skb); in sock_queue_err_skb()
4217 skb_dst_force(skb); in sock_queue_err_skb()
4219 skb_queue_tail(&sk->sk_error_queue, skb); in sock_queue_err_skb()
4226 static bool is_icmp_err_skb(const struct sk_buff *skb) in is_icmp_err_skb() argument
4228 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || in is_icmp_err_skb()
4229 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); in is_icmp_err_skb()
4235 struct sk_buff *skb, *skb_next = NULL; in sock_dequeue_err_skb() local
4240 skb = __skb_dequeue(q); in sock_dequeue_err_skb()
4241 if (skb && (skb_next = skb_peek(q))) { in sock_dequeue_err_skb()
4248 if (is_icmp_err_skb(skb) && !icmp_next) in sock_dequeue_err_skb()
4254 return skb; in sock_dequeue_err_skb()
4271 struct sk_buff *skb_clone_sk(struct sk_buff *skb) in skb_clone_sk() argument
4273 struct sock *sk = skb->sk; in skb_clone_sk()
4279 clone = skb_clone(skb, GFP_ATOMIC); in skb_clone_sk()
4292 static void __skb_complete_tx_timestamp(struct sk_buff *skb, in __skb_complete_tx_timestamp() argument
4300 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); in __skb_complete_tx_timestamp()
4302 serr = SKB_EXT_ERR(skb); in __skb_complete_tx_timestamp()
4308 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; in __skb_complete_tx_timestamp()
4310 serr->ee.ee_data = skb_shinfo(skb)->tskey; in __skb_complete_tx_timestamp()
4316 err = sock_queue_err_skb(sk, skb); in __skb_complete_tx_timestamp()
4319 kfree_skb(skb); in __skb_complete_tx_timestamp()
4336 void skb_complete_tx_timestamp(struct sk_buff *skb, in skb_complete_tx_timestamp() argument
4339 struct sock *sk = skb->sk; in skb_complete_tx_timestamp()
4348 *skb_hwtstamps(skb) = *hwtstamps; in skb_complete_tx_timestamp()
4349 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); in skb_complete_tx_timestamp()
4355 kfree_skb(skb); in skb_complete_tx_timestamp()
4363 struct sk_buff *skb; in __skb_tstamp_tx() local
4382 skb = tcp_get_timestamping_opt_stats(sk); in __skb_tstamp_tx()
4386 skb = alloc_skb(0, GFP_ATOMIC); in __skb_tstamp_tx()
4388 skb = skb_clone(orig_skb, GFP_ATOMIC); in __skb_tstamp_tx()
4390 if (!skb) in __skb_tstamp_tx()
4394 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & in __skb_tstamp_tx()
4396 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; in __skb_tstamp_tx()
4400 *skb_hwtstamps(skb) = *hwtstamps; in __skb_tstamp_tx()
4402 skb->tstamp = ktime_get_real(); in __skb_tstamp_tx()
4404 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); in __skb_tstamp_tx()
4416 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) in skb_complete_wifi_ack() argument
4418 struct sock *sk = skb->sk; in skb_complete_wifi_ack()
4422 skb->wifi_acked_valid = 1; in skb_complete_wifi_ack()
4423 skb->wifi_acked = acked; in skb_complete_wifi_ack()
4425 serr = SKB_EXT_ERR(skb); in skb_complete_wifi_ack()
4434 err = sock_queue_err_skb(sk, skb); in skb_complete_wifi_ack()
4438 kfree_skb(skb); in skb_complete_wifi_ack()
4454 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) in skb_partial_csum_set() argument
4457 u32 csum_start = skb_headroom(skb) + (u32)start; in skb_partial_csum_set()
4459 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) { in skb_partial_csum_set()
4461 start, off, skb_headroom(skb), skb_headlen(skb)); in skb_partial_csum_set()
4464 skb->ip_summed = CHECKSUM_PARTIAL; in skb_partial_csum_set()
4465 skb->csum_start = csum_start; in skb_partial_csum_set()
4466 skb->csum_offset = off; in skb_partial_csum_set()
4467 skb_set_transport_header(skb, start); in skb_partial_csum_set()
4472 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, in skb_maybe_pull_tail() argument
4475 if (skb_headlen(skb) >= len) in skb_maybe_pull_tail()
4481 if (max > skb->len) in skb_maybe_pull_tail()
4482 max = skb->len; in skb_maybe_pull_tail()
4484 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) in skb_maybe_pull_tail()
4487 if (skb_headlen(skb) < len) in skb_maybe_pull_tail()
4495 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, in skb_checksum_setup_ip() argument
4503 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), in skb_checksum_setup_ip()
4505 if (!err && !skb_partial_csum_set(skb, off, in skb_checksum_setup_ip()
4509 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; in skb_checksum_setup_ip()
4512 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), in skb_checksum_setup_ip()
4514 if (!err && !skb_partial_csum_set(skb, off, in skb_checksum_setup_ip()
4518 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; in skb_checksum_setup_ip()
4529 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) in skb_checksum_setup_ipv4() argument
4538 err = skb_maybe_pull_tail(skb, in skb_checksum_setup_ipv4()
4544 if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) in skb_checksum_setup_ipv4()
4547 off = ip_hdrlen(skb); in skb_checksum_setup_ipv4()
4554 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); in skb_checksum_setup_ipv4()
4559 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, in skb_checksum_setup_ipv4()
4560 ip_hdr(skb)->daddr, in skb_checksum_setup_ipv4()
4561 skb->len - off, in skb_checksum_setup_ipv4()
4562 ip_hdr(skb)->protocol, 0); in skb_checksum_setup_ipv4()
4574 #define OPT_HDR(type, skb, off) \ argument
4575 (type *)(skb_network_header(skb) + (off))
4577 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) in skb_checksum_setup_ipv6() argument
4592 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); in skb_checksum_setup_ipv6()
4596 nexthdr = ipv6_hdr(skb)->nexthdr; in skb_checksum_setup_ipv6()
4598 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); in skb_checksum_setup_ipv6()
4606 err = skb_maybe_pull_tail(skb, in skb_checksum_setup_ipv6()
4613 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); in skb_checksum_setup_ipv6()
4621 err = skb_maybe_pull_tail(skb, in skb_checksum_setup_ipv6()
4628 hp = OPT_HDR(struct ip_auth_hdr, skb, off); in skb_checksum_setup_ipv6()
4636 err = skb_maybe_pull_tail(skb, in skb_checksum_setup_ipv6()
4643 hp = OPT_HDR(struct frag_hdr, skb, off); in skb_checksum_setup_ipv6()
4663 csum = skb_checksum_setup_ip(skb, nexthdr, off); in skb_checksum_setup_ipv6()
4668 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, in skb_checksum_setup_ipv6()
4669 &ipv6_hdr(skb)->daddr, in skb_checksum_setup_ipv6()
4670 skb->len - off, nexthdr, 0); in skb_checksum_setup_ipv6()
4682 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) in skb_checksum_setup() argument
4686 switch (skb->protocol) { in skb_checksum_setup()
4688 err = skb_checksum_setup_ipv4(skb, recalculate); in skb_checksum_setup()
4692 err = skb_checksum_setup_ipv6(skb, recalculate); in skb_checksum_setup()
4717 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, in skb_checksum_maybe_trim() argument
4721 unsigned int len = skb_transport_offset(skb) + transport_len; in skb_checksum_maybe_trim()
4724 if (skb->len < len) in skb_checksum_maybe_trim()
4726 else if (skb->len == len) in skb_checksum_maybe_trim()
4727 return skb; in skb_checksum_maybe_trim()
4729 skb_chk = skb_clone(skb, GFP_ATOMIC); in skb_checksum_maybe_trim()
4757 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, in skb_checksum_trimmed() argument
4759 __sum16(*skb_chkf)(struct sk_buff *skb)) in skb_checksum_trimmed() argument
4762 unsigned int offset = skb_transport_offset(skb); in skb_checksum_trimmed()
4765 skb_chk = skb_checksum_maybe_trim(skb, transport_len); in skb_checksum_trimmed()
4782 if (skb_chk && skb_chk != skb) in skb_checksum_trimmed()
4790 void __skb_warn_lro_forwarding(const struct sk_buff *skb) in __skb_warn_lro_forwarding() argument
4793 skb->dev->name); in __skb_warn_lro_forwarding()
4797 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) in kfree_skb_partial() argument
4800 skb_release_head_state(skb); in kfree_skb_partial()
4801 kmem_cache_free(skbuff_head_cache, skb); in kfree_skb_partial()
4803 __kfree_skb(skb); in kfree_skb_partial()
4905 void skb_scrub_packet(struct sk_buff *skb, bool xnet) in skb_scrub_packet() argument
4907 skb->pkt_type = PACKET_HOST; in skb_scrub_packet()
4908 skb->skb_iif = 0; in skb_scrub_packet()
4909 skb->ignore_df = 0; in skb_scrub_packet()
4910 skb_dst_drop(skb); in skb_scrub_packet()
4911 secpath_reset(skb); in skb_scrub_packet()
4912 nf_reset(skb); in skb_scrub_packet()
4913 nf_reset_trace(skb); in skb_scrub_packet()
4918 ipvs_reset(skb); in skb_scrub_packet()
4919 skb->mark = 0; in skb_scrub_packet()
4920 skb->tstamp = 0; in skb_scrub_packet()
4934 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) in skb_gso_transport_seglen() argument
4936 const struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_gso_transport_seglen()
4939 if (skb->encapsulation) { in skb_gso_transport_seglen()
4940 thlen = skb_inner_transport_header(skb) - in skb_gso_transport_seglen()
4941 skb_transport_header(skb); in skb_gso_transport_seglen()
4944 thlen += inner_tcp_hdrlen(skb); in skb_gso_transport_seglen()
4946 thlen = tcp_hdrlen(skb); in skb_gso_transport_seglen()
4947 } else if (unlikely(skb_is_gso_sctp(skb))) { in skb_gso_transport_seglen()
4969 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb) in skb_gso_network_seglen() argument
4971 unsigned int hdr_len = skb_transport_header(skb) - in skb_gso_network_seglen()
4972 skb_network_header(skb); in skb_gso_network_seglen()
4974 return hdr_len + skb_gso_transport_seglen(skb); in skb_gso_network_seglen()
4986 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb) in skb_gso_mac_seglen() argument
4988 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb); in skb_gso_mac_seglen()
4990 return hdr_len + skb_gso_transport_seglen(skb); in skb_gso_mac_seglen()
5012 static inline bool skb_gso_size_check(const struct sk_buff *skb, in skb_gso_size_check() argument
5015 const struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_gso_size_check()
5024 skb_walk_frags(skb, iter) { in skb_gso_size_check()
5042 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu) in skb_gso_validate_network_len() argument
5044 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu); in skb_gso_validate_network_len()
5057 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len) in skb_gso_validate_mac_len() argument
5059 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len); in skb_gso_validate_mac_len()
5063 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) in skb_reorder_vlan_header() argument
5067 if (skb_cow(skb, skb_headroom(skb)) < 0) { in skb_reorder_vlan_header()
5068 kfree_skb(skb); in skb_reorder_vlan_header()
5072 mac_len = skb->data - skb_mac_header(skb); in skb_reorder_vlan_header()
5074 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), in skb_reorder_vlan_header()
5077 skb->mac_header += VLAN_HLEN; in skb_reorder_vlan_header()
5078 return skb; in skb_reorder_vlan_header()
5081 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) in skb_vlan_untag() argument
5086 if (unlikely(skb_vlan_tag_present(skb))) { in skb_vlan_untag()
5088 return skb; in skb_vlan_untag()
5091 skb = skb_share_check(skb, GFP_ATOMIC); in skb_vlan_untag()
5092 if (unlikely(!skb)) in skb_vlan_untag()
5095 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) in skb_vlan_untag()
5098 vhdr = (struct vlan_hdr *)skb->data; in skb_vlan_untag()
5100 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); in skb_vlan_untag()
5102 skb_pull_rcsum(skb, VLAN_HLEN); in skb_vlan_untag()
5103 vlan_set_encap_proto(skb, vhdr); in skb_vlan_untag()
5105 skb = skb_reorder_vlan_header(skb); in skb_vlan_untag()
5106 if (unlikely(!skb)) in skb_vlan_untag()
5109 skb_reset_network_header(skb); in skb_vlan_untag()
5110 skb_reset_transport_header(skb); in skb_vlan_untag()
5111 skb_reset_mac_len(skb); in skb_vlan_untag()
5113 return skb; in skb_vlan_untag()
5116 kfree_skb(skb); in skb_vlan_untag()
5121 int skb_ensure_writable(struct sk_buff *skb, int write_len) in skb_ensure_writable() argument
5123 if (!pskb_may_pull(skb, write_len)) in skb_ensure_writable()
5126 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) in skb_ensure_writable()
5129 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); in skb_ensure_writable()
5136 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) in __skb_vlan_pop() argument
5139 int offset = skb->data - skb_mac_header(skb); in __skb_vlan_pop()
5148 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); in __skb_vlan_pop()
5152 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); in __skb_vlan_pop()
5154 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); in __skb_vlan_pop()
5157 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); in __skb_vlan_pop()
5158 __skb_pull(skb, VLAN_HLEN); in __skb_vlan_pop()
5160 vlan_set_encap_proto(skb, vhdr); in __skb_vlan_pop()
5161 skb->mac_header += VLAN_HLEN; in __skb_vlan_pop()
5163 if (skb_network_offset(skb) < ETH_HLEN) in __skb_vlan_pop()
5164 skb_set_network_header(skb, ETH_HLEN); in __skb_vlan_pop()
5166 skb_reset_mac_len(skb); in __skb_vlan_pop()
5175 int skb_vlan_pop(struct sk_buff *skb) in skb_vlan_pop() argument
5181 if (likely(skb_vlan_tag_present(skb))) { in skb_vlan_pop()
5182 skb->vlan_tci = 0; in skb_vlan_pop()
5184 if (unlikely(!eth_type_vlan(skb->protocol))) in skb_vlan_pop()
5187 err = __skb_vlan_pop(skb, &vlan_tci); in skb_vlan_pop()
5192 if (likely(!eth_type_vlan(skb->protocol))) in skb_vlan_pop()
5195 vlan_proto = skb->protocol; in skb_vlan_pop()
5196 err = __skb_vlan_pop(skb, &vlan_tci); in skb_vlan_pop()
5200 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); in skb_vlan_pop()
5208 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) in skb_vlan_push() argument
5210 if (skb_vlan_tag_present(skb)) { in skb_vlan_push()
5211 int offset = skb->data - skb_mac_header(skb); in skb_vlan_push()
5220 err = __vlan_insert_tag(skb, skb->vlan_proto, in skb_vlan_push()
5221 skb_vlan_tag_get(skb)); in skb_vlan_push()
5225 skb->protocol = skb->vlan_proto; in skb_vlan_push()
5226 skb->mac_len += VLAN_HLEN; in skb_vlan_push()
5228 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); in skb_vlan_push()
5230 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); in skb_vlan_push()
5254 struct sk_buff *skb; in alloc_skb_with_frags() local
5271 skb = alloc_skb(header_len, gfp_head); in alloc_skb_with_frags()
5272 if (!skb) in alloc_skb_with_frags()
5275 skb->truesize += npages << PAGE_SHIFT; in alloc_skb_with_frags()
5300 skb_fill_page_desc(skb, i, page, 0, chunk); in alloc_skb_with_frags()
5304 return skb; in alloc_skb_with_frags()
5307 kfree_skb(skb); in alloc_skb_with_frags()
5313 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, in pskb_carve_inside_header() argument
5317 int size = skb_end_offset(skb); in pskb_carve_inside_header()
5323 if (skb_pfmemalloc(skb)) in pskb_carve_inside_header()
5334 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); in pskb_carve_inside_header()
5335 skb->len -= off; in pskb_carve_inside_header()
5338 skb_shinfo(skb), in pskb_carve_inside_header()
5340 frags[skb_shinfo(skb)->nr_frags])); in pskb_carve_inside_header()
5341 if (skb_cloned(skb)) { in pskb_carve_inside_header()
5343 if (skb_orphan_frags(skb, gfp_mask)) { in pskb_carve_inside_header()
5347 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) in pskb_carve_inside_header()
5348 skb_frag_ref(skb, i); in pskb_carve_inside_header()
5349 if (skb_has_frag_list(skb)) in pskb_carve_inside_header()
5350 skb_clone_fraglist(skb); in pskb_carve_inside_header()
5351 skb_release_data(skb); in pskb_carve_inside_header()
5356 skb_free_head(skb); in pskb_carve_inside_header()
5359 skb->head = data; in pskb_carve_inside_header()
5360 skb->data = data; in pskb_carve_inside_header()
5361 skb->head_frag = 0; in pskb_carve_inside_header()
5363 skb->end = size; in pskb_carve_inside_header()
5365 skb->end = skb->head + size; in pskb_carve_inside_header()
5367 skb_set_tail_pointer(skb, skb_headlen(skb)); in pskb_carve_inside_header()
5368 skb_headers_offset_update(skb, 0); in pskb_carve_inside_header()
5369 skb->cloned = 0; in pskb_carve_inside_header()
5370 skb->hdr_len = 0; in pskb_carve_inside_header()
5371 skb->nohdr = 0; in pskb_carve_inside_header()
5372 atomic_set(&skb_shinfo(skb)->dataref, 1); in pskb_carve_inside_header()
5377 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
5382 static int pskb_carve_frag_list(struct sk_buff *skb, in pskb_carve_frag_list() argument
5436 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, in pskb_carve_inside_nonlinear() argument
5440 int size = skb_end_offset(skb); in pskb_carve_inside_nonlinear()
5442 const int nfrags = skb_shinfo(skb)->nr_frags; in pskb_carve_inside_nonlinear()
5447 if (skb_pfmemalloc(skb)) in pskb_carve_inside_nonlinear()
5458 skb_shinfo(skb), offsetof(struct skb_shared_info, in pskb_carve_inside_nonlinear()
5459 frags[skb_shinfo(skb)->nr_frags])); in pskb_carve_inside_nonlinear()
5460 if (skb_orphan_frags(skb, gfp_mask)) { in pskb_carve_inside_nonlinear()
5466 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); in pskb_carve_inside_nonlinear()
5469 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; in pskb_carve_inside_nonlinear()
5483 skb_frag_ref(skb, i); in pskb_carve_inside_nonlinear()
5489 if (skb_has_frag_list(skb)) in pskb_carve_inside_nonlinear()
5490 skb_clone_fraglist(skb); in pskb_carve_inside_nonlinear()
5494 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask); in pskb_carve_inside_nonlinear()
5496 skb_release_data(skb); in pskb_carve_inside_nonlinear()
5498 skb->head = data; in pskb_carve_inside_nonlinear()
5499 skb->head_frag = 0; in pskb_carve_inside_nonlinear()
5500 skb->data = data; in pskb_carve_inside_nonlinear()
5502 skb->end = size; in pskb_carve_inside_nonlinear()
5504 skb->end = skb->head + size; in pskb_carve_inside_nonlinear()
5506 skb_reset_tail_pointer(skb); in pskb_carve_inside_nonlinear()
5507 skb_headers_offset_update(skb, 0); in pskb_carve_inside_nonlinear()
5508 skb->cloned = 0; in pskb_carve_inside_nonlinear()
5509 skb->hdr_len = 0; in pskb_carve_inside_nonlinear()
5510 skb->nohdr = 0; in pskb_carve_inside_nonlinear()
5511 skb->len -= off; in pskb_carve_inside_nonlinear()
5512 skb->data_len = skb->len; in pskb_carve_inside_nonlinear()
5513 atomic_set(&skb_shinfo(skb)->dataref, 1); in pskb_carve_inside_nonlinear()
5518 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) in pskb_carve() argument
5520 int headlen = skb_headlen(skb); in pskb_carve()
5523 return pskb_carve_inside_header(skb, len, headlen, gfp); in pskb_carve()
5525 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); in pskb_carve()
5531 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, in pskb_extract() argument
5534 struct sk_buff *clone = skb_clone(skb, gfp); in pskb_extract()
5560 void skb_condense(struct sk_buff *skb) in skb_condense() argument
5562 if (skb->data_len) { in skb_condense()
5563 if (skb->data_len > skb->end - skb->tail || in skb_condense()
5564 skb_cloned(skb)) in skb_condense()
5568 __pskb_pull_tail(skb, skb->data_len); in skb_condense()
5577 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); in skb_condense()