Lines Matching refs:skb
486 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
705 void (*destructor)(struct sk_buff *skb);
871 static inline bool skb_pfmemalloc(const struct sk_buff *skb) in skb_pfmemalloc() argument
873 return unlikely(skb->pfmemalloc); in skb_pfmemalloc()
890 static inline struct dst_entry *skb_dst(const struct sk_buff *skb) in skb_dst() argument
895 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) && in skb_dst()
898 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK); in skb_dst()
909 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst) in skb_dst_set() argument
911 skb->_skb_refdst = (unsigned long)dst; in skb_dst_set()
924 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst) in skb_dst_set_noref() argument
927 skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; in skb_dst_set_noref()
934 static inline bool skb_dst_is_noref(const struct sk_buff *skb) in skb_dst_is_noref() argument
936 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb); in skb_dst_is_noref()
939 static inline struct rtable *skb_rtable(const struct sk_buff *skb) in skb_rtable() argument
941 return (struct rtable *)skb_dst(skb); in skb_rtable()
953 static inline unsigned int skb_napi_id(const struct sk_buff *skb) in skb_napi_id() argument
956 return skb->napi_id; in skb_napi_id()
963 static inline bool skb_unref(struct sk_buff *skb) in skb_unref() argument
965 if (unlikely(!skb)) in skb_unref()
967 if (likely(refcount_read(&skb->users) == 1)) in skb_unref()
969 else if (likely(!refcount_dec_and_test(&skb->users))) in skb_unref()
975 void skb_release_head_state(struct sk_buff *skb);
976 void kfree_skb(struct sk_buff *skb);
978 void skb_tx_error(struct sk_buff *skb);
979 void consume_skb(struct sk_buff *skb);
980 void __consume_stateless_skb(struct sk_buff *skb);
981 void __kfree_skb(struct sk_buff *skb);
984 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1023 const struct sk_buff *skb) in skb_fclone_busy() argument
1027 fclones = container_of(skb, struct sk_buff_fclones, skb1); in skb_fclone_busy()
1029 return skb->fclone == SKB_FCLONE_ORIG && in skb_fclone_busy()
1041 void skb_headers_offset_update(struct sk_buff *skb, int off);
1042 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
1043 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
1045 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
1046 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1048 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, in __pskb_copy() argument
1051 return __pskb_copy_fclone(skb, headroom, gfp_mask, false); in __pskb_copy()
1054 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
1055 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
1057 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
1059 int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
1061 int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
1063 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
1064 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
1077 static inline int skb_pad(struct sk_buff *skb, int pad) in skb_pad() argument
1079 return __skb_pad(skb, pad, true); in skb_pad()
1083 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
1085 int len, int odd, struct sk_buff *skb),
1088 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
1101 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1107 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1143 static inline void skb_clear_hash(struct sk_buff *skb) in skb_clear_hash() argument
1145 skb->hash = 0; in skb_clear_hash()
1146 skb->sw_hash = 0; in skb_clear_hash()
1147 skb->l4_hash = 0; in skb_clear_hash()
1150 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb) in skb_clear_hash_if_not_l4() argument
1152 if (!skb->l4_hash) in skb_clear_hash_if_not_l4()
1153 skb_clear_hash(skb); in skb_clear_hash_if_not_l4()
1157 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4) in __skb_set_hash() argument
1159 skb->l4_hash = is_l4; in __skb_set_hash()
1160 skb->sw_hash = is_sw; in __skb_set_hash()
1161 skb->hash = hash; in __skb_set_hash()
1165 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type) in skb_set_hash() argument
1168 __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4); in skb_set_hash()
1172 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4) in __skb_set_sw_hash() argument
1174 __skb_set_hash(skb, hash, true, is_l4); in __skb_set_sw_hash()
1177 void __skb_get_hash(struct sk_buff *skb);
1178 u32 __skb_get_hash_symmetric(const struct sk_buff *skb);
1179 u32 skb_get_poff(const struct sk_buff *skb);
1180 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
1182 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
1185 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb, in skb_flow_get_ports() argument
1188 return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0); in skb_flow_get_ports()
1195 bool __skb_flow_dissect(const struct sk_buff *skb,
1201 static inline bool skb_flow_dissect(const struct sk_buff *skb, in skb_flow_dissect() argument
1205 return __skb_flow_dissect(skb, flow_dissector, target_container, in skb_flow_dissect()
1209 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb, in skb_flow_dissect_flow_keys() argument
1214 return __skb_flow_dissect(skb, &flow_keys_dissector, flow, in skb_flow_dissect_flow_keys()
1219 skb_flow_dissect_flow_keys_basic(const struct sk_buff *skb, in skb_flow_dissect_flow_keys_basic() argument
1225 return __skb_flow_dissect(skb, &flow_keys_basic_dissector, flow, in skb_flow_dissect_flow_keys_basic()
1230 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
1234 static inline __u32 skb_get_hash(struct sk_buff *skb) in skb_get_hash() argument
1236 if (!skb->l4_hash && !skb->sw_hash) in skb_get_hash()
1237 __skb_get_hash(skb); in skb_get_hash()
1239 return skb->hash; in skb_get_hash()
1242 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) in skb_get_hash_flowi6() argument
1244 if (!skb->l4_hash && !skb->sw_hash) { in skb_get_hash_flowi6()
1248 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); in skb_get_hash_flowi6()
1251 return skb->hash; in skb_get_hash_flowi6()
1254 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb);
1256 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb) in skb_get_hash_raw() argument
1258 return skb->hash; in skb_get_hash_raw()
1269 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) in skb_end_pointer() argument
1271 return skb->head + skb->end; in skb_end_pointer()
1274 static inline unsigned int skb_end_offset(const struct sk_buff *skb) in skb_end_offset() argument
1276 return skb->end; in skb_end_offset()
1279 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) in skb_end_pointer() argument
1281 return skb->end; in skb_end_pointer()
1284 static inline unsigned int skb_end_offset(const struct sk_buff *skb) in skb_end_offset() argument
1286 return skb->end - skb->head; in skb_end_offset()
1293 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) in skb_hwtstamps() argument
1295 return &skb_shinfo(skb)->hwtstamps; in skb_hwtstamps()
1298 static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb) in skb_zcopy() argument
1300 bool is_zcopy = skb && skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY; in skb_zcopy()
1302 return is_zcopy ? skb_uarg(skb) : NULL; in skb_zcopy()
1305 static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg) in skb_zcopy_set() argument
1307 if (skb && uarg && !skb_zcopy(skb)) { in skb_zcopy_set()
1309 skb_shinfo(skb)->destructor_arg = uarg; in skb_zcopy_set()
1310 skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG; in skb_zcopy_set()
1315 static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy) in skb_zcopy_clear() argument
1317 struct ubuf_info *uarg = skb_zcopy(skb); in skb_zcopy_clear()
1327 skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG; in skb_zcopy_clear()
1332 static inline void skb_zcopy_abort(struct sk_buff *skb) in skb_zcopy_abort() argument
1334 struct ubuf_info *uarg = skb_zcopy(skb); in skb_zcopy_abort()
1338 skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG; in skb_zcopy_abort()
1361 const struct sk_buff *skb) in skb_queue_is_last() argument
1363 return skb->next == (const struct sk_buff *) list; in skb_queue_is_last()
1374 const struct sk_buff *skb) in skb_queue_is_first() argument
1376 return skb->prev == (const struct sk_buff *) list; in skb_queue_is_first()
1388 const struct sk_buff *skb) in skb_queue_next() argument
1393 BUG_ON(skb_queue_is_last(list, skb)); in skb_queue_next()
1394 return skb->next; in skb_queue_next()
1406 const struct sk_buff *skb) in skb_queue_prev() argument
1411 BUG_ON(skb_queue_is_first(list, skb)); in skb_queue_prev()
1412 return skb->prev; in skb_queue_prev()
1422 static inline struct sk_buff *skb_get(struct sk_buff *skb) in skb_get() argument
1424 refcount_inc(&skb->users); in skb_get()
1425 return skb; in skb_get()
1440 static inline int skb_cloned(const struct sk_buff *skb) in skb_cloned() argument
1442 return skb->cloned && in skb_cloned()
1443 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1; in skb_cloned()
1446 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) in skb_unclone() argument
1450 if (skb_cloned(skb)) in skb_unclone()
1451 return pskb_expand_head(skb, 0, 0, pri); in skb_unclone()
1463 static inline int skb_header_cloned(const struct sk_buff *skb) in skb_header_cloned() argument
1467 if (!skb->cloned) in skb_header_cloned()
1470 dataref = atomic_read(&skb_shinfo(skb)->dataref); in skb_header_cloned()
1475 static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri) in skb_header_unclone() argument
1479 if (skb_header_cloned(skb)) in skb_header_unclone()
1480 return pskb_expand_head(skb, 0, 0, pri); in skb_header_unclone()
1489 static inline void __skb_header_release(struct sk_buff *skb) in __skb_header_release() argument
1491 skb->nohdr = 1; in __skb_header_release()
1492 atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT)); in __skb_header_release()
1503 static inline int skb_shared(const struct sk_buff *skb) in skb_shared() argument
1505 return refcount_read(&skb->users) != 1; in skb_shared()
1521 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri) in skb_share_check() argument
1524 if (skb_shared(skb)) { in skb_share_check()
1525 struct sk_buff *nskb = skb_clone(skb, pri); in skb_share_check()
1528 consume_skb(skb); in skb_share_check()
1530 kfree_skb(skb); in skb_share_check()
1531 skb = nskb; in skb_share_check()
1533 return skb; in skb_share_check()
1556 static inline struct sk_buff *skb_unshare(struct sk_buff *skb, in skb_unshare() argument
1560 if (skb_cloned(skb)) { in skb_unshare()
1561 struct sk_buff *nskb = skb_copy(skb, pri); in skb_unshare()
1565 consume_skb(skb); in skb_unshare()
1567 kfree_skb(skb); in skb_unshare()
1568 skb = nskb; in skb_unshare()
1570 return skb; in skb_unshare()
1588 struct sk_buff *skb = list_->next; in skb_peek() local
1590 if (skb == (struct sk_buff *)list_) in skb_peek()
1591 skb = NULL; in skb_peek()
1592 return skb; in skb_peek()
1604 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb, in skb_peek_next() argument
1607 struct sk_buff *next = skb->next; in skb_peek_next()
1629 struct sk_buff *skb = list_->prev; in skb_peek_tail() local
1631 if (skb == (struct sk_buff *)list_) in skb_peek_tail()
1632 skb = NULL; in skb_peek_tail()
1633 return skb; in skb_peek_tail()
1846 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
1847 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) in __skb_unlink() argument
1852 next = skb->next; in __skb_unlink()
1853 prev = skb->prev; in __skb_unlink()
1854 skb->next = skb->prev = NULL; in __skb_unlink()
1870 struct sk_buff *skb = skb_peek(list); in __skb_dequeue() local
1871 if (skb) in __skb_dequeue()
1872 __skb_unlink(skb, list); in __skb_dequeue()
1873 return skb; in __skb_dequeue()
1887 struct sk_buff *skb = skb_peek_tail(list); in __skb_dequeue_tail() local
1888 if (skb) in __skb_dequeue_tail()
1889 __skb_unlink(skb, list); in __skb_dequeue_tail()
1890 return skb; in __skb_dequeue_tail()
1894 static inline bool skb_is_nonlinear(const struct sk_buff *skb) in skb_is_nonlinear() argument
1896 return skb->data_len; in skb_is_nonlinear()
1899 static inline unsigned int skb_headlen(const struct sk_buff *skb) in skb_headlen() argument
1901 return skb->len - skb->data_len; in skb_headlen()
1904 static inline unsigned int __skb_pagelen(const struct sk_buff *skb) in __skb_pagelen() argument
1908 for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--) in __skb_pagelen()
1909 len += skb_frag_size(&skb_shinfo(skb)->frags[i]); in __skb_pagelen()
1913 static inline unsigned int skb_pagelen(const struct sk_buff *skb) in skb_pagelen() argument
1915 return skb_headlen(skb) + __skb_pagelen(skb); in skb_pagelen()
1931 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, in __skb_fill_page_desc() argument
1934 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in __skb_fill_page_desc()
1947 skb->pfmemalloc = true; in __skb_fill_page_desc()
1964 static inline void skb_fill_page_desc(struct sk_buff *skb, int i, in skb_fill_page_desc() argument
1967 __skb_fill_page_desc(skb, i, page, off, size); in skb_fill_page_desc()
1968 skb_shinfo(skb)->nr_frags = i + 1; in skb_fill_page_desc()
1971 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
1974 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
1977 #define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags) argument
1978 #define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb)) argument
1979 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb)) argument
1982 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) in skb_tail_pointer() argument
1984 return skb->head + skb->tail; in skb_tail_pointer()
1987 static inline void skb_reset_tail_pointer(struct sk_buff *skb) in skb_reset_tail_pointer() argument
1989 skb->tail = skb->data - skb->head; in skb_reset_tail_pointer()
1992 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) in skb_set_tail_pointer() argument
1994 skb_reset_tail_pointer(skb); in skb_set_tail_pointer()
1995 skb->tail += offset; in skb_set_tail_pointer()
1999 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) in skb_tail_pointer() argument
2001 return skb->tail; in skb_tail_pointer()
2004 static inline void skb_reset_tail_pointer(struct sk_buff *skb) in skb_reset_tail_pointer() argument
2006 skb->tail = skb->data; in skb_reset_tail_pointer()
2009 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) in skb_set_tail_pointer() argument
2011 skb->tail = skb->data + offset; in skb_set_tail_pointer()
2019 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
2020 void *skb_put(struct sk_buff *skb, unsigned int len);
2021 static inline void *__skb_put(struct sk_buff *skb, unsigned int len) in __skb_put() argument
2023 void *tmp = skb_tail_pointer(skb); in __skb_put()
2024 SKB_LINEAR_ASSERT(skb); in __skb_put()
2025 skb->tail += len; in __skb_put()
2026 skb->len += len; in __skb_put()
2030 static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len) in __skb_put_zero() argument
2032 void *tmp = __skb_put(skb, len); in __skb_put_zero()
2038 static inline void *__skb_put_data(struct sk_buff *skb, const void *data, in __skb_put_data() argument
2041 void *tmp = __skb_put(skb, len); in __skb_put_data()
2047 static inline void __skb_put_u8(struct sk_buff *skb, u8 val) in __skb_put_u8() argument
2049 *(u8 *)__skb_put(skb, 1) = val; in __skb_put_u8()
2052 static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len) in skb_put_zero() argument
2054 void *tmp = skb_put(skb, len); in skb_put_zero()
2061 static inline void *skb_put_data(struct sk_buff *skb, const void *data, in skb_put_data() argument
2064 void *tmp = skb_put(skb, len); in skb_put_data()
2071 static inline void skb_put_u8(struct sk_buff *skb, u8 val) in skb_put_u8() argument
2073 *(u8 *)skb_put(skb, 1) = val; in skb_put_u8()
2076 void *skb_push(struct sk_buff *skb, unsigned int len);
2077 static inline void *__skb_push(struct sk_buff *skb, unsigned int len) in __skb_push() argument
2079 skb->data -= len; in __skb_push()
2080 skb->len += len; in __skb_push()
2081 return skb->data; in __skb_push()
2084 void *skb_pull(struct sk_buff *skb, unsigned int len);
2085 static inline void *__skb_pull(struct sk_buff *skb, unsigned int len) in __skb_pull() argument
2087 skb->len -= len; in __skb_pull()
2088 BUG_ON(skb->len < skb->data_len); in __skb_pull()
2089 return skb->data += len; in __skb_pull()
2092 static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len) in skb_pull_inline() argument
2094 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); in skb_pull_inline()
2097 void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2099 static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len) in __pskb_pull() argument
2101 if (len > skb_headlen(skb) && in __pskb_pull()
2102 !__pskb_pull_tail(skb, len - skb_headlen(skb))) in __pskb_pull()
2104 skb->len -= len; in __pskb_pull()
2105 return skb->data += len; in __pskb_pull()
2108 static inline void *pskb_pull(struct sk_buff *skb, unsigned int len) in pskb_pull() argument
2110 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len); in pskb_pull()
2113 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len) in pskb_may_pull() argument
2115 if (likely(len <= skb_headlen(skb))) in pskb_may_pull()
2117 if (unlikely(len > skb->len)) in pskb_may_pull()
2119 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; in pskb_may_pull()
2122 void skb_condense(struct sk_buff *skb);
2130 static inline unsigned int skb_headroom(const struct sk_buff *skb) in skb_headroom() argument
2132 return skb->data - skb->head; in skb_headroom()
2141 static inline int skb_tailroom(const struct sk_buff *skb) in skb_tailroom() argument
2143 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail; in skb_tailroom()
2153 static inline int skb_availroom(const struct sk_buff *skb) in skb_availroom() argument
2155 if (skb_is_nonlinear(skb)) in skb_availroom()
2158 return skb->end - skb->tail - skb->reserved_tailroom; in skb_availroom()
2169 static inline void skb_reserve(struct sk_buff *skb, int len) in skb_reserve() argument
2171 skb->data += len; in skb_reserve()
2172 skb->tail += len; in skb_reserve()
2187 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu, in skb_tailroom_reserve() argument
2190 SKB_LINEAR_ASSERT(skb); in skb_tailroom_reserve()
2191 if (mtu < skb_tailroom(skb) - needed_tailroom) in skb_tailroom_reserve()
2193 skb->reserved_tailroom = skb_tailroom(skb) - mtu; in skb_tailroom_reserve()
2196 skb->reserved_tailroom = needed_tailroom; in skb_tailroom_reserve()
2202 static inline void skb_set_inner_protocol(struct sk_buff *skb, in skb_set_inner_protocol() argument
2205 skb->inner_protocol = protocol; in skb_set_inner_protocol()
2206 skb->inner_protocol_type = ENCAP_TYPE_ETHER; in skb_set_inner_protocol()
2209 static inline void skb_set_inner_ipproto(struct sk_buff *skb, in skb_set_inner_ipproto() argument
2212 skb->inner_ipproto = ipproto; in skb_set_inner_ipproto()
2213 skb->inner_protocol_type = ENCAP_TYPE_IPPROTO; in skb_set_inner_ipproto()
2216 static inline void skb_reset_inner_headers(struct sk_buff *skb) in skb_reset_inner_headers() argument
2218 skb->inner_mac_header = skb->mac_header; in skb_reset_inner_headers()
2219 skb->inner_network_header = skb->network_header; in skb_reset_inner_headers()
2220 skb->inner_transport_header = skb->transport_header; in skb_reset_inner_headers()
2223 static inline void skb_reset_mac_len(struct sk_buff *skb) in skb_reset_mac_len() argument
2225 skb->mac_len = skb->network_header - skb->mac_header; in skb_reset_mac_len()
2229 *skb) in skb_inner_transport_header()
2231 return skb->head + skb->inner_transport_header; in skb_inner_transport_header()
2234 static inline int skb_inner_transport_offset(const struct sk_buff *skb) in skb_inner_transport_offset() argument
2236 return skb_inner_transport_header(skb) - skb->data; in skb_inner_transport_offset()
2239 static inline void skb_reset_inner_transport_header(struct sk_buff *skb) in skb_reset_inner_transport_header() argument
2241 skb->inner_transport_header = skb->data - skb->head; in skb_reset_inner_transport_header()
2244 static inline void skb_set_inner_transport_header(struct sk_buff *skb, in skb_set_inner_transport_header() argument
2247 skb_reset_inner_transport_header(skb); in skb_set_inner_transport_header()
2248 skb->inner_transport_header += offset; in skb_set_inner_transport_header()
2251 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb) in skb_inner_network_header() argument
2253 return skb->head + skb->inner_network_header; in skb_inner_network_header()
2256 static inline void skb_reset_inner_network_header(struct sk_buff *skb) in skb_reset_inner_network_header() argument
2258 skb->inner_network_header = skb->data - skb->head; in skb_reset_inner_network_header()
2261 static inline void skb_set_inner_network_header(struct sk_buff *skb, in skb_set_inner_network_header() argument
2264 skb_reset_inner_network_header(skb); in skb_set_inner_network_header()
2265 skb->inner_network_header += offset; in skb_set_inner_network_header()
2268 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb) in skb_inner_mac_header() argument
2270 return skb->head + skb->inner_mac_header; in skb_inner_mac_header()
2273 static inline void skb_reset_inner_mac_header(struct sk_buff *skb) in skb_reset_inner_mac_header() argument
2275 skb->inner_mac_header = skb->data - skb->head; in skb_reset_inner_mac_header()
2278 static inline void skb_set_inner_mac_header(struct sk_buff *skb, in skb_set_inner_mac_header() argument
2281 skb_reset_inner_mac_header(skb); in skb_set_inner_mac_header()
2282 skb->inner_mac_header += offset; in skb_set_inner_mac_header()
2284 static inline bool skb_transport_header_was_set(const struct sk_buff *skb) in skb_transport_header_was_set() argument
2286 return skb->transport_header != (typeof(skb->transport_header))~0U; in skb_transport_header_was_set()
2289 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) in skb_transport_header() argument
2291 return skb->head + skb->transport_header; in skb_transport_header()
2294 static inline void skb_reset_transport_header(struct sk_buff *skb) in skb_reset_transport_header() argument
2296 skb->transport_header = skb->data - skb->head; in skb_reset_transport_header()
2299 static inline void skb_set_transport_header(struct sk_buff *skb, in skb_set_transport_header() argument
2302 skb_reset_transport_header(skb); in skb_set_transport_header()
2303 skb->transport_header += offset; in skb_set_transport_header()
2306 static inline unsigned char *skb_network_header(const struct sk_buff *skb) in skb_network_header() argument
2308 return skb->head + skb->network_header; in skb_network_header()
2311 static inline void skb_reset_network_header(struct sk_buff *skb) in skb_reset_network_header() argument
2313 skb->network_header = skb->data - skb->head; in skb_reset_network_header()
2316 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) in skb_set_network_header() argument
2318 skb_reset_network_header(skb); in skb_set_network_header()
2319 skb->network_header += offset; in skb_set_network_header()
2322 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) in skb_mac_header() argument
2324 return skb->head + skb->mac_header; in skb_mac_header()
2327 static inline int skb_mac_offset(const struct sk_buff *skb) in skb_mac_offset() argument
2329 return skb_mac_header(skb) - skb->data; in skb_mac_offset()
2332 static inline u32 skb_mac_header_len(const struct sk_buff *skb) in skb_mac_header_len() argument
2334 return skb->network_header - skb->mac_header; in skb_mac_header_len()
2337 static inline int skb_mac_header_was_set(const struct sk_buff *skb) in skb_mac_header_was_set() argument
2339 return skb->mac_header != (typeof(skb->mac_header))~0U; in skb_mac_header_was_set()
2342 static inline void skb_reset_mac_header(struct sk_buff *skb) in skb_reset_mac_header() argument
2344 skb->mac_header = skb->data - skb->head; in skb_reset_mac_header()
2347 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) in skb_set_mac_header() argument
2349 skb_reset_mac_header(skb); in skb_set_mac_header()
2350 skb->mac_header += offset; in skb_set_mac_header()
2353 static inline void skb_pop_mac_header(struct sk_buff *skb) in skb_pop_mac_header() argument
2355 skb->mac_header = skb->network_header; in skb_pop_mac_header()
2358 static inline void skb_probe_transport_header(struct sk_buff *skb, in skb_probe_transport_header() argument
2363 if (skb_transport_header_was_set(skb)) in skb_probe_transport_header()
2366 if (skb_flow_dissect_flow_keys_basic(skb, &keys, NULL, 0, 0, 0, 0)) in skb_probe_transport_header()
2367 skb_set_transport_header(skb, keys.control.thoff); in skb_probe_transport_header()
2369 skb_set_transport_header(skb, offset_hint); in skb_probe_transport_header()
2372 static inline void skb_mac_header_rebuild(struct sk_buff *skb) in skb_mac_header_rebuild() argument
2374 if (skb_mac_header_was_set(skb)) { in skb_mac_header_rebuild()
2375 const unsigned char *old_mac = skb_mac_header(skb); in skb_mac_header_rebuild()
2377 skb_set_mac_header(skb, -skb->mac_len); in skb_mac_header_rebuild()
2378 memmove(skb_mac_header(skb), old_mac, skb->mac_len); in skb_mac_header_rebuild()
2382 static inline int skb_checksum_start_offset(const struct sk_buff *skb) in skb_checksum_start_offset() argument
2384 return skb->csum_start - skb_headroom(skb); in skb_checksum_start_offset()
2387 static inline unsigned char *skb_checksum_start(const struct sk_buff *skb) in skb_checksum_start() argument
2389 return skb->head + skb->csum_start; in skb_checksum_start()
2392 static inline int skb_transport_offset(const struct sk_buff *skb) in skb_transport_offset() argument
2394 return skb_transport_header(skb) - skb->data; in skb_transport_offset()
2397 static inline u32 skb_network_header_len(const struct sk_buff *skb) in skb_network_header_len() argument
2399 return skb->transport_header - skb->network_header; in skb_network_header_len()
2402 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb) in skb_inner_network_header_len() argument
2404 return skb->inner_transport_header - skb->inner_network_header; in skb_inner_network_header_len()
2407 static inline int skb_network_offset(const struct sk_buff *skb) in skb_network_offset() argument
2409 return skb_network_header(skb) - skb->data; in skb_network_offset()
2412 static inline int skb_inner_network_offset(const struct sk_buff *skb) in skb_inner_network_offset() argument
2414 return skb_inner_network_header(skb) - skb->data; in skb_inner_network_offset()
2417 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len) in pskb_network_may_pull() argument
2419 return pskb_may_pull(skb, skb_network_offset(skb) + len); in pskb_network_may_pull()
2470 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
2472 static inline void __skb_set_length(struct sk_buff *skb, unsigned int len) in __skb_set_length() argument
2474 if (unlikely(skb_is_nonlinear(skb))) { in __skb_set_length()
2478 skb->len = len; in __skb_set_length()
2479 skb_set_tail_pointer(skb, len); in __skb_set_length()
2482 static inline void __skb_trim(struct sk_buff *skb, unsigned int len) in __skb_trim() argument
2484 __skb_set_length(skb, len); in __skb_trim()
2487 void skb_trim(struct sk_buff *skb, unsigned int len);
2489 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) in __pskb_trim() argument
2491 if (skb->data_len) in __pskb_trim()
2492 return ___pskb_trim(skb, len); in __pskb_trim()
2493 __skb_trim(skb, len); in __pskb_trim()
2497 static inline int pskb_trim(struct sk_buff *skb, unsigned int len) in pskb_trim() argument
2499 return (len < skb->len) ? __pskb_trim(skb, len) : 0; in pskb_trim()
2511 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len) in pskb_trim_unique() argument
2513 int err = pskb_trim(skb, len); in pskb_trim_unique()
2517 static inline int __skb_grow(struct sk_buff *skb, unsigned int len) in __skb_grow() argument
2519 unsigned int diff = len - skb->len; in __skb_grow()
2521 if (skb_tailroom(skb) < diff) { in __skb_grow()
2522 int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb), in __skb_grow()
2527 __skb_set_length(skb, len); in __skb_grow()
2539 static inline void skb_orphan(struct sk_buff *skb) in skb_orphan() argument
2541 if (skb->destructor) { in skb_orphan()
2542 skb->destructor(skb); in skb_orphan()
2543 skb->destructor = NULL; in skb_orphan()
2544 skb->sk = NULL; in skb_orphan()
2546 BUG_ON(skb->sk); in skb_orphan()
2559 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) in skb_orphan_frags() argument
2561 if (likely(!skb_zcopy(skb))) in skb_orphan_frags()
2563 if (skb_uarg(skb)->callback == sock_zerocopy_callback) in skb_orphan_frags()
2565 return skb_copy_ubufs(skb, gfp_mask); in skb_orphan_frags()
2569 static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask) in skb_orphan_frags_rx() argument
2571 if (likely(!skb_zcopy(skb))) in skb_orphan_frags_rx()
2573 return skb_copy_ubufs(skb, gfp_mask); in skb_orphan_frags_rx()
2587 struct sk_buff *skb; in __skb_queue_purge() local
2588 while ((skb = __skb_dequeue(list)) != NULL) in __skb_queue_purge()
2589 kfree_skb(skb); in __skb_queue_purge()
2635 struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp); in __netdev_alloc_skb_ip_align() local
2637 if (NET_IP_ALIGN && skb) in __netdev_alloc_skb_ip_align()
2638 skb_reserve(skb, NET_IP_ALIGN); in __netdev_alloc_skb_ip_align()
2639 return skb; in __netdev_alloc_skb_ip_align()
2661 void napi_consume_skb(struct sk_buff *skb, int budget);
2664 void __kfree_skb_defer(struct sk_buff *skb);
2720 struct sk_buff *skb) in skb_propagate_pfmemalloc() argument
2723 skb->pfmemalloc = true; in skb_propagate_pfmemalloc()
2755 static inline void skb_frag_ref(struct sk_buff *skb, int f) in skb_frag_ref() argument
2757 __skb_frag_ref(&skb_shinfo(skb)->frags[f]); in skb_frag_ref()
2778 static inline void skb_frag_unref(struct sk_buff *skb, int f) in skb_frag_unref() argument
2780 __skb_frag_unref(&skb_shinfo(skb)->frags[f]); in skb_frag_unref()
2831 static inline void skb_frag_set_page(struct sk_buff *skb, int f, in skb_frag_set_page() argument
2834 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page); in skb_frag_set_page()
2859 static inline struct sk_buff *pskb_copy(struct sk_buff *skb, in pskb_copy() argument
2862 return __pskb_copy(skb, skb_headroom(skb), gfp_mask); in pskb_copy()
2866 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb, in pskb_copy_for_clone() argument
2869 return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true); in pskb_copy_for_clone()
2881 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len) in skb_clone_writable() argument
2883 return !skb_header_cloned(skb) && in skb_clone_writable()
2884 skb_headroom(skb) + len <= skb->hdr_len; in skb_clone_writable()
2887 static inline int skb_try_make_writable(struct sk_buff *skb, in skb_try_make_writable() argument
2890 return skb_cloned(skb) && !skb_clone_writable(skb, write_len) && in skb_try_make_writable()
2891 pskb_expand_head(skb, 0, 0, GFP_ATOMIC); in skb_try_make_writable()
2894 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom, in __skb_cow() argument
2899 if (headroom > skb_headroom(skb)) in __skb_cow()
2900 delta = headroom - skb_headroom(skb); in __skb_cow()
2903 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0, in __skb_cow()
2920 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom) in skb_cow() argument
2922 return __skb_cow(skb, headroom, skb_cloned(skb)); in skb_cow()
2935 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom) in skb_cow_head() argument
2937 return __skb_cow(skb, headroom, skb_header_cloned(skb)); in skb_cow_head()
2950 static inline int skb_padto(struct sk_buff *skb, unsigned int len) in skb_padto() argument
2952 unsigned int size = skb->len; in skb_padto()
2955 return skb_pad(skb, len - size); in skb_padto()
2969 static inline int __skb_put_padto(struct sk_buff *skb, unsigned int len, in __skb_put_padto() argument
2972 unsigned int size = skb->len; in __skb_put_padto()
2976 if (__skb_pad(skb, len, free_on_error)) in __skb_put_padto()
2978 __skb_put(skb, len); in __skb_put_padto()
2993 static inline int skb_put_padto(struct sk_buff *skb, unsigned int len) in skb_put_padto() argument
2995 return __skb_put_padto(skb, len, true); in skb_put_padto()
2998 static inline int skb_add_data(struct sk_buff *skb, in skb_add_data() argument
3001 const int off = skb->len; in skb_add_data()
3003 if (skb->ip_summed == CHECKSUM_NONE) { in skb_add_data()
3005 if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy, in skb_add_data()
3007 skb->csum = csum_block_add(skb->csum, csum, off); in skb_add_data()
3010 } else if (copy_from_iter_full(skb_put(skb, copy), copy, from)) in skb_add_data()
3013 __skb_trim(skb, off); in skb_add_data()
3017 static inline bool skb_can_coalesce(struct sk_buff *skb, int i, in skb_can_coalesce() argument
3020 if (skb_zcopy(skb)) in skb_can_coalesce()
3023 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; in skb_can_coalesce()
3031 static inline int __skb_linearize(struct sk_buff *skb) in __skb_linearize() argument
3033 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM; in __skb_linearize()
3043 static inline int skb_linearize(struct sk_buff *skb) in skb_linearize() argument
3045 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0; in skb_linearize()
3055 static inline bool skb_has_shared_frag(const struct sk_buff *skb) in skb_has_shared_frag() argument
3057 return skb_is_nonlinear(skb) && in skb_has_shared_frag()
3058 skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG; in skb_has_shared_frag()
3068 static inline int skb_linearize_cow(struct sk_buff *skb) in skb_linearize_cow() argument
3070 return skb_is_nonlinear(skb) || skb_cloned(skb) ? in skb_linearize_cow()
3071 __skb_linearize(skb) : 0; in skb_linearize_cow()
3075 __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, in __skb_postpull_rcsum() argument
3078 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_postpull_rcsum()
3079 skb->csum = csum_block_sub(skb->csum, in __skb_postpull_rcsum()
3081 else if (skb->ip_summed == CHECKSUM_PARTIAL && in __skb_postpull_rcsum()
3082 skb_checksum_start_offset(skb) < 0) in __skb_postpull_rcsum()
3083 skb->ip_summed = CHECKSUM_NONE; in __skb_postpull_rcsum()
3096 static inline void skb_postpull_rcsum(struct sk_buff *skb, in skb_postpull_rcsum() argument
3099 __skb_postpull_rcsum(skb, start, len, 0); in skb_postpull_rcsum()
3103 __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, in __skb_postpush_rcsum() argument
3106 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_postpush_rcsum()
3107 skb->csum = csum_block_add(skb->csum, in __skb_postpush_rcsum()
3120 static inline void skb_postpush_rcsum(struct sk_buff *skb, in skb_postpush_rcsum() argument
3123 __skb_postpush_rcsum(skb, start, len, 0); in skb_postpush_rcsum()
3126 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
3139 static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len) in skb_push_rcsum() argument
3141 skb_push(skb, len); in skb_push_rcsum()
3142 skb_postpush_rcsum(skb, skb->data, len); in skb_push_rcsum()
3143 return skb->data; in skb_push_rcsum()
3146 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
3156 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) in pskb_trim_rcsum() argument
3158 if (likely(len >= skb->len)) in pskb_trim_rcsum()
3160 return pskb_trim_rcsum_slow(skb, len); in pskb_trim_rcsum()
3163 static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len) in __skb_trim_rcsum() argument
3165 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_trim_rcsum()
3166 skb->ip_summed = CHECKSUM_NONE; in __skb_trim_rcsum()
3167 __skb_trim(skb, len); in __skb_trim_rcsum()
3171 static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len) in __skb_grow_rcsum() argument
3173 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_grow_rcsum()
3174 skb->ip_summed = CHECKSUM_NONE; in __skb_grow_rcsum()
3175 return __skb_grow(skb, len); in __skb_grow_rcsum()
3181 #define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode)) argument
3182 #define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode)) argument
3184 #define skb_queue_walk(queue, skb) \ argument
3185 for (skb = (queue)->next; \
3186 skb != (struct sk_buff *)(queue); \
3187 skb = skb->next)
3189 #define skb_queue_walk_safe(queue, skb, tmp) \ argument
3190 for (skb = (queue)->next, tmp = skb->next; \
3191 skb != (struct sk_buff *)(queue); \
3192 skb = tmp, tmp = skb->next)
3194 #define skb_queue_walk_from(queue, skb) \ argument
3195 for (; skb != (struct sk_buff *)(queue); \
3196 skb = skb->next)
3198 #define skb_rbtree_walk(skb, root) \ argument
3199 for (skb = skb_rb_first(root); skb != NULL; \
3200 skb = skb_rb_next(skb))
3202 #define skb_rbtree_walk_from(skb) \ argument
3203 for (; skb != NULL; \
3204 skb = skb_rb_next(skb))
3206 #define skb_rbtree_walk_from_safe(skb, tmp) \ argument
3207 for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL); \
3208 skb = tmp)
3210 #define skb_queue_walk_from_safe(queue, skb, tmp) \ argument
3211 for (tmp = skb->next; \
3212 skb != (struct sk_buff *)(queue); \
3213 skb = tmp, tmp = skb->next)
3215 #define skb_queue_reverse_walk(queue, skb) \ argument
3216 for (skb = (queue)->prev; \
3217 skb != (struct sk_buff *)(queue); \
3218 skb = skb->prev)
3220 #define skb_queue_reverse_walk_safe(queue, skb, tmp) \ argument
3221 for (skb = (queue)->prev, tmp = skb->prev; \
3222 skb != (struct sk_buff *)(queue); \
3223 skb = tmp, tmp = skb->prev)
3225 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \ argument
3226 for (tmp = skb->prev; \
3227 skb != (struct sk_buff *)(queue); \
3228 skb = tmp, tmp = skb->prev)
3230 static inline bool skb_has_frag_list(const struct sk_buff *skb) in skb_has_frag_list() argument
3232 return skb_shinfo(skb)->frag_list != NULL; in skb_has_frag_list()
3235 static inline void skb_frag_list_init(struct sk_buff *skb) in skb_frag_list_init() argument
3237 skb_shinfo(skb)->frag_list = NULL; in skb_frag_list_init()
3240 #define skb_walk_frags(skb, iter) \ argument
3241 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
3245 const struct sk_buff *skb);
3250 struct sk_buff *skb),
3255 struct sk_buff *skb),
3260 struct sk_buff *skb),
3273 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
3275 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
3277 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
3278 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
3279 void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len);
3281 struct sk_buff *skb) in skb_free_datagram_locked() argument
3283 __skb_free_datagram_locked(sk, skb, 0); in skb_free_datagram_locked()
3285 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
3286 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
3287 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
3288 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
3290 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3293 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3295 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
3296 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
3300 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
3301 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
3302 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
3303 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu);
3304 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len);
3305 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
3306 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
3307 int skb_ensure_writable(struct sk_buff *skb, int write_len);
3308 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
3309 int skb_vlan_pop(struct sk_buff *skb);
3310 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
3311 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
3331 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
3333 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
3337 __skb_header_pointer(const struct sk_buff *skb, int offset, in __skb_header_pointer() argument
3343 if (!skb || in __skb_header_pointer()
3344 skb_copy_bits(skb, offset, buffer, len) < 0) in __skb_header_pointer()
3351 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer) in skb_header_pointer() argument
3353 return __skb_header_pointer(skb, offset, len, skb->data, in skb_header_pointer()
3354 skb_headlen(skb), buffer); in skb_header_pointer()
3367 static inline bool skb_needs_linearize(struct sk_buff *skb, in skb_needs_linearize() argument
3370 return skb_is_nonlinear(skb) && in skb_needs_linearize()
3371 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) || in skb_needs_linearize()
3372 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG))); in skb_needs_linearize()
3375 static inline void skb_copy_from_linear_data(const struct sk_buff *skb, in skb_copy_from_linear_data() argument
3379 memcpy(to, skb->data, len); in skb_copy_from_linear_data()
3382 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb, in skb_copy_from_linear_data_offset() argument
3386 memcpy(to, skb->data + offset, len); in skb_copy_from_linear_data_offset()
3389 static inline void skb_copy_to_linear_data(struct sk_buff *skb, in skb_copy_to_linear_data() argument
3393 memcpy(skb->data, from, len); in skb_copy_to_linear_data()
3396 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb, in skb_copy_to_linear_data_offset() argument
3401 memcpy(skb->data + offset, from, len); in skb_copy_to_linear_data_offset()
3406 static inline ktime_t skb_get_ktime(const struct sk_buff *skb) in skb_get_ktime() argument
3408 return skb->tstamp; in skb_get_ktime()
3420 static inline void skb_get_timestamp(const struct sk_buff *skb, in skb_get_timestamp() argument
3423 *stamp = ktime_to_timeval(skb->tstamp); in skb_get_timestamp()
3426 static inline void skb_get_timestampns(const struct sk_buff *skb, in skb_get_timestampns() argument
3429 *stamp = ktime_to_timespec(skb->tstamp); in skb_get_timestampns()
3432 static inline void __net_timestamp(struct sk_buff *skb) in __net_timestamp() argument
3434 skb->tstamp = ktime_get_real(); in __net_timestamp()
3447 static inline u8 skb_metadata_len(const struct sk_buff *skb) in skb_metadata_len() argument
3449 return skb_shinfo(skb)->meta_len; in skb_metadata_len()
3452 static inline void *skb_metadata_end(const struct sk_buff *skb) in skb_metadata_end() argument
3454 return skb_mac_header(skb); in skb_metadata_end()
3500 static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len) in skb_metadata_set() argument
3502 skb_shinfo(skb)->meta_len = meta_len; in skb_metadata_set()
3505 static inline void skb_metadata_clear(struct sk_buff *skb) in skb_metadata_clear() argument
3507 skb_metadata_set(skb, 0); in skb_metadata_clear()
3510 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
3514 void skb_clone_tx_timestamp(struct sk_buff *skb);
3515 bool skb_defer_rx_timestamp(struct sk_buff *skb);
3519 static inline void skb_clone_tx_timestamp(struct sk_buff *skb) in skb_clone_tx_timestamp() argument
3523 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb) in skb_defer_rx_timestamp() argument
3542 void skb_complete_tx_timestamp(struct sk_buff *skb,
3575 static inline void skb_tx_timestamp(struct sk_buff *skb) in skb_tx_timestamp() argument
3577 skb_clone_tx_timestamp(skb); in skb_tx_timestamp()
3578 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP) in skb_tx_timestamp()
3579 skb_tstamp_tx(skb, NULL); in skb_tx_timestamp()
3589 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
3591 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
3592 __sum16 __skb_checksum_complete(struct sk_buff *skb);
3594 static inline int skb_csum_unnecessary(const struct sk_buff *skb) in skb_csum_unnecessary() argument
3596 return ((skb->ip_summed == CHECKSUM_UNNECESSARY) || in skb_csum_unnecessary()
3597 skb->csum_valid || in skb_csum_unnecessary()
3598 (skb->ip_summed == CHECKSUM_PARTIAL && in skb_csum_unnecessary()
3599 skb_checksum_start_offset(skb) >= 0)); in skb_csum_unnecessary()
3618 static inline __sum16 skb_checksum_complete(struct sk_buff *skb) in skb_checksum_complete() argument
3620 return skb_csum_unnecessary(skb) ? in skb_checksum_complete()
3621 0 : __skb_checksum_complete(skb); in skb_checksum_complete()
3624 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb) in __skb_decr_checksum_unnecessary() argument
3626 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_decr_checksum_unnecessary()
3627 if (skb->csum_level == 0) in __skb_decr_checksum_unnecessary()
3628 skb->ip_summed = CHECKSUM_NONE; in __skb_decr_checksum_unnecessary()
3630 skb->csum_level--; in __skb_decr_checksum_unnecessary()
3634 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb) in __skb_incr_checksum_unnecessary() argument
3636 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_incr_checksum_unnecessary()
3637 if (skb->csum_level < SKB_MAX_CSUM_LEVEL) in __skb_incr_checksum_unnecessary()
3638 skb->csum_level++; in __skb_incr_checksum_unnecessary()
3639 } else if (skb->ip_summed == CHECKSUM_NONE) { in __skb_incr_checksum_unnecessary()
3640 skb->ip_summed = CHECKSUM_UNNECESSARY; in __skb_incr_checksum_unnecessary()
3641 skb->csum_level = 0; in __skb_incr_checksum_unnecessary()
3650 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb, in __skb_checksum_validate_needed() argument
3654 if (skb_csum_unnecessary(skb) || (zero_okay && !check)) { in __skb_checksum_validate_needed()
3655 skb->csum_valid = 1; in __skb_checksum_validate_needed()
3656 __skb_decr_checksum_unnecessary(skb); in __skb_checksum_validate_needed()
3674 static inline void skb_checksum_complete_unset(struct sk_buff *skb) in skb_checksum_complete_unset() argument
3676 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_complete_unset()
3677 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_complete_unset()
3689 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb, in __skb_checksum_validate_complete() argument
3693 if (skb->ip_summed == CHECKSUM_COMPLETE) { in __skb_checksum_validate_complete()
3694 if (!csum_fold(csum_add(psum, skb->csum))) { in __skb_checksum_validate_complete()
3695 skb->csum_valid = 1; in __skb_checksum_validate_complete()
3700 skb->csum = psum; in __skb_checksum_validate_complete()
3702 if (complete || skb->len <= CHECKSUM_BREAK) { in __skb_checksum_validate_complete()
3705 csum = __skb_checksum_complete(skb); in __skb_checksum_validate_complete()
3706 skb->csum_valid = !csum; in __skb_checksum_validate_complete()
3713 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto) in null_compute_pseudo() argument
3728 #define __skb_checksum_validate(skb, proto, complete, \ argument
3732 skb->csum_valid = 0; \
3733 if (__skb_checksum_validate_needed(skb, zero_okay, check)) \
3734 __ret = __skb_checksum_validate_complete(skb, \
3735 complete, compute_pseudo(skb, proto)); \
3739 #define skb_checksum_init(skb, proto, compute_pseudo) \ argument
3740 __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
3742 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \ argument
3743 __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
3745 #define skb_checksum_validate(skb, proto, compute_pseudo) \ argument
3746 __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
3748 #define skb_checksum_validate_zero_check(skb, proto, check, \ argument
3750 __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
3752 #define skb_checksum_simple_validate(skb) \ argument
3753 __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
3755 static inline bool __skb_checksum_convert_check(struct sk_buff *skb) in __skb_checksum_convert_check() argument
3757 return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid); in __skb_checksum_convert_check()
3760 static inline void __skb_checksum_convert(struct sk_buff *skb, in __skb_checksum_convert() argument
3763 skb->csum = ~pseudo; in __skb_checksum_convert()
3764 skb->ip_summed = CHECKSUM_COMPLETE; in __skb_checksum_convert()
3767 #define skb_checksum_try_convert(skb, proto, check, compute_pseudo) \ argument
3769 if (__skb_checksum_convert_check(skb)) \
3770 __skb_checksum_convert(skb, check, \
3771 compute_pseudo(skb, proto)); \
3774 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr, in skb_remcsum_adjust_partial() argument
3777 skb->ip_summed = CHECKSUM_PARTIAL; in skb_remcsum_adjust_partial()
3778 skb->csum_start = ((unsigned char *)ptr + start) - skb->head; in skb_remcsum_adjust_partial()
3779 skb->csum_offset = offset - start; in skb_remcsum_adjust_partial()
3787 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr, in skb_remcsum_process() argument
3793 skb_remcsum_adjust_partial(skb, ptr, start, offset); in skb_remcsum_process()
3797 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) { in skb_remcsum_process()
3798 __skb_checksum_complete(skb); in skb_remcsum_process()
3799 skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data); in skb_remcsum_process()
3802 delta = remcsum_adjust(ptr, skb->csum, start, offset); in skb_remcsum_process()
3805 skb->csum = csum_add(skb->csum, delta); in skb_remcsum_process()
3808 static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb) in skb_nfct() argument
3811 return (void *)(skb->_nfct & SKB_NFCT_PTRMASK); in skb_nfct()
3842 static inline void nf_reset(struct sk_buff *skb) in nf_reset() argument
3845 nf_conntrack_put(skb_nfct(skb)); in nf_reset()
3846 skb->_nfct = 0; in nf_reset()
3849 nf_bridge_put(skb->nf_bridge); in nf_reset()
3850 skb->nf_bridge = NULL; in nf_reset()
3854 static inline void nf_reset_trace(struct sk_buff *skb) in nf_reset_trace() argument
3857 skb->nf_trace = 0; in nf_reset_trace()
3861 static inline void ipvs_reset(struct sk_buff *skb) in ipvs_reset() argument
3864 skb->ipvs_property = 0; in ipvs_reset()
3903 static inline void skb_init_secmark(struct sk_buff *skb) in skb_init_secmark() argument
3905 skb->secmark = 0; in skb_init_secmark()
3911 static inline void skb_init_secmark(struct sk_buff *skb) in skb_init_secmark() argument
3915 static inline bool skb_irq_freeable(const struct sk_buff *skb) in skb_irq_freeable() argument
3917 return !skb->destructor && in skb_irq_freeable()
3919 !skb->sp && in skb_irq_freeable()
3921 !skb_nfct(skb) && in skb_irq_freeable()
3922 !skb->_skb_refdst && in skb_irq_freeable()
3923 !skb_has_frag_list(skb); in skb_irq_freeable()
3926 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping) in skb_set_queue_mapping() argument
3928 skb->queue_mapping = queue_mapping; in skb_set_queue_mapping()
3931 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb) in skb_get_queue_mapping() argument
3933 return skb->queue_mapping; in skb_get_queue_mapping()
3941 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue) in skb_record_rx_queue() argument
3943 skb->queue_mapping = rx_queue + 1; in skb_record_rx_queue()
3946 static inline u16 skb_get_rx_queue(const struct sk_buff *skb) in skb_get_rx_queue() argument
3948 return skb->queue_mapping - 1; in skb_get_rx_queue()
3951 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb) in skb_rx_queue_recorded() argument
3953 return skb->queue_mapping != 0; in skb_rx_queue_recorded()
3956 static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val) in skb_set_dst_pending_confirm() argument
3958 skb->dst_pending_confirm = val; in skb_set_dst_pending_confirm()
3961 static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb) in skb_get_dst_pending_confirm() argument
3963 return skb->dst_pending_confirm != 0; in skb_get_dst_pending_confirm()
3966 static inline struct sec_path *skb_sec_path(struct sk_buff *skb) in skb_sec_path() argument
3969 return skb->sp; in skb_sec_path()
3991 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_SGO_CB_OFFSET)) argument
3999 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra) in gso_pskb_expand_head() argument
4004 headroom = skb_headroom(skb); in gso_pskb_expand_head()
4005 ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC); in gso_pskb_expand_head()
4009 new_headroom = skb_headroom(skb); in gso_pskb_expand_head()
4010 SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom); in gso_pskb_expand_head()
4014 static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res) in gso_reset_checksum() argument
4017 if (skb->remcsum_offload) in gso_reset_checksum()
4020 SKB_GSO_CB(skb)->csum = res; in gso_reset_checksum()
4021 SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head; in gso_reset_checksum()
4032 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res) in gso_make_checksum() argument
4034 unsigned char *csum_start = skb_transport_header(skb); in gso_make_checksum()
4035 int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start; in gso_make_checksum()
4036 __wsum partial = SKB_GSO_CB(skb)->csum; in gso_make_checksum()
4038 SKB_GSO_CB(skb)->csum = res; in gso_make_checksum()
4039 SKB_GSO_CB(skb)->csum_start = csum_start - skb->head; in gso_make_checksum()
4044 static inline bool skb_is_gso(const struct sk_buff *skb) in skb_is_gso() argument
4046 return skb_shinfo(skb)->gso_size; in skb_is_gso()
4050 static inline bool skb_is_gso_v6(const struct sk_buff *skb) in skb_is_gso_v6() argument
4052 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; in skb_is_gso_v6()
4056 static inline bool skb_is_gso_sctp(const struct sk_buff *skb) in skb_is_gso_sctp() argument
4058 return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP; in skb_is_gso_sctp()
4061 static inline void skb_gso_reset(struct sk_buff *skb) in skb_gso_reset() argument
4063 skb_shinfo(skb)->gso_size = 0; in skb_gso_reset()
4064 skb_shinfo(skb)->gso_segs = 0; in skb_gso_reset()
4065 skb_shinfo(skb)->gso_type = 0; in skb_gso_reset()
4084 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
4086 static inline bool skb_warn_if_lro(const struct sk_buff *skb) in skb_warn_if_lro() argument
4090 const struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_warn_if_lro()
4092 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 && in skb_warn_if_lro()
4094 __skb_warn_lro_forwarding(skb); in skb_warn_if_lro()
4100 static inline void skb_forward_csum(struct sk_buff *skb) in skb_forward_csum() argument
4103 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_forward_csum()
4104 skb->ip_summed = CHECKSUM_NONE; in skb_forward_csum()
4115 static inline void skb_checksum_none_assert(const struct sk_buff *skb) in skb_checksum_none_assert() argument
4118 BUG_ON(skb->ip_summed != CHECKSUM_NONE); in skb_checksum_none_assert()
4122 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
4124 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
4125 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4127 __sum16(*skb_chkf)(struct sk_buff *skb));
4138 static inline bool skb_head_is_locked(const struct sk_buff *skb) in skb_head_is_locked() argument
4140 return !skb->head_frag || skb_cloned(skb); in skb_head_is_locked()
4152 static inline __wsum lco_csum(struct sk_buff *skb) in lco_csum() argument
4154 unsigned char *csum_start = skb_checksum_start(skb); in lco_csum()
4155 unsigned char *l4_hdr = skb_transport_header(skb); in lco_csum()
4160 skb->csum_offset)); in lco_csum()