Lines Matching +full:mmp +full:- +full:timer
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
7 * Florian La Roche, <rzsfl@rz.uni-sb.de>
31 #include <linux/dma-mapping.h>
57 * NETIF_F_HW_CSUM - The driver (or its device) is able to compute one
63 * NETIF_F_IP_CSUM - Driver (device) is only able to checksum plain
72 * NETIF_F_IPV6_CSUM - Driver (device) is only able to checksum plain
82 * NETIF_F_RXCSUM - Driver (device) performs receive checksum offload.
89 * verification is set in skb->ip_summed. Possible values are:
95 * not in skb->csum. Thus, skb->csum is undefined in this case.
102 * if their checksums are okay. skb->csum is still undefined in this case
115 * skb->csum_level indicates the number of consecutive checksums found in
117 * For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
119 * GRE (checksum flag is set) and TCP, skb->csum_level would be set to
122 * checksum is bad, skb->csum_level would be set to zero (TCP checksum is
128 * packet as seen by netif_rx() and fills in skb->csum. This means the
132 * - Even if device supports only some protocols, but is able to produce
133 * skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
134 * - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
143 * referred to by skb->csum_start + skb->csum_offset and any preceding
148 * C. Checksumming on transmit for non-GSO. The stack requests checksum offload
149 * in the skb->ip_summed for a packet. Values are:
154 * from skb->csum_start up to the end, and to record/write the checksum at
155 * offset skb->csum_start + skb->csum_offset. A driver may verify that the
158 * checksum refers to a legitimate transport layer checksum -- it is the
189 * D. Non-IP checksum (CRC) offloads
191 * NETIF_F_SCTP_CRC - This feature indicates that a device is capable of
198 * value of skb->csum_not_inet; skb_crc32c_csum_help is provided to resolve
201 * NETIF_F_FCOE_CRC - This feature indicates that a device is capable of
226 /* Maximum value in skb->csum_level */
231 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
233 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
265 /* always valid & non-NULL from FORWARD on, for physdev match */
318 /* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
326 * skb_frag_size() - Returns the size of a skb fragment
331 return frag->bv_len; in skb_frag_size()
335 * skb_frag_size_set() - Sets the size of a skb fragment
341 frag->bv_len = size; in skb_frag_size_set()
345 * skb_frag_size_add() - Increments the size of a skb fragment by @delta
351 frag->bv_len += delta; in skb_frag_size_add()
355 * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
361 frag->bv_len -= delta; in skb_frag_size_sub()
365 * skb_frag_must_loop - Test if %p is a high memory page
378 * skb_frag_foreach_page - loop over pages in a fragment
381 * @f_off: offset from start of f->bv_page
385 * non-zero only on first page.
390 * A fragment can hold a compound page, in which case per-page
396 p_off = (f_off) & (PAGE_SIZE - 1), \
398 min_t(u32, f_len, PAGE_SIZE - p_off) : f_len, \
402 p_len = min_t(u32, f_len - copied, PAGE_SIZE)) \
407 * struct skb_shared_hwtstamps - hardware time stamps
412 * skb->tstamp.
490 } mmp; member
493 #define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
495 int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
496 void mm_unaccount_pinned_pages(struct mmpin *mmp);
513 * the end of the header data, ie. at skb->end.
542 * to the payload part of skb->data. The lower 16 bits hold references to
543 * the entire skb->data. A clone of a headerless skb holds the length of
544 * the header in skb->hdr_len.
546 * All users must obey the rule that the skb->data reference count must be
550 * care about modifications to the header part of skb->data.
553 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
615 * struct sk_buff - socket buffer
620 * for retransmit timer
636 * @csum_start: Offset from skb->head where checksumming should start
649 * @offload_fwd_mark: Packet was L2-forwarded in hardware
650 * @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
660 * @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
663 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
676 * @l4_hash: indicate hash is a canonical 4-tuple hash over transport
704 * skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
717 * @users: User count - see {datagram,tcp}.c
938 /* only useable after checking ->active_extensions != 0 */
953 * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
958 return unlikely(skb->pfmemalloc); in skb_pfmemalloc()
969 * skb_dst - returns skb dst_entry
979 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) && in skb_dst()
982 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK); in skb_dst()
986 * skb_dst_set - sets skb dst
995 skb->slow_gro |= !!dst; in skb_dst_set()
996 skb->_skb_refdst = (unsigned long)dst; in skb_dst_set()
1000 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
1012 skb->slow_gro |= !!dst; in skb_dst_set_noref()
1013 skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; in skb_dst_set_noref()
1017 * skb_dst_is_noref - Test if skb dst isn't refcounted
1022 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb); in skb_dst_is_noref()
1026 * skb_rtable - Returns the skb &rtable
1034 /* For mangling skb->pkt_type from user space side from applications
1044 * skb_napi_id - Returns the skb's NAPI id
1050 return skb->napi_id; in skb_napi_id()
1057 * skb_unref - decrement the skb's reference count
1066 if (likely(refcount_read(&skb->users) == 1)) in skb_unref()
1068 else if (likely(!refcount_dec_and_test(&skb->users))) in skb_unref()
1107 * alloc_skb - allocate a network buffer
1136 * skb_fclone_busy - check if fclone is busy
1151 return skb->fclone == SKB_FCLONE_ORIG && in skb_fclone_busy()
1152 refcount_read(&fclones->fclone_ref) > 1 && in skb_fclone_busy()
1153 READ_ONCE(fclones->skb2.sk) == sk; in skb_fclone_busy()
1157 * alloc_skb_fclone - allocate a network buffer from fclone cache
1197 * skb_pad - zero pad the tail of an skb
1242 * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
1271 skb->hash = 0; in skb_clear_hash()
1272 skb->sw_hash = 0; in skb_clear_hash()
1273 skb->l4_hash = 0; in skb_clear_hash()
1278 if (!skb->l4_hash) in skb_clear_hash_if_not_l4()
1285 skb->l4_hash = is_l4; in __skb_set_hash()
1286 skb->sw_hash = is_sw; in __skb_set_hash()
1287 skb->hash = hash; in __skb_set_hash()
1385 if (!skb->l4_hash && !skb->sw_hash) in skb_get_hash()
1388 return skb->hash; in skb_get_hash()
1393 if (!skb->l4_hash && !skb->sw_hash) { in skb_get_hash_flowi6()
1400 return skb->hash; in skb_get_hash_flowi6()
1408 return skb->hash; in skb_get_hash_raw()
1413 to->hash = from->hash; in skb_copy_hash()
1414 to->sw_hash = from->sw_hash; in skb_copy_hash()
1415 to->l4_hash = from->l4_hash; in skb_copy_hash()
1422 to->decrypted = from->decrypted; in skb_copy_decrypted()
1429 return skb->head + skb->end; in skb_end_pointer()
1434 return skb->end; in skb_end_offset()
1439 return skb->end; in skb_end_pointer()
1444 return skb->end - skb->head; in skb_end_offset()
1453 return &skb_shinfo(skb)->hwtstamps; in skb_hwtstamps()
1458 bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE; in skb_zcopy()
1465 refcount_inc(&uarg->refcnt); in net_zcopy_get()
1470 skb_shinfo(skb)->destructor_arg = uarg; in skb_zcopy_init()
1471 skb_shinfo(skb)->flags |= uarg->flags; in skb_zcopy_init()
1488 skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); in skb_zcopy_set_nouarg()
1489 skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; in skb_zcopy_set_nouarg()
1494 return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL; in skb_zcopy_is_nouarg()
1499 return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); in skb_zcopy_get_nouarg()
1505 uarg->callback(NULL, uarg, true); in net_zcopy_put()
1511 if (uarg->callback == msg_zerocopy_callback) in net_zcopy_put_abort()
1525 uarg->callback(skb, uarg, zerocopy_success); in skb_zcopy_clear()
1527 skb_shinfo(skb)->flags &= ~SKBFL_ZEROCOPY_FRAG; in skb_zcopy_clear()
1533 skb->next = NULL; in skb_mark_not_on_list()
1536 /* Iterate through singly-linked GSO fragments of an skb. */
1538 for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \
1539 (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)
1543 __list_del_entry(&skb->list); in skb_list_del_init()
1548 * skb_queue_empty - check if a queue is empty
1555 return list->next == (const struct sk_buff *) list; in skb_queue_empty()
1559 * skb_queue_empty_lockless - check if a queue is empty
1567 return READ_ONCE(list->next) == (const struct sk_buff *) list; in skb_queue_empty_lockless()
1572 * skb_queue_is_last - check if skb is the last entry in the queue
1581 return skb->next == (const struct sk_buff *) list; in skb_queue_is_last()
1585 * skb_queue_is_first - check if skb is the first entry in the queue
1594 return skb->prev == (const struct sk_buff *) list; in skb_queue_is_first()
1598 * skb_queue_next - return the next packet in the queue
1612 return skb->next; in skb_queue_next()
1616 * skb_queue_prev - return the prev packet in the queue
1630 return skb->prev; in skb_queue_prev()
1634 * skb_get - reference buffer
1642 refcount_inc(&skb->users); in skb_get()
1651 * skb_cloned - is the buffer a clone
1660 return skb->cloned && in skb_cloned()
1661 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1; in skb_cloned()
1675 * skb_header_cloned - is the header a clone
1685 if (!skb->cloned) in skb_header_cloned()
1688 dataref = atomic_read(&skb_shinfo(skb)->dataref); in skb_header_cloned()
1689 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT); in skb_header_cloned()
1704 * __skb_header_release - release reference to header
1709 skb->nohdr = 1; in __skb_header_release()
1710 atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT)); in __skb_header_release()
1715 * skb_shared - is the buffer shared
1723 return refcount_read(&skb->users) != 1; in skb_shared()
1727 * skb_share_check - check if buffer is shared and if so clone it
1762 * skb_unshare - make a copy of a shared buffer
1792 * skb_peek - peek at the head of an &sk_buff_head
1806 struct sk_buff *skb = list_->next; in skb_peek()
1814 * __skb_peek - peek at the head of a non-empty &sk_buff_head
1821 return list_->next; in __skb_peek()
1825 * skb_peek_next - peek skb following the given one from a queue
1836 struct sk_buff *next = skb->next; in skb_peek_next()
1844 * skb_peek_tail - peek at the tail of an &sk_buff_head
1858 struct sk_buff *skb = READ_ONCE(list_->prev); in skb_peek_tail()
1867 * skb_queue_len - get queue length
1874 return list_->qlen; in skb_queue_len()
1878 * skb_queue_len_lockless - get queue length
1886 return READ_ONCE(list_->qlen); in skb_queue_len_lockless()
1890 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
1896 * the spinlock. It can also be used for on-stack sk_buff_head
1901 list->prev = list->next = (struct sk_buff *)list; in __skb_queue_head_init()
1902 list->qlen = 0; in __skb_queue_head_init()
1907 * this is needed for now since a whole lot of users of the skb-queue
1915 spin_lock_init(&list->lock); in skb_queue_head_init()
1923 lockdep_set_class(&list->lock, class); in skb_queue_head_init_class()
1929 * The "__skb_xxxx()" functions are the non-atomic ones that
1939 WRITE_ONCE(newsk->next, next); in __skb_insert()
1940 WRITE_ONCE(newsk->prev, prev); in __skb_insert()
1941 WRITE_ONCE(next->prev, newsk); in __skb_insert()
1942 WRITE_ONCE(prev->next, newsk); in __skb_insert()
1943 WRITE_ONCE(list->qlen, list->qlen + 1); in __skb_insert()
1950 struct sk_buff *first = list->next; in __skb_queue_splice()
1951 struct sk_buff *last = list->prev; in __skb_queue_splice()
1953 WRITE_ONCE(first->prev, prev); in __skb_queue_splice()
1954 WRITE_ONCE(prev->next, first); in __skb_queue_splice()
1956 WRITE_ONCE(last->next, next); in __skb_queue_splice()
1957 WRITE_ONCE(next->prev, last); in __skb_queue_splice()
1961 * skb_queue_splice - join two skb lists, this is designed for stacks
1969 __skb_queue_splice(list, (struct sk_buff *) head, head->next); in skb_queue_splice()
1970 head->qlen += list->qlen; in skb_queue_splice()
1975 * skb_queue_splice_init - join two skb lists and reinitialise the emptied list
1985 __skb_queue_splice(list, (struct sk_buff *) head, head->next); in skb_queue_splice_init()
1986 head->qlen += list->qlen; in skb_queue_splice_init()
1992 * skb_queue_splice_tail - join two skb lists, each list being a queue
2000 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); in skb_queue_splice_tail()
2001 head->qlen += list->qlen; in skb_queue_splice_tail()
2006 * skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
2017 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); in skb_queue_splice_tail_init()
2018 head->qlen += list->qlen; in skb_queue_splice_tail_init()
2024 * __skb_queue_after - queue a buffer at the list head
2038 __skb_insert(newsk, prev, prev->next, list); in __skb_queue_after()
2048 __skb_insert(newsk, next->prev, next, list); in __skb_queue_before()
2052 * __skb_queue_head - queue a buffer at the list head
2069 * __skb_queue_tail - queue a buffer at the list tail
2094 WRITE_ONCE(list->qlen, list->qlen - 1); in __skb_unlink()
2095 next = skb->next; in __skb_unlink()
2096 prev = skb->prev; in __skb_unlink()
2097 skb->next = skb->prev = NULL; in __skb_unlink()
2098 WRITE_ONCE(next->prev, prev); in __skb_unlink()
2099 WRITE_ONCE(prev->next, next); in __skb_unlink()
2103 * __skb_dequeue - remove from the head of the queue
2120 * __skb_dequeue_tail - remove from the tail of the queue
2139 return skb->data_len; in skb_is_nonlinear()
2144 return skb->len - skb->data_len; in skb_headlen()
2151 for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--) in __skb_pagelen()
2152 len += skb_frag_size(&skb_shinfo(skb)->frags[i]); in __skb_pagelen()
2162 * __skb_fill_page_desc - initialise a paged fragment in an skb
2177 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in __skb_fill_page_desc()
2184 frag->bv_page = page; in __skb_fill_page_desc()
2185 frag->bv_offset = off; in __skb_fill_page_desc()
2190 skb->pfmemalloc = true; in __skb_fill_page_desc()
2194 * skb_fill_page_desc - initialise a paged fragment in an skb
2201 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
2211 skb_shinfo(skb)->nr_frags = i + 1; in skb_fill_page_desc()
2225 return skb->head + skb->tail; in skb_tail_pointer()
2230 skb->tail = skb->data - skb->head; in skb_reset_tail_pointer()
2236 skb->tail += offset; in skb_set_tail_pointer()
2242 return skb->tail; in skb_tail_pointer()
2247 skb->tail = skb->data; in skb_reset_tail_pointer()
2252 skb->tail = skb->data + offset; in skb_set_tail_pointer()
2266 skb->tail += len; in __skb_put()
2267 skb->len += len; in __skb_put()
2320 skb->data -= len; in __skb_push()
2321 skb->len += len; in __skb_push()
2322 return skb->data; in __skb_push()
2328 skb->len -= len; in __skb_pull()
2329 BUG_ON(skb->len < skb->data_len); in __skb_pull()
2330 return skb->data += len; in __skb_pull()
2335 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); in skb_pull_inline()
2343 !__pskb_pull_tail(skb, len - skb_headlen(skb))) in __pskb_pull()
2345 skb->len -= len; in __pskb_pull()
2346 return skb->data += len; in __pskb_pull()
2351 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len); in pskb_pull()
2358 if (unlikely(len > skb->len)) in pskb_may_pull()
2360 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; in pskb_may_pull()
2366 * skb_headroom - bytes at buffer head
2373 return skb->data - skb->head; in skb_headroom()
2377 * skb_tailroom - bytes at buffer end
2384 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail; in skb_tailroom()
2388 * skb_availroom - bytes at buffer end
2399 return skb->end - skb->tail - skb->reserved_tailroom; in skb_availroom()
2403 * skb_reserve - adjust headroom
2412 skb->data += len; in skb_reserve()
2413 skb->tail += len; in skb_reserve()
2417 * skb_tailroom_reserve - adjust reserved_tailroom
2432 if (mtu < skb_tailroom(skb) - needed_tailroom) in skb_tailroom_reserve()
2434 skb->reserved_tailroom = skb_tailroom(skb) - mtu; in skb_tailroom_reserve()
2437 skb->reserved_tailroom = needed_tailroom; in skb_tailroom_reserve()
2446 skb->inner_protocol = protocol; in skb_set_inner_protocol()
2447 skb->inner_protocol_type = ENCAP_TYPE_ETHER; in skb_set_inner_protocol()
2453 skb->inner_ipproto = ipproto; in skb_set_inner_ipproto()
2454 skb->inner_protocol_type = ENCAP_TYPE_IPPROTO; in skb_set_inner_ipproto()
2459 skb->inner_mac_header = skb->mac_header; in skb_reset_inner_headers()
2460 skb->inner_network_header = skb->network_header; in skb_reset_inner_headers()
2461 skb->inner_transport_header = skb->transport_header; in skb_reset_inner_headers()
2466 skb->mac_len = skb->network_header - skb->mac_header; in skb_reset_mac_len()
2472 return skb->head + skb->inner_transport_header; in skb_inner_transport_header()
2477 return skb_inner_transport_header(skb) - skb->data; in skb_inner_transport_offset()
2482 skb->inner_transport_header = skb->data - skb->head; in skb_reset_inner_transport_header()
2489 skb->inner_transport_header += offset; in skb_set_inner_transport_header()
2494 return skb->head + skb->inner_network_header; in skb_inner_network_header()
2499 skb->inner_network_header = skb->data - skb->head; in skb_reset_inner_network_header()
2506 skb->inner_network_header += offset; in skb_set_inner_network_header()
2511 return skb->head + skb->inner_mac_header; in skb_inner_mac_header()
2516 skb->inner_mac_header = skb->data - skb->head; in skb_reset_inner_mac_header()
2523 skb->inner_mac_header += offset; in skb_set_inner_mac_header()
2527 return skb->transport_header != (typeof(skb->transport_header))~0U; in skb_transport_header_was_set()
2532 return skb->head + skb->transport_header; in skb_transport_header()
2537 skb->transport_header = skb->data - skb->head; in skb_reset_transport_header()
2544 skb->transport_header += offset; in skb_set_transport_header()
2549 return skb->head + skb->network_header; in skb_network_header()
2554 skb->network_header = skb->data - skb->head; in skb_reset_network_header()
2560 skb->network_header += offset; in skb_set_network_header()
2565 return skb->head + skb->mac_header; in skb_mac_header()
2570 return skb_mac_header(skb) - skb->data; in skb_mac_offset()
2575 return skb->network_header - skb->mac_header; in skb_mac_header_len()
2580 return skb->mac_header != (typeof(skb->mac_header))~0U; in skb_mac_header_was_set()
2585 skb->mac_header = (typeof(skb->mac_header))~0U; in skb_unset_mac_header()
2590 skb->mac_header = skb->data - skb->head; in skb_reset_mac_header()
2596 skb->mac_header += offset; in skb_set_mac_header()
2601 skb->mac_header = skb->network_header; in skb_pop_mac_header()
2621 skb_set_mac_header(skb, -skb->mac_len); in skb_mac_header_rebuild()
2622 memmove(skb_mac_header(skb), old_mac, skb->mac_len); in skb_mac_header_rebuild()
2628 return skb->csum_start - skb_headroom(skb); in skb_checksum_start_offset()
2633 return skb->head + skb->csum_start; in skb_checksum_start()
2638 return skb_transport_header(skb) - skb->data; in skb_transport_offset()
2643 return skb->transport_header - skb->network_header; in skb_network_header_len()
2648 return skb->inner_transport_header - skb->inner_network_header; in skb_inner_network_header_len()
2653 return skb_network_header(skb) - skb->data; in skb_network_offset()
2658 return skb_inner_network_header(skb) - skb->data; in skb_inner_network_offset()
2720 skb->len = len; in __skb_set_length()
2733 if (skb->data_len) in __pskb_trim()
2741 return (len < skb->len) ? __pskb_trim(skb, len) : 0; in pskb_trim()
2745 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer
2750 * the skb is not cloned so we should never get an error due to out-
2751 * of-memory.
2761 unsigned int diff = len - skb->len; in __skb_grow()
2764 int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb), in __skb_grow()
2774 * skb_orphan - orphan a buffer
2783 if (skb->destructor) { in skb_orphan()
2784 skb->destructor(skb); in skb_orphan()
2785 skb->destructor = NULL; in skb_orphan()
2786 skb->sk = NULL; in skb_orphan()
2788 BUG_ON(skb->sk); in skb_orphan()
2793 * skb_orphan_frags - orphan the frags contained in a buffer
2806 skb_uarg(skb)->callback == msg_zerocopy_callback) in skb_orphan_frags()
2820 * __skb_queue_purge - empty a list
2840 * netdev_alloc_frag - allocate a page fragment
2855 return __netdev_alloc_frag_align(fragsz, -align); in netdev_alloc_frag_align()
2862 * netdev_alloc_skb - allocate an skbuff for rx on a specific device
2926 return __napi_alloc_frag_align(fragsz, -align); in napi_alloc_frag_align()
2942 * __dev_alloc_pages - allocate page for network Rx
2972 * __dev_alloc_page - allocate a page for network Rx
2990 * dev_page_is_reusable - check whether a page can be reused for network Rx
3006 * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
3014 skb->pfmemalloc = true; in skb_propagate_pfmemalloc()
3018 * skb_frag_off() - Returns the offset of a skb fragment
3023 return frag->bv_offset; in skb_frag_off()
3027 * skb_frag_off_add() - Increments the offset of a skb fragment by @delta
3033 frag->bv_offset += delta; in skb_frag_off_add()
3037 * skb_frag_off_set() - Sets the offset of a skb fragment
3043 frag->bv_offset = offset; in skb_frag_off_set()
3047 * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
3054 fragto->bv_offset = fragfrom->bv_offset; in skb_frag_off_copy()
3058 * skb_frag_page - retrieve the page referred to by a paged fragment
3065 return frag->bv_page; in skb_frag_page()
3069 * __skb_frag_ref - take an addition reference on a paged fragment.
3080 * skb_frag_ref - take an addition reference on a paged fragment of an skb.
3088 __skb_frag_ref(&skb_shinfo(skb)->frags[f]); in skb_frag_ref()
3092 * __skb_frag_unref - release a reference on a paged fragment.
3111 * skb_frag_unref - release a reference on a paged fragment of an skb.
3119 __skb_frag_unref(&skb_shinfo(skb)->frags[f], skb->pp_recycle); in skb_frag_unref()
3123 * skb_frag_address - gets the address of the data contained in a paged fragment
3135 * skb_frag_address_safe - gets the address of the data contained in a paged fragment
3151 * skb_frag_page_copy() - sets the page in a fragment from another fragment
3158 fragto->bv_page = fragfrom->bv_page; in skb_frag_page_copy()
3162 * __skb_frag_set_page - sets the page contained in a paged fragment
3170 frag->bv_page = page; in __skb_frag_set_page()
3174 * skb_frag_set_page - sets the page contained in a paged fragment of an skb
3184 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page); in skb_frag_set_page()
3190 * skb_frag_dma_map - maps a paged fragment via the DMA API
3224 * skb_clone_writable - is the header of a clone writable
3234 skb_headroom(skb) + len <= skb->hdr_len; in skb_clone_writable()
3250 delta = headroom - skb_headroom(skb); in __skb_cow()
3259 * skb_cow - copy header of skb when it is required
3267 * The result is skb with writable area skb->head...skb->tail
3276 * skb_cow_head - skb_cow but only making the head writable
3291 * skb_padto - pad an skbuff up to a minimal size
3302 unsigned int size = skb->len; in skb_padto()
3305 return skb_pad(skb, len - size); in skb_padto()
3309 * __skb_put_padto - increase size and pad an skbuff up to a minimal size
3323 unsigned int size = skb->len; in __skb_put_padto()
3326 len -= size; in __skb_put_padto()
3328 return -ENOMEM; in __skb_put_padto()
3335 * skb_put_padto - increase size and pad an skbuff up to a minimal size
3352 const int off = skb->len; in skb_add_data()
3354 if (skb->ip_summed == CHECKSUM_NONE) { in skb_add_data()
3358 skb->csum = csum_block_add(skb->csum, csum, off); in skb_add_data()
3365 return -EFAULT; in skb_add_data()
3374 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; in skb_can_coalesce()
3384 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM; in __skb_linearize()
3388 * skb_linearize - convert paged skb to linear one
3391 * If there is no free memory -ENOMEM is returned, otherwise zero
3400 * skb_has_shared_frag - can any frag be overwritten
3409 skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; in skb_has_shared_frag()
3413 * skb_linearize_cow - make sure skb is linear and writable
3416 * If there is no free memory -ENOMEM is returned, otherwise zero
3429 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_postpull_rcsum()
3430 skb->csum = csum_block_sub(skb->csum, in __skb_postpull_rcsum()
3432 else if (skb->ip_summed == CHECKSUM_PARTIAL && in __skb_postpull_rcsum()
3434 skb->ip_summed = CHECKSUM_NONE; in __skb_postpull_rcsum()
3438 * skb_postpull_rcsum - update checksum for received skb after pull
3457 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_postpush_rcsum()
3458 skb->csum = csum_block_add(skb->csum, in __skb_postpush_rcsum()
3463 * skb_postpush_rcsum - update checksum for received skb after push
3480 * skb_push_rcsum - push skb and update receive checksum
3493 skb_postpush_rcsum(skb, skb->data, len); in skb_push_rcsum()
3494 return skb->data; in skb_push_rcsum()
3499 * pskb_trim_rcsum - trim received skb and update checksum
3510 if (likely(len >= skb->len)) in pskb_trim_rcsum()
3517 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_trim_rcsum()
3518 skb->ip_summed = CHECKSUM_NONE; in __skb_trim_rcsum()
3525 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_grow_rcsum()
3526 skb->ip_summed = CHECKSUM_NONE; in __skb_grow_rcsum()
3533 #define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode))
3534 #define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode))
3537 for (skb = (queue)->next; \
3539 skb = skb->next)
3542 for (skb = (queue)->next, tmp = skb->next; \
3544 skb = tmp, tmp = skb->next)
3548 skb = skb->next)
3563 for (tmp = skb->next; \
3565 skb = tmp, tmp = skb->next)
3568 for (skb = (queue)->prev; \
3570 skb = skb->prev)
3573 for (skb = (queue)->prev, tmp = skb->prev; \
3575 skb = tmp, tmp = skb->prev)
3578 for (tmp = skb->prev; \
3580 skb = tmp, tmp = skb->prev)
3584 return skb_shinfo(skb)->frag_list != NULL; in skb_has_frag_list()
3589 skb_shinfo(skb)->frag_list = NULL; in skb_frag_list_init()
3593 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
3620 return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size); in skb_copy_datagram_msg()
3679 return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT; in memcpy_from_msg()
3684 return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT; in memcpy_to_msg()
3703 if (likely(hlen - offset >= len)) in __skb_header_pointer()
3715 return __skb_header_pointer(skb, offset, len, skb->data, in skb_header_pointer()
3720 * skb_needs_linearize - check if we need to linearize a given skb
3734 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG))); in skb_needs_linearize()
3741 memcpy(to, skb->data, len); in skb_copy_from_linear_data()
3748 memcpy(to, skb->data + offset, len); in skb_copy_from_linear_data_offset()
3755 memcpy(skb->data, from, len); in skb_copy_to_linear_data()
3763 memcpy(skb->data + offset, from, len); in skb_copy_to_linear_data_offset()
3770 return skb->tstamp; in skb_get_ktime()
3774 * skb_get_timestamp - get timestamp from a skb
3785 *stamp = ns_to_kernel_old_timeval(skb->tstamp); in skb_get_timestamp()
3791 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); in skb_get_new_timestamp()
3793 stamp->tv_sec = ts.tv_sec; in skb_get_new_timestamp()
3794 stamp->tv_usec = ts.tv_nsec / 1000; in skb_get_new_timestamp()
3800 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); in skb_get_timestampns()
3802 stamp->tv_sec = ts.tv_sec; in skb_get_timestampns()
3803 stamp->tv_nsec = ts.tv_nsec; in skb_get_timestampns()
3809 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); in skb_get_new_timestampns()
3811 stamp->tv_sec = ts.tv_sec; in skb_get_new_timestampns()
3812 stamp->tv_nsec = ts.tv_nsec; in skb_get_new_timestampns()
3817 skb->tstamp = ktime_get_real(); in __net_timestamp()
3832 return skb_shinfo(skb)->meta_len; in skb_metadata_len()
3851 #define __it(x, op) (x -= sizeof(u##op)) in __skb_metadata_differs()
3872 return memcmp(a - meta_len, b - meta_len, meta_len); in __skb_metadata_differs()
3891 skb_shinfo(skb)->meta_len = meta_len; in skb_metadata_set()
3920 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
3939 * skb_tstamp_tx - queue clone of skb with send time stamps
3953 * skb_tx_timestamp() - Driver hook for transmit timestamping
3967 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP) in skb_tx_timestamp()
3972 * skb_complete_wifi_ack - deliver skb with wifi status
3985 return ((skb->ip_summed == CHECKSUM_UNNECESSARY) || in skb_csum_unnecessary()
3986 skb->csum_valid || in skb_csum_unnecessary()
3987 (skb->ip_summed == CHECKSUM_PARTIAL && in skb_csum_unnecessary()
3992 * skb_checksum_complete - Calculate checksum of an entire packet
3996 * the value of skb->csum. The latter can be used to supply the
4004 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
4015 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_decr_checksum_unnecessary()
4016 if (skb->csum_level == 0) in __skb_decr_checksum_unnecessary()
4017 skb->ip_summed = CHECKSUM_NONE; in __skb_decr_checksum_unnecessary()
4019 skb->csum_level--; in __skb_decr_checksum_unnecessary()
4025 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_incr_checksum_unnecessary()
4026 if (skb->csum_level < SKB_MAX_CSUM_LEVEL) in __skb_incr_checksum_unnecessary()
4027 skb->csum_level++; in __skb_incr_checksum_unnecessary()
4028 } else if (skb->ip_summed == CHECKSUM_NONE) { in __skb_incr_checksum_unnecessary()
4029 skb->ip_summed = CHECKSUM_UNNECESSARY; in __skb_incr_checksum_unnecessary()
4030 skb->csum_level = 0; in __skb_incr_checksum_unnecessary()
4036 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_reset_checksum_unnecessary()
4037 skb->ip_summed = CHECKSUM_NONE; in __skb_reset_checksum_unnecessary()
4038 skb->csum_level = 0; in __skb_reset_checksum_unnecessary()
4052 skb->csum_valid = 1; in __skb_checksum_validate_needed()
4065 /* Unset checksum-complete
4068 * (uncompressed for instance) and checksum-complete value is
4073 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_complete_unset()
4074 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_complete_unset()
4082 * checksum is stored in skb->csum for use in __skb_checksum_complete
4083 * non-zero: value of invalid checksum
4090 if (skb->ip_summed == CHECKSUM_COMPLETE) { in __skb_checksum_validate_complete()
4091 if (!csum_fold(csum_add(psum, skb->csum))) { in __skb_checksum_validate_complete()
4092 skb->csum_valid = 1; in __skb_checksum_validate_complete()
4097 skb->csum = psum; in __skb_checksum_validate_complete()
4099 if (complete || skb->len <= CHECKSUM_BREAK) { in __skb_checksum_validate_complete()
4103 skb->csum_valid = !csum; in __skb_checksum_validate_complete()
4123 * non-zero: value of invalid checksum
4129 skb->csum_valid = 0; \
4154 return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid); in __skb_checksum_convert_check()
4159 skb->csum = ~pseudo; in __skb_checksum_convert()
4160 skb->ip_summed = CHECKSUM_COMPLETE; in __skb_checksum_convert()
4172 skb->ip_summed = CHECKSUM_PARTIAL; in skb_remcsum_adjust_partial()
4173 skb->csum_start = ((unsigned char *)ptr + start) - skb->head; in skb_remcsum_adjust_partial()
4174 skb->csum_offset = offset - start; in skb_remcsum_adjust_partial()
4178 * When called, ptr indicates the starting point for skb->csum when
4180 * here, skb_postpull_rcsum is done so skb->csum start is ptr.
4192 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) { in skb_remcsum_process()
4194 skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data); in skb_remcsum_process()
4197 delta = remcsum_adjust(ptr, skb->csum, start, offset); in skb_remcsum_process()
4199 /* Adjust skb->csum since we changed the packet */ in skb_remcsum_process()
4200 skb->csum = csum_add(skb->csum, delta); in skb_remcsum_process()
4206 return (void *)(skb->_nfct & NFCT_PTRMASK); in skb_nfct()
4215 return skb->_nfct; in skb_get_nfct()
4224 skb->slow_gro |= !!nfct; in skb_set_nfct()
4225 skb->_nfct = nfct; in skb_set_nfct()
4247 * struct skb_ext - sk_buff extensions
4272 if (skb->active_extensions) in skb_ext_put()
4273 __skb_ext_put(skb->extensions); in skb_ext_put()
4279 dst->active_extensions = src->active_extensions; in __skb_ext_copy()
4281 if (src->active_extensions) { in __skb_ext_copy()
4282 struct skb_ext *ext = src->extensions; in __skb_ext_copy()
4284 refcount_inc(&ext->refcnt); in __skb_ext_copy()
4285 dst->extensions = ext; in __skb_ext_copy()
4297 return !!ext->offset[i]; in __skb_ext_exist()
4302 return skb->active_extensions & (1 << id); in skb_ext_exist()
4314 struct skb_ext *ext = skb->extensions; in skb_ext_find()
4316 return (void *)ext + (ext->offset[id] << 3); in skb_ext_find()
4324 if (unlikely(skb->active_extensions)) { in skb_ext_reset()
4325 __skb_ext_put(skb->extensions); in skb_ext_reset()
4326 skb->active_extensions = 0; in skb_ext_reset()
4332 return unlikely(skb->active_extensions); in skb_has_extensions()
4347 skb->_nfct = 0; in nf_reset_ct()
4354 skb->nf_trace = 0; in nf_reset_trace()
4361 skb->ipvs_property = 0; in ipvs_reset()
4370 dst->_nfct = src->_nfct; in __nf_copy()
4375 dst->nf_trace = src->nf_trace; in __nf_copy()
4384 dst->slow_gro = src->slow_gro; in nf_copy()
4391 to->secmark = from->secmark; in skb_copy_secmark()
4396 skb->secmark = 0; in skb_init_secmark()
4417 return !skb->destructor && in skb_irq_freeable()
4420 !skb->_skb_refdst && in skb_irq_freeable()
4426 skb->queue_mapping = queue_mapping; in skb_set_queue_mapping()
4431 return skb->queue_mapping; in skb_get_queue_mapping()
4436 to->queue_mapping = from->queue_mapping; in skb_copy_queue_mapping()
4441 skb->queue_mapping = rx_queue + 1; in skb_record_rx_queue()
4446 return skb->queue_mapping - 1; in skb_get_rx_queue()
4451 return skb->queue_mapping != 0; in skb_rx_queue_recorded()
4456 skb->dst_pending_confirm = val; in skb_set_dst_pending_confirm()
4461 return skb->dst_pending_confirm != 0; in skb_get_dst_pending_confirm()
4473 /* Keeps track of mac header offset relative to skb->head.
4475 * For non-tunnel skb it points to skb_mac_header() and for
4489 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET))
4493 return (skb_mac_header(inner_skb) - inner_skb->head) - in skb_tnl_header_len()
4494 SKB_GSO_CB(inner_skb)->mac_offset; in skb_tnl_header_len()
4508 SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom); in gso_pskb_expand_head()
4515 if (skb->remcsum_offload) in gso_reset_checksum()
4518 SKB_GSO_CB(skb)->csum = res; in gso_reset_checksum()
4519 SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head; in gso_reset_checksum()
4523 * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
4524 * then add in skb->csum (checksum from csum_start to end of packet).
4525 * skb->csum and csum_start are then updated to reflect the checksum of the
4526 * resultant packet starting from the transport header-- the resultant checksum
4533 int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start; in gso_make_checksum()
4534 __wsum partial = SKB_GSO_CB(skb)->csum; in gso_make_checksum()
4536 SKB_GSO_CB(skb)->csum = res; in gso_make_checksum()
4537 SKB_GSO_CB(skb)->csum_start = csum_start - skb->head; in gso_make_checksum()
4544 return skb_shinfo(skb)->gso_size; in skb_is_gso()
4550 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; in skb_is_gso_v6()
4556 return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP; in skb_is_gso_sctp()
4562 return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6); in skb_is_gso_tcp()
4567 skb_shinfo(skb)->gso_size = 0; in skb_gso_reset()
4568 skb_shinfo(skb)->gso_segs = 0; in skb_gso_reset()
4569 skb_shinfo(skb)->gso_type = 0; in skb_gso_reset()
4575 if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS)) in skb_increase_gso_size()
4577 shinfo->gso_size += increment; in skb_increase_gso_size()
4583 if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS)) in skb_decrease_gso_size()
4585 shinfo->gso_size -= decrement; in skb_decrease_gso_size()
4596 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 && in skb_warn_if_lro()
4597 unlikely(shinfo->gso_type == 0)) { in skb_warn_if_lro()
4607 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_forward_csum()
4608 skb->ip_summed = CHECKSUM_NONE; in skb_forward_csum()
4612 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
4622 BUG_ON(skb->ip_summed != CHECKSUM_NONE); in skb_checksum_none_assert()
4634 * skb_head_is_locked - Determine if the skb->head is locked down
4644 return !skb->head_frag || skb_cloned(skb); in skb_head_is_locked()
4650 * See Documentation/networking/checksum-offloads.rst for
4653 * pseudo-header) before calling.
4664 skb->csum_offset)); in lco_csum()
4669 return csum_partial(l4_hdr, csum_start - l4_hdr, partial); in lco_csum()
4674 return skb->redirected; in skb_is_redirected()
4679 skb->redirected = 1; in skb_set_redirected()
4681 skb->from_ingress = from_ingress; in skb_set_redirected()
4682 if (skb->from_ingress) in skb_set_redirected()
4683 skb->tstamp = 0; in skb_set_redirected()
4689 skb->redirected = 0; in skb_reset_redirect()
4694 return skb->csum_not_inet; in skb_csum_is_sctp()
4701 skb->kcov_handle = kcov_handle; in skb_set_kcov_handle()
4708 return skb->kcov_handle; in skb_get_kcov_handle()
4717 skb->pp_recycle = 1; in skb_mark_for_recycle()
4723 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) in skb_pp_recycle()