Lines Matching refs:skb
884 void (*destructor)(struct sk_buff *skb);
1089 static inline bool skb_pfmemalloc(const struct sk_buff *skb) in skb_pfmemalloc() argument
1091 return unlikely(skb->pfmemalloc); in skb_pfmemalloc()
1107 static inline struct dst_entry *skb_dst(const struct sk_buff *skb) in skb_dst() argument
1112 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) && in skb_dst()
1115 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK); in skb_dst()
1126 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst) in skb_dst_set() argument
1128 skb->slow_gro |= !!dst; in skb_dst_set()
1129 skb->_skb_refdst = (unsigned long)dst; in skb_dst_set()
1142 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst) in skb_dst_set_noref() argument
1145 skb->slow_gro |= !!dst; in skb_dst_set_noref()
1146 skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; in skb_dst_set_noref()
1153 static inline bool skb_dst_is_noref(const struct sk_buff *skb) in skb_dst_is_noref() argument
1155 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb); in skb_dst_is_noref()
1162 static inline struct rtable *skb_rtable(const struct sk_buff *skb) in skb_rtable() argument
1164 return (struct rtable *)skb_dst(skb); in skb_rtable()
1180 static inline unsigned int skb_napi_id(const struct sk_buff *skb) in skb_napi_id() argument
1183 return skb->napi_id; in skb_napi_id()
1195 static inline bool skb_unref(struct sk_buff *skb) in skb_unref() argument
1197 if (unlikely(!skb)) in skb_unref()
1199 if (likely(refcount_read(&skb->users) == 1)) in skb_unref()
1201 else if (likely(!refcount_dec_and_test(&skb->users))) in skb_unref()
1208 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason);
1214 static inline void kfree_skb(struct sk_buff *skb) in kfree_skb() argument
1216 kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED); in kfree_skb()
1219 void skb_release_head_state(struct sk_buff *skb);
1222 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1223 void skb_tx_error(struct sk_buff *skb);
1231 void consume_skb(struct sk_buff *skb);
1233 static inline void consume_skb(struct sk_buff *skb) in consume_skb() argument
1235 return kfree_skb(skb); in consume_skb()
1239 void __consume_stateless_skb(struct sk_buff *skb);
1240 void __kfree_skb(struct sk_buff *skb);
1243 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1251 struct sk_buff *build_skb_around(struct sk_buff *skb,
1253 void skb_attempt_defer_free(struct sk_buff *skb);
1296 const struct sk_buff *skb) in skb_fclone_busy() argument
1300 fclones = container_of(skb, struct sk_buff_fclones, skb1); in skb_fclone_busy()
1302 return skb->fclone == SKB_FCLONE_ORIG && in skb_fclone_busy()
1321 void skb_headers_offset_update(struct sk_buff *skb, int off);
1322 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
1323 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
1325 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
1326 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1328 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, in __pskb_copy() argument
1331 return __pskb_copy_fclone(skb, headroom, gfp_mask, false); in __pskb_copy()
1334 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
1335 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
1337 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom);
1338 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
1340 int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
1342 int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
1344 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
1345 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
1358 static inline int skb_pad(struct sk_buff *skb, int pad) in skb_pad() argument
1360 return __skb_pad(skb, pad, true); in skb_pad()
1364 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
1378 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1384 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1420 static inline void skb_clear_hash(struct sk_buff *skb) in skb_clear_hash() argument
1422 skb->hash = 0; in skb_clear_hash()
1423 skb->sw_hash = 0; in skb_clear_hash()
1424 skb->l4_hash = 0; in skb_clear_hash()
1427 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb) in skb_clear_hash_if_not_l4() argument
1429 if (!skb->l4_hash) in skb_clear_hash_if_not_l4()
1430 skb_clear_hash(skb); in skb_clear_hash_if_not_l4()
1434 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4) in __skb_set_hash() argument
1436 skb->l4_hash = is_l4; in __skb_set_hash()
1437 skb->sw_hash = is_sw; in __skb_set_hash()
1438 skb->hash = hash; in __skb_set_hash()
1442 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type) in skb_set_hash() argument
1445 __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4); in skb_set_hash()
1449 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4) in __skb_set_sw_hash() argument
1451 __skb_set_hash(skb, hash, true, is_l4); in __skb_set_sw_hash()
1454 void __skb_get_hash(struct sk_buff *skb);
1455 u32 __skb_get_hash_symmetric(const struct sk_buff *skb);
1456 u32 skb_get_poff(const struct sk_buff *skb);
1457 u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1459 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
1462 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb, in skb_flow_get_ports() argument
1465 return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0); in skb_flow_get_ports()
1477 const struct sk_buff *skb,
1482 static inline bool skb_flow_dissect(const struct sk_buff *skb, in skb_flow_dissect() argument
1486 return __skb_flow_dissect(NULL, skb, flow_dissector, in skb_flow_dissect()
1490 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb, in skb_flow_dissect_flow_keys() argument
1495 return __skb_flow_dissect(NULL, skb, &flow_keys_dissector, in skb_flow_dissect_flow_keys()
1501 const struct sk_buff *skb, in skb_flow_dissect_flow_keys_basic() argument
1507 return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow, in skb_flow_dissect_flow_keys_basic()
1511 void skb_flow_dissect_meta(const struct sk_buff *skb,
1520 skb_flow_dissect_ct(const struct sk_buff *skb,
1526 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
1530 void skb_flow_dissect_hash(const struct sk_buff *skb,
1534 static inline __u32 skb_get_hash(struct sk_buff *skb) in skb_get_hash() argument
1536 if (!skb->l4_hash && !skb->sw_hash) in skb_get_hash()
1537 __skb_get_hash(skb); in skb_get_hash()
1539 return skb->hash; in skb_get_hash()
1542 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) in skb_get_hash_flowi6() argument
1544 if (!skb->l4_hash && !skb->sw_hash) { in skb_get_hash_flowi6()
1548 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); in skb_get_hash_flowi6()
1551 return skb->hash; in skb_get_hash_flowi6()
1554 __u32 skb_get_hash_perturb(const struct sk_buff *skb,
1557 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb) in skb_get_hash_raw() argument
1559 return skb->hash; in skb_get_hash_raw()
1578 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) in skb_end_pointer() argument
1580 return skb->head + skb->end; in skb_end_pointer()
1583 static inline unsigned int skb_end_offset(const struct sk_buff *skb) in skb_end_offset() argument
1585 return skb->end; in skb_end_offset()
1588 static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) in skb_set_end_offset() argument
1590 skb->end = offset; in skb_set_end_offset()
1593 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) in skb_end_pointer() argument
1595 return skb->end; in skb_end_pointer()
1598 static inline unsigned int skb_end_offset(const struct sk_buff *skb) in skb_end_offset() argument
1600 return skb->end - skb->head; in skb_end_offset()
1603 static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) in skb_set_end_offset() argument
1605 skb->end = skb->head + offset; in skb_set_end_offset()
1614 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1618 struct sk_buff *skb, struct iov_iter *from,
1621 static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb, in skb_zerocopy_iter_dgram() argument
1624 return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len); in skb_zerocopy_iter_dgram()
1627 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1634 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) in skb_hwtstamps() argument
1636 return &skb_shinfo(skb)->hwtstamps; in skb_hwtstamps()
1639 static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb) in skb_zcopy() argument
1641 bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE; in skb_zcopy()
1643 return is_zcopy ? skb_uarg(skb) : NULL; in skb_zcopy()
1646 static inline bool skb_zcopy_pure(const struct sk_buff *skb) in skb_zcopy_pure() argument
1648 return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY; in skb_zcopy_pure()
1651 static inline bool skb_zcopy_managed(const struct sk_buff *skb) in skb_zcopy_managed() argument
1653 return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS; in skb_zcopy_managed()
1667 static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg) in skb_zcopy_init() argument
1669 skb_shinfo(skb)->destructor_arg = uarg; in skb_zcopy_init()
1670 skb_shinfo(skb)->flags |= uarg->flags; in skb_zcopy_init()
1673 static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, in skb_zcopy_set() argument
1676 if (skb && uarg && !skb_zcopy(skb)) { in skb_zcopy_set()
1681 skb_zcopy_init(skb, uarg); in skb_zcopy_set()
1685 static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val) in skb_zcopy_set_nouarg() argument
1687 skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); in skb_zcopy_set_nouarg()
1688 skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; in skb_zcopy_set_nouarg()
1691 static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb) in skb_zcopy_is_nouarg() argument
1693 return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL; in skb_zcopy_is_nouarg()
1696 static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) in skb_zcopy_get_nouarg() argument
1698 return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); in skb_zcopy_get_nouarg()
1718 static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) in skb_zcopy_clear() argument
1720 struct ubuf_info *uarg = skb_zcopy(skb); in skb_zcopy_clear()
1723 if (!skb_zcopy_is_nouarg(skb)) in skb_zcopy_clear()
1724 uarg->callback(skb, uarg, zerocopy_success); in skb_zcopy_clear()
1726 skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY; in skb_zcopy_clear()
1730 void __skb_zcopy_downgrade_managed(struct sk_buff *skb);
1732 static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb) in skb_zcopy_downgrade_managed() argument
1734 if (unlikely(skb_zcopy_managed(skb))) in skb_zcopy_downgrade_managed()
1735 __skb_zcopy_downgrade_managed(skb); in skb_zcopy_downgrade_managed()
1738 static inline void skb_mark_not_on_list(struct sk_buff *skb) in skb_mark_not_on_list() argument
1740 skb->next = NULL; in skb_mark_not_on_list()
1744 #define skb_list_walk_safe(first, skb, next_skb) \ argument
1745 for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \
1746 (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)
1748 static inline void skb_list_del_init(struct sk_buff *skb) in skb_list_del_init() argument
1750 __list_del_entry(&skb->list); in skb_list_del_init()
1751 skb_mark_not_on_list(skb); in skb_list_del_init()
1786 const struct sk_buff *skb) in skb_queue_is_last() argument
1788 return skb->next == (const struct sk_buff *) list; in skb_queue_is_last()
1799 const struct sk_buff *skb) in skb_queue_is_first() argument
1801 return skb->prev == (const struct sk_buff *) list; in skb_queue_is_first()
1813 const struct sk_buff *skb) in skb_queue_next() argument
1818 BUG_ON(skb_queue_is_last(list, skb)); in skb_queue_next()
1819 return skb->next; in skb_queue_next()
1831 const struct sk_buff *skb) in skb_queue_prev() argument
1836 BUG_ON(skb_queue_is_first(list, skb)); in skb_queue_prev()
1837 return skb->prev; in skb_queue_prev()
1847 static inline struct sk_buff *skb_get(struct sk_buff *skb) in skb_get() argument
1849 refcount_inc(&skb->users); in skb_get()
1850 return skb; in skb_get()
1865 static inline int skb_cloned(const struct sk_buff *skb) in skb_cloned() argument
1867 return skb->cloned && in skb_cloned()
1868 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1; in skb_cloned()
1871 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) in skb_unclone() argument
1875 if (skb_cloned(skb)) in skb_unclone()
1876 return pskb_expand_head(skb, 0, 0, pri); in skb_unclone()
1887 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
1888 static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) in skb_unclone_keeptruesize() argument
1892 if (skb_cloned(skb)) in skb_unclone_keeptruesize()
1893 return __skb_unclone_keeptruesize(skb, pri); in skb_unclone_keeptruesize()
1904 static inline int skb_header_cloned(const struct sk_buff *skb) in skb_header_cloned() argument
1908 if (!skb->cloned) in skb_header_cloned()
1911 dataref = atomic_read(&skb_shinfo(skb)->dataref); in skb_header_cloned()
1916 static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri) in skb_header_unclone() argument
1920 if (skb_header_cloned(skb)) in skb_header_unclone()
1921 return pskb_expand_head(skb, 0, 0, pri); in skb_header_unclone()
1932 static inline void __skb_header_release(struct sk_buff *skb) in __skb_header_release() argument
1934 skb->nohdr = 1; in __skb_header_release()
1935 atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT)); in __skb_header_release()
1946 static inline int skb_shared(const struct sk_buff *skb) in skb_shared() argument
1948 return refcount_read(&skb->users) != 1; in skb_shared()
1964 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri) in skb_share_check() argument
1967 if (skb_shared(skb)) { in skb_share_check()
1968 struct sk_buff *nskb = skb_clone(skb, pri); in skb_share_check()
1971 consume_skb(skb); in skb_share_check()
1973 kfree_skb(skb); in skb_share_check()
1974 skb = nskb; in skb_share_check()
1976 return skb; in skb_share_check()
1999 static inline struct sk_buff *skb_unshare(struct sk_buff *skb, in skb_unshare() argument
2003 if (skb_cloned(skb)) { in skb_unshare()
2004 struct sk_buff *nskb = skb_copy(skb, pri); in skb_unshare()
2008 consume_skb(skb); in skb_unshare()
2010 kfree_skb(skb); in skb_unshare()
2011 skb = nskb; in skb_unshare()
2013 return skb; in skb_unshare()
2031 struct sk_buff *skb = list_->next; in skb_peek() local
2033 if (skb == (struct sk_buff *)list_) in skb_peek()
2034 skb = NULL; in skb_peek()
2035 return skb; in skb_peek()
2058 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb, in skb_peek_next() argument
2061 struct sk_buff *next = skb->next; in skb_peek_next()
2083 struct sk_buff *skb = READ_ONCE(list_->prev); in skb_peek_tail() local
2085 if (skb == (struct sk_buff *)list_) in skb_peek_tail()
2086 skb = NULL; in skb_peek_tail()
2087 return skb; in skb_peek_tail()
2314 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
2315 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) in __skb_unlink() argument
2320 next = skb->next; in __skb_unlink()
2321 prev = skb->prev; in __skb_unlink()
2322 skb->next = skb->prev = NULL; in __skb_unlink()
2337 struct sk_buff *skb = skb_peek(list); in __skb_dequeue() local
2338 if (skb) in __skb_dequeue()
2339 __skb_unlink(skb, list); in __skb_dequeue()
2340 return skb; in __skb_dequeue()
2354 struct sk_buff *skb = skb_peek_tail(list); in __skb_dequeue_tail() local
2355 if (skb) in __skb_dequeue_tail()
2356 __skb_unlink(skb, list); in __skb_dequeue_tail()
2357 return skb; in __skb_dequeue_tail()
2362 static inline bool skb_is_nonlinear(const struct sk_buff *skb) in skb_is_nonlinear() argument
2364 return skb->data_len; in skb_is_nonlinear()
2367 static inline unsigned int skb_headlen(const struct sk_buff *skb) in skb_headlen() argument
2369 return skb->len - skb->data_len; in skb_headlen()
2372 static inline unsigned int __skb_pagelen(const struct sk_buff *skb) in __skb_pagelen() argument
2376 for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--) in __skb_pagelen()
2377 len += skb_frag_size(&skb_shinfo(skb)->frags[i]); in __skb_pagelen()
2381 static inline unsigned int skb_pagelen(const struct sk_buff *skb) in skb_pagelen() argument
2383 return skb_headlen(skb) + __skb_pagelen(skb); in skb_pagelen()
2407 static inline void skb_len_add(struct sk_buff *skb, int delta) in skb_len_add() argument
2409 skb->len += delta; in skb_len_add()
2410 skb->data_len += delta; in skb_len_add()
2411 skb->truesize += delta; in skb_len_add()
2427 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, in __skb_fill_page_desc() argument
2430 __skb_fill_page_desc_noacc(skb_shinfo(skb), i, page, off, size); in __skb_fill_page_desc()
2433 skb->pfmemalloc = true; in __skb_fill_page_desc()
2450 static inline void skb_fill_page_desc(struct sk_buff *skb, int i, in skb_fill_page_desc() argument
2453 __skb_fill_page_desc(skb, i, page, off, size); in skb_fill_page_desc()
2454 skb_shinfo(skb)->nr_frags = i + 1; in skb_fill_page_desc()
2468 static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i, in skb_fill_page_desc_noacc() argument
2472 struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_fill_page_desc_noacc()
2478 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
2481 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
2484 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb)) argument
2487 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) in skb_tail_pointer() argument
2489 return skb->head + skb->tail; in skb_tail_pointer()
2492 static inline void skb_reset_tail_pointer(struct sk_buff *skb) in skb_reset_tail_pointer() argument
2494 skb->tail = skb->data - skb->head; in skb_reset_tail_pointer()
2497 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) in skb_set_tail_pointer() argument
2499 skb_reset_tail_pointer(skb); in skb_set_tail_pointer()
2500 skb->tail += offset; in skb_set_tail_pointer()
2504 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) in skb_tail_pointer() argument
2506 return skb->tail; in skb_tail_pointer()
2509 static inline void skb_reset_tail_pointer(struct sk_buff *skb) in skb_reset_tail_pointer() argument
2511 skb->tail = skb->data; in skb_reset_tail_pointer()
2514 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) in skb_set_tail_pointer() argument
2516 skb->tail = skb->data + offset; in skb_set_tail_pointer()
2521 static inline void skb_assert_len(struct sk_buff *skb) in skb_assert_len() argument
2524 if (WARN_ONCE(!skb->len, "%s\n", __func__)) in skb_assert_len()
2525 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); in skb_assert_len()
2532 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
2533 void *skb_put(struct sk_buff *skb, unsigned int len);
2534 static inline void *__skb_put(struct sk_buff *skb, unsigned int len) in __skb_put() argument
2536 void *tmp = skb_tail_pointer(skb); in __skb_put()
2537 SKB_LINEAR_ASSERT(skb); in __skb_put()
2538 skb->tail += len; in __skb_put()
2539 skb->len += len; in __skb_put()
2543 static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len) in __skb_put_zero() argument
2545 void *tmp = __skb_put(skb, len); in __skb_put_zero()
2551 static inline void *__skb_put_data(struct sk_buff *skb, const void *data, in __skb_put_data() argument
2554 void *tmp = __skb_put(skb, len); in __skb_put_data()
2560 static inline void __skb_put_u8(struct sk_buff *skb, u8 val) in __skb_put_u8() argument
2562 *(u8 *)__skb_put(skb, 1) = val; in __skb_put_u8()
2565 static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len) in skb_put_zero() argument
2567 void *tmp = skb_put(skb, len); in skb_put_zero()
2574 static inline void *skb_put_data(struct sk_buff *skb, const void *data, in skb_put_data() argument
2577 void *tmp = skb_put(skb, len); in skb_put_data()
2584 static inline void skb_put_u8(struct sk_buff *skb, u8 val) in skb_put_u8() argument
2586 *(u8 *)skb_put(skb, 1) = val; in skb_put_u8()
2589 void *skb_push(struct sk_buff *skb, unsigned int len);
2590 static inline void *__skb_push(struct sk_buff *skb, unsigned int len) in __skb_push() argument
2592 skb->data -= len; in __skb_push()
2593 skb->len += len; in __skb_push()
2594 return skb->data; in __skb_push()
2597 void *skb_pull(struct sk_buff *skb, unsigned int len);
2598 static inline void *__skb_pull(struct sk_buff *skb, unsigned int len) in __skb_pull() argument
2600 skb->len -= len; in __skb_pull()
2601 if (unlikely(skb->len < skb->data_len)) { in __skb_pull()
2603 skb->len += len; in __skb_pull()
2605 skb_dump(KERN_ERR, skb, false); in __skb_pull()
2609 return skb->data += len; in __skb_pull()
2612 static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len) in skb_pull_inline() argument
2614 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); in skb_pull_inline()
2617 void *skb_pull_data(struct sk_buff *skb, size_t len);
2619 void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2621 static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len) in pskb_may_pull() argument
2623 if (likely(len <= skb_headlen(skb))) in pskb_may_pull()
2625 if (unlikely(len > skb->len)) in pskb_may_pull()
2627 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; in pskb_may_pull()
2630 static inline void *pskb_pull(struct sk_buff *skb, unsigned int len) in pskb_pull() argument
2632 if (!pskb_may_pull(skb, len)) in pskb_pull()
2635 skb->len -= len; in pskb_pull()
2636 return skb->data += len; in pskb_pull()
2639 void skb_condense(struct sk_buff *skb);
2647 static inline unsigned int skb_headroom(const struct sk_buff *skb) in skb_headroom() argument
2649 return skb->data - skb->head; in skb_headroom()
2658 static inline int skb_tailroom(const struct sk_buff *skb) in skb_tailroom() argument
2660 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail; in skb_tailroom()
2670 static inline int skb_availroom(const struct sk_buff *skb) in skb_availroom() argument
2672 if (skb_is_nonlinear(skb)) in skb_availroom()
2675 return skb->end - skb->tail - skb->reserved_tailroom; in skb_availroom()
2686 static inline void skb_reserve(struct sk_buff *skb, int len) in skb_reserve() argument
2688 skb->data += len; in skb_reserve()
2689 skb->tail += len; in skb_reserve()
2704 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu, in skb_tailroom_reserve() argument
2707 SKB_LINEAR_ASSERT(skb); in skb_tailroom_reserve()
2708 if (mtu < skb_tailroom(skb) - needed_tailroom) in skb_tailroom_reserve()
2710 skb->reserved_tailroom = skb_tailroom(skb) - mtu; in skb_tailroom_reserve()
2713 skb->reserved_tailroom = needed_tailroom; in skb_tailroom_reserve()
2719 static inline void skb_set_inner_protocol(struct sk_buff *skb, in skb_set_inner_protocol() argument
2722 skb->inner_protocol = protocol; in skb_set_inner_protocol()
2723 skb->inner_protocol_type = ENCAP_TYPE_ETHER; in skb_set_inner_protocol()
2726 static inline void skb_set_inner_ipproto(struct sk_buff *skb, in skb_set_inner_ipproto() argument
2729 skb->inner_ipproto = ipproto; in skb_set_inner_ipproto()
2730 skb->inner_protocol_type = ENCAP_TYPE_IPPROTO; in skb_set_inner_ipproto()
2733 static inline void skb_reset_inner_headers(struct sk_buff *skb) in skb_reset_inner_headers() argument
2735 skb->inner_mac_header = skb->mac_header; in skb_reset_inner_headers()
2736 skb->inner_network_header = skb->network_header; in skb_reset_inner_headers()
2737 skb->inner_transport_header = skb->transport_header; in skb_reset_inner_headers()
2740 static inline void skb_reset_mac_len(struct sk_buff *skb) in skb_reset_mac_len() argument
2742 skb->mac_len = skb->network_header - skb->mac_header; in skb_reset_mac_len()
2746 *skb) in skb_inner_transport_header()
2748 return skb->head + skb->inner_transport_header; in skb_inner_transport_header()
2751 static inline int skb_inner_transport_offset(const struct sk_buff *skb) in skb_inner_transport_offset() argument
2753 return skb_inner_transport_header(skb) - skb->data; in skb_inner_transport_offset()
2756 static inline void skb_reset_inner_transport_header(struct sk_buff *skb) in skb_reset_inner_transport_header() argument
2758 skb->inner_transport_header = skb->data - skb->head; in skb_reset_inner_transport_header()
2761 static inline void skb_set_inner_transport_header(struct sk_buff *skb, in skb_set_inner_transport_header() argument
2764 skb_reset_inner_transport_header(skb); in skb_set_inner_transport_header()
2765 skb->inner_transport_header += offset; in skb_set_inner_transport_header()
2768 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb) in skb_inner_network_header() argument
2770 return skb->head + skb->inner_network_header; in skb_inner_network_header()
2773 static inline void skb_reset_inner_network_header(struct sk_buff *skb) in skb_reset_inner_network_header() argument
2775 skb->inner_network_header = skb->data - skb->head; in skb_reset_inner_network_header()
2778 static inline void skb_set_inner_network_header(struct sk_buff *skb, in skb_set_inner_network_header() argument
2781 skb_reset_inner_network_header(skb); in skb_set_inner_network_header()
2782 skb->inner_network_header += offset; in skb_set_inner_network_header()
2785 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb) in skb_inner_mac_header() argument
2787 return skb->head + skb->inner_mac_header; in skb_inner_mac_header()
2790 static inline void skb_reset_inner_mac_header(struct sk_buff *skb) in skb_reset_inner_mac_header() argument
2792 skb->inner_mac_header = skb->data - skb->head; in skb_reset_inner_mac_header()
2795 static inline void skb_set_inner_mac_header(struct sk_buff *skb, in skb_set_inner_mac_header() argument
2798 skb_reset_inner_mac_header(skb); in skb_set_inner_mac_header()
2799 skb->inner_mac_header += offset; in skb_set_inner_mac_header()
2801 static inline bool skb_transport_header_was_set(const struct sk_buff *skb) in skb_transport_header_was_set() argument
2803 return skb->transport_header != (typeof(skb->transport_header))~0U; in skb_transport_header_was_set()
2806 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) in skb_transport_header() argument
2808 DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb)); in skb_transport_header()
2809 return skb->head + skb->transport_header; in skb_transport_header()
2812 static inline void skb_reset_transport_header(struct sk_buff *skb) in skb_reset_transport_header() argument
2814 skb->transport_header = skb->data - skb->head; in skb_reset_transport_header()
2817 static inline void skb_set_transport_header(struct sk_buff *skb, in skb_set_transport_header() argument
2820 skb_reset_transport_header(skb); in skb_set_transport_header()
2821 skb->transport_header += offset; in skb_set_transport_header()
2824 static inline unsigned char *skb_network_header(const struct sk_buff *skb) in skb_network_header() argument
2826 return skb->head + skb->network_header; in skb_network_header()
2829 static inline void skb_reset_network_header(struct sk_buff *skb) in skb_reset_network_header() argument
2831 skb->network_header = skb->data - skb->head; in skb_reset_network_header()
2834 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) in skb_set_network_header() argument
2836 skb_reset_network_header(skb); in skb_set_network_header()
2837 skb->network_header += offset; in skb_set_network_header()
2840 static inline int skb_mac_header_was_set(const struct sk_buff *skb) in skb_mac_header_was_set() argument
2842 return skb->mac_header != (typeof(skb->mac_header))~0U; in skb_mac_header_was_set()
2845 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) in skb_mac_header() argument
2847 DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); in skb_mac_header()
2848 return skb->head + skb->mac_header; in skb_mac_header()
2851 static inline int skb_mac_offset(const struct sk_buff *skb) in skb_mac_offset() argument
2853 return skb_mac_header(skb) - skb->data; in skb_mac_offset()
2856 static inline u32 skb_mac_header_len(const struct sk_buff *skb) in skb_mac_header_len() argument
2858 DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); in skb_mac_header_len()
2859 return skb->network_header - skb->mac_header; in skb_mac_header_len()
2862 static inline void skb_unset_mac_header(struct sk_buff *skb) in skb_unset_mac_header() argument
2864 skb->mac_header = (typeof(skb->mac_header))~0U; in skb_unset_mac_header()
2867 static inline void skb_reset_mac_header(struct sk_buff *skb) in skb_reset_mac_header() argument
2869 skb->mac_header = skb->data - skb->head; in skb_reset_mac_header()
2872 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) in skb_set_mac_header() argument
2874 skb_reset_mac_header(skb); in skb_set_mac_header()
2875 skb->mac_header += offset; in skb_set_mac_header()
2878 static inline void skb_pop_mac_header(struct sk_buff *skb) in skb_pop_mac_header() argument
2880 skb->mac_header = skb->network_header; in skb_pop_mac_header()
2883 static inline void skb_probe_transport_header(struct sk_buff *skb) in skb_probe_transport_header() argument
2887 if (skb_transport_header_was_set(skb)) in skb_probe_transport_header()
2890 if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, in skb_probe_transport_header()
2892 skb_set_transport_header(skb, keys.control.thoff); in skb_probe_transport_header()
2895 static inline void skb_mac_header_rebuild(struct sk_buff *skb) in skb_mac_header_rebuild() argument
2897 if (skb_mac_header_was_set(skb)) { in skb_mac_header_rebuild()
2898 const unsigned char *old_mac = skb_mac_header(skb); in skb_mac_header_rebuild()
2900 skb_set_mac_header(skb, -skb->mac_len); in skb_mac_header_rebuild()
2901 memmove(skb_mac_header(skb), old_mac, skb->mac_len); in skb_mac_header_rebuild()
2905 static inline int skb_checksum_start_offset(const struct sk_buff *skb) in skb_checksum_start_offset() argument
2907 return skb->csum_start - skb_headroom(skb); in skb_checksum_start_offset()
2910 static inline unsigned char *skb_checksum_start(const struct sk_buff *skb) in skb_checksum_start() argument
2912 return skb->head + skb->csum_start; in skb_checksum_start()
2915 static inline int skb_transport_offset(const struct sk_buff *skb) in skb_transport_offset() argument
2917 return skb_transport_header(skb) - skb->data; in skb_transport_offset()
2920 static inline u32 skb_network_header_len(const struct sk_buff *skb) in skb_network_header_len() argument
2922 return skb->transport_header - skb->network_header; in skb_network_header_len()
2925 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb) in skb_inner_network_header_len() argument
2927 return skb->inner_transport_header - skb->inner_network_header; in skb_inner_network_header_len()
2930 static inline int skb_network_offset(const struct sk_buff *skb) in skb_network_offset() argument
2932 return skb_network_header(skb) - skb->data; in skb_network_offset()
2935 static inline int skb_inner_network_offset(const struct sk_buff *skb) in skb_inner_network_offset() argument
2937 return skb_inner_network_header(skb) - skb->data; in skb_inner_network_offset()
2940 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len) in pskb_network_may_pull() argument
2942 return pskb_may_pull(skb, skb_network_offset(skb) + len); in pskb_network_may_pull()
2993 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
2995 static inline void __skb_set_length(struct sk_buff *skb, unsigned int len) in __skb_set_length() argument
2997 if (WARN_ON(skb_is_nonlinear(skb))) in __skb_set_length()
2999 skb->len = len; in __skb_set_length()
3000 skb_set_tail_pointer(skb, len); in __skb_set_length()
3003 static inline void __skb_trim(struct sk_buff *skb, unsigned int len) in __skb_trim() argument
3005 __skb_set_length(skb, len); in __skb_trim()
3008 void skb_trim(struct sk_buff *skb, unsigned int len);
3010 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) in __pskb_trim() argument
3012 if (skb->data_len) in __pskb_trim()
3013 return ___pskb_trim(skb, len); in __pskb_trim()
3014 __skb_trim(skb, len); in __pskb_trim()
3018 static inline int pskb_trim(struct sk_buff *skb, unsigned int len) in pskb_trim() argument
3020 return (len < skb->len) ? __pskb_trim(skb, len) : 0; in pskb_trim()
3032 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len) in pskb_trim_unique() argument
3034 int err = pskb_trim(skb, len); in pskb_trim_unique()
3038 static inline int __skb_grow(struct sk_buff *skb, unsigned int len) in __skb_grow() argument
3040 unsigned int diff = len - skb->len; in __skb_grow()
3042 if (skb_tailroom(skb) < diff) { in __skb_grow()
3043 int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb), in __skb_grow()
3048 __skb_set_length(skb, len); in __skb_grow()
3060 static inline void skb_orphan(struct sk_buff *skb) in skb_orphan() argument
3062 if (skb->destructor) { in skb_orphan()
3063 skb->destructor(skb); in skb_orphan()
3064 skb->destructor = NULL; in skb_orphan()
3065 skb->sk = NULL; in skb_orphan()
3067 BUG_ON(skb->sk); in skb_orphan()
3080 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) in skb_orphan_frags() argument
3082 if (likely(!skb_zcopy(skb))) in skb_orphan_frags()
3084 if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN) in skb_orphan_frags()
3086 return skb_copy_ubufs(skb, gfp_mask); in skb_orphan_frags()
3090 static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask) in skb_orphan_frags_rx() argument
3092 if (likely(!skb_zcopy(skb))) in skb_orphan_frags_rx()
3094 return skb_copy_ubufs(skb, gfp_mask); in skb_orphan_frags_rx()
3107 struct sk_buff *skb; in __skb_queue_purge() local
3108 while ((skb = __skb_dequeue(list)) != NULL) in __skb_queue_purge()
3109 kfree_skb(skb); in __skb_queue_purge()
3175 struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp); in __netdev_alloc_skb_ip_align() local
3177 if (NET_IP_ALIGN && skb) in __netdev_alloc_skb_ip_align()
3178 skb_reserve(skb, NET_IP_ALIGN); in __netdev_alloc_skb_ip_align()
3179 return skb; in __netdev_alloc_skb_ip_align()
3214 void napi_consume_skb(struct sk_buff *skb, int budget);
3216 void napi_skb_free_stolen_head(struct sk_buff *skb);
3217 void __kfree_skb_defer(struct sk_buff *skb);
3289 struct sk_buff *skb) in skb_propagate_pfmemalloc() argument
3292 skb->pfmemalloc = true; in skb_propagate_pfmemalloc()
3364 static inline void skb_frag_ref(struct sk_buff *skb, int f) in skb_frag_ref() argument
3366 __skb_frag_ref(&skb_shinfo(skb)->frags[f]); in skb_frag_ref()
3395 static inline void skb_frag_unref(struct sk_buff *skb, int f) in skb_frag_unref() argument
3397 struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_frag_unref()
3399 if (!skb_zcopy_managed(skb)) in skb_frag_unref()
3400 __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle); in skb_frag_unref()
3462 static inline void skb_frag_set_page(struct sk_buff *skb, int f, in skb_frag_set_page() argument
3465 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page); in skb_frag_set_page()
3490 static inline struct sk_buff *pskb_copy(struct sk_buff *skb, in pskb_copy() argument
3493 return __pskb_copy(skb, skb_headroom(skb), gfp_mask); in pskb_copy()
3497 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb, in pskb_copy_for_clone() argument
3500 return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true); in pskb_copy_for_clone()
3512 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len) in skb_clone_writable() argument
3514 return !skb_header_cloned(skb) && in skb_clone_writable()
3515 skb_headroom(skb) + len <= skb->hdr_len; in skb_clone_writable()
3518 static inline int skb_try_make_writable(struct sk_buff *skb, in skb_try_make_writable() argument
3521 return skb_cloned(skb) && !skb_clone_writable(skb, write_len) && in skb_try_make_writable()
3522 pskb_expand_head(skb, 0, 0, GFP_ATOMIC); in skb_try_make_writable()
3525 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom, in __skb_cow() argument
3530 if (headroom > skb_headroom(skb)) in __skb_cow()
3531 delta = headroom - skb_headroom(skb); in __skb_cow()
3534 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0, in __skb_cow()
3551 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom) in skb_cow() argument
3553 return __skb_cow(skb, headroom, skb_cloned(skb)); in skb_cow()
3566 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom) in skb_cow_head() argument
3568 return __skb_cow(skb, headroom, skb_header_cloned(skb)); in skb_cow_head()
3581 static inline int skb_padto(struct sk_buff *skb, unsigned int len) in skb_padto() argument
3583 unsigned int size = skb->len; in skb_padto()
3586 return skb_pad(skb, len - size); in skb_padto()
3600 static inline int __must_check __skb_put_padto(struct sk_buff *skb, in __skb_put_padto() argument
3604 unsigned int size = skb->len; in __skb_put_padto()
3608 if (__skb_pad(skb, len, free_on_error)) in __skb_put_padto()
3610 __skb_put(skb, len); in __skb_put_padto()
3625 static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len) in skb_put_padto() argument
3627 return __skb_put_padto(skb, len, true); in skb_put_padto()
3630 static inline int skb_add_data(struct sk_buff *skb, in skb_add_data() argument
3633 const int off = skb->len; in skb_add_data()
3635 if (skb->ip_summed == CHECKSUM_NONE) { in skb_add_data()
3637 if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy, in skb_add_data()
3639 skb->csum = csum_block_add(skb->csum, csum, off); in skb_add_data()
3642 } else if (copy_from_iter_full(skb_put(skb, copy), copy, from)) in skb_add_data()
3645 __skb_trim(skb, off); in skb_add_data()
3649 static inline bool skb_can_coalesce(struct sk_buff *skb, int i, in skb_can_coalesce() argument
3652 if (skb_zcopy(skb)) in skb_can_coalesce()
3655 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; in skb_can_coalesce()
3663 static inline int __skb_linearize(struct sk_buff *skb) in __skb_linearize() argument
3665 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM; in __skb_linearize()
3675 static inline int skb_linearize(struct sk_buff *skb) in skb_linearize() argument
3677 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0; in skb_linearize()
3687 static inline bool skb_has_shared_frag(const struct sk_buff *skb) in skb_has_shared_frag() argument
3689 return skb_is_nonlinear(skb) && in skb_has_shared_frag()
3690 skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; in skb_has_shared_frag()
3700 static inline int skb_linearize_cow(struct sk_buff *skb) in skb_linearize_cow() argument
3702 return skb_is_nonlinear(skb) || skb_cloned(skb) ? in skb_linearize_cow()
3703 __skb_linearize(skb) : 0; in skb_linearize_cow()
3707 __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, in __skb_postpull_rcsum() argument
3710 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_postpull_rcsum()
3711 skb->csum = csum_block_sub(skb->csum, in __skb_postpull_rcsum()
3713 else if (skb->ip_summed == CHECKSUM_PARTIAL && in __skb_postpull_rcsum()
3714 skb_checksum_start_offset(skb) < 0) in __skb_postpull_rcsum()
3715 skb->ip_summed = CHECKSUM_NONE; in __skb_postpull_rcsum()
3728 static inline void skb_postpull_rcsum(struct sk_buff *skb, in skb_postpull_rcsum() argument
3731 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_postpull_rcsum()
3732 skb->csum = wsum_negate(csum_partial(start, len, in skb_postpull_rcsum()
3733 wsum_negate(skb->csum))); in skb_postpull_rcsum()
3734 else if (skb->ip_summed == CHECKSUM_PARTIAL && in skb_postpull_rcsum()
3735 skb_checksum_start_offset(skb) < 0) in skb_postpull_rcsum()
3736 skb->ip_summed = CHECKSUM_NONE; in skb_postpull_rcsum()
3740 __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, in __skb_postpush_rcsum() argument
3743 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_postpush_rcsum()
3744 skb->csum = csum_block_add(skb->csum, in __skb_postpush_rcsum()
3757 static inline void skb_postpush_rcsum(struct sk_buff *skb, in skb_postpush_rcsum() argument
3760 __skb_postpush_rcsum(skb, start, len, 0); in skb_postpush_rcsum()
3763 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
3776 static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len) in skb_push_rcsum() argument
3778 skb_push(skb, len); in skb_push_rcsum()
3779 skb_postpush_rcsum(skb, skb->data, len); in skb_push_rcsum()
3780 return skb->data; in skb_push_rcsum()
3783 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
3794 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) in pskb_trim_rcsum() argument
3796 if (likely(len >= skb->len)) in pskb_trim_rcsum()
3798 return pskb_trim_rcsum_slow(skb, len); in pskb_trim_rcsum()
3801 static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len) in __skb_trim_rcsum() argument
3803 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_trim_rcsum()
3804 skb->ip_summed = CHECKSUM_NONE; in __skb_trim_rcsum()
3805 __skb_trim(skb, len); in __skb_trim_rcsum()
3809 static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len) in __skb_grow_rcsum() argument
3811 if (skb->ip_summed == CHECKSUM_COMPLETE) in __skb_grow_rcsum()
3812 skb->ip_summed = CHECKSUM_NONE; in __skb_grow_rcsum()
3813 return __skb_grow(skb, len); in __skb_grow_rcsum()
3819 #define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode)) argument
3820 #define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode)) argument
3822 #define skb_queue_walk(queue, skb) \ argument
3823 for (skb = (queue)->next; \
3824 skb != (struct sk_buff *)(queue); \
3825 skb = skb->next)
3827 #define skb_queue_walk_safe(queue, skb, tmp) \ argument
3828 for (skb = (queue)->next, tmp = skb->next; \
3829 skb != (struct sk_buff *)(queue); \
3830 skb = tmp, tmp = skb->next)
3832 #define skb_queue_walk_from(queue, skb) \ argument
3833 for (; skb != (struct sk_buff *)(queue); \
3834 skb = skb->next)
3836 #define skb_rbtree_walk(skb, root) \ argument
3837 for (skb = skb_rb_first(root); skb != NULL; \
3838 skb = skb_rb_next(skb))
3840 #define skb_rbtree_walk_from(skb) \ argument
3841 for (; skb != NULL; \
3842 skb = skb_rb_next(skb))
3844 #define skb_rbtree_walk_from_safe(skb, tmp) \ argument
3845 for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL); \
3846 skb = tmp)
3848 #define skb_queue_walk_from_safe(queue, skb, tmp) \ argument
3849 for (tmp = skb->next; \
3850 skb != (struct sk_buff *)(queue); \
3851 skb = tmp, tmp = skb->next)
3853 #define skb_queue_reverse_walk(queue, skb) \ argument
3854 for (skb = (queue)->prev; \
3855 skb != (struct sk_buff *)(queue); \
3856 skb = skb->prev)
3858 #define skb_queue_reverse_walk_safe(queue, skb, tmp) \ argument
3859 for (skb = (queue)->prev, tmp = skb->prev; \
3860 skb != (struct sk_buff *)(queue); \
3861 skb = tmp, tmp = skb->prev)
3863 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \ argument
3864 for (tmp = skb->prev; \
3865 skb != (struct sk_buff *)(queue); \
3866 skb = tmp, tmp = skb->prev)
3868 static inline bool skb_has_frag_list(const struct sk_buff *skb) in skb_has_frag_list() argument
3870 return skb_shinfo(skb)->frag_list != NULL; in skb_has_frag_list()
3873 static inline void skb_frag_list_init(struct sk_buff *skb) in skb_frag_list_init() argument
3875 skb_shinfo(skb)->frag_list = NULL; in skb_frag_list_init()
3878 #define skb_walk_frags(skb, iter) \ argument
3879 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
3884 const struct sk_buff *skb);
3907 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
3909 int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
3912 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
3914 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
3915 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
3916 void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len);
3918 struct sk_buff *skb) in skb_free_datagram_locked() argument
3920 __skb_free_datagram_locked(sk, skb, 0); in skb_free_datagram_locked()
3922 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
3923 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
3924 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
3925 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
3927 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3930 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3932 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
3933 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
3937 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
3938 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
3939 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
3940 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu);
3941 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len);
3942 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
3943 struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
3945 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
3946 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len);
3947 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
3948 int skb_vlan_pop(struct sk_buff *skb);
3949 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
3950 int skb_eth_pop(struct sk_buff *skb);
3951 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
3953 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
3955 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
3957 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
3958 int skb_mpls_dec_ttl(struct sk_buff *skb);
3959 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
3979 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
3981 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
3985 __skb_header_pointer(const struct sk_buff *skb, int offset, int len, in __skb_header_pointer() argument
3991 if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0)) in __skb_header_pointer()
3998 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer) in skb_header_pointer() argument
4000 return __skb_header_pointer(skb, offset, len, skb->data, in skb_header_pointer()
4001 skb_headlen(skb), buffer); in skb_header_pointer()
4014 static inline bool skb_needs_linearize(struct sk_buff *skb, in skb_needs_linearize() argument
4017 return skb_is_nonlinear(skb) && in skb_needs_linearize()
4018 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) || in skb_needs_linearize()
4019 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG))); in skb_needs_linearize()
4022 static inline void skb_copy_from_linear_data(const struct sk_buff *skb, in skb_copy_from_linear_data() argument
4026 memcpy(to, skb->data, len); in skb_copy_from_linear_data()
4029 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb, in skb_copy_from_linear_data_offset() argument
4033 memcpy(to, skb->data + offset, len); in skb_copy_from_linear_data_offset()
4036 static inline void skb_copy_to_linear_data(struct sk_buff *skb, in skb_copy_to_linear_data() argument
4040 memcpy(skb->data, from, len); in skb_copy_to_linear_data()
4043 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb, in skb_copy_to_linear_data_offset() argument
4048 memcpy(skb->data + offset, from, len); in skb_copy_to_linear_data_offset()
4053 static inline ktime_t skb_get_ktime(const struct sk_buff *skb) in skb_get_ktime() argument
4055 return skb->tstamp; in skb_get_ktime()
4067 static inline void skb_get_timestamp(const struct sk_buff *skb, in skb_get_timestamp() argument
4070 *stamp = ns_to_kernel_old_timeval(skb->tstamp); in skb_get_timestamp()
4073 static inline void skb_get_new_timestamp(const struct sk_buff *skb, in skb_get_new_timestamp() argument
4076 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); in skb_get_new_timestamp()
4082 static inline void skb_get_timestampns(const struct sk_buff *skb, in skb_get_timestampns() argument
4085 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); in skb_get_timestampns()
4091 static inline void skb_get_new_timestampns(const struct sk_buff *skb, in skb_get_new_timestampns() argument
4094 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); in skb_get_new_timestampns()
4100 static inline void __net_timestamp(struct sk_buff *skb) in __net_timestamp() argument
4102 skb->tstamp = ktime_get_real(); in __net_timestamp()
4103 skb->mono_delivery_time = 0; in __net_timestamp()
4111 static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt, in skb_set_delivery_time() argument
4114 skb->tstamp = kt; in skb_set_delivery_time()
4115 skb->mono_delivery_time = kt && mono; in skb_set_delivery_time()
4123 static inline void skb_clear_delivery_time(struct sk_buff *skb) in skb_clear_delivery_time() argument
4125 if (skb->mono_delivery_time) { in skb_clear_delivery_time()
4126 skb->mono_delivery_time = 0; in skb_clear_delivery_time()
4128 skb->tstamp = ktime_get_real(); in skb_clear_delivery_time()
4130 skb->tstamp = 0; in skb_clear_delivery_time()
4134 static inline void skb_clear_tstamp(struct sk_buff *skb) in skb_clear_tstamp() argument
4136 if (skb->mono_delivery_time) in skb_clear_tstamp()
4139 skb->tstamp = 0; in skb_clear_tstamp()
4142 static inline ktime_t skb_tstamp(const struct sk_buff *skb) in skb_tstamp() argument
4144 if (skb->mono_delivery_time) in skb_tstamp()
4147 return skb->tstamp; in skb_tstamp()
4150 static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond) in skb_tstamp_cond() argument
4152 if (!skb->mono_delivery_time && skb->tstamp) in skb_tstamp_cond()
4153 return skb->tstamp; in skb_tstamp_cond()
4161 static inline u8 skb_metadata_len(const struct sk_buff *skb) in skb_metadata_len() argument
4163 return skb_shinfo(skb)->meta_len; in skb_metadata_len()
4166 static inline void *skb_metadata_end(const struct sk_buff *skb) in skb_metadata_end() argument
4168 return skb_mac_header(skb); in skb_metadata_end()
4220 static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len) in skb_metadata_set() argument
4222 skb_shinfo(skb)->meta_len = meta_len; in skb_metadata_set()
4225 static inline void skb_metadata_clear(struct sk_buff *skb) in skb_metadata_clear() argument
4227 skb_metadata_set(skb, 0); in skb_metadata_clear()
4230 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
4234 void skb_clone_tx_timestamp(struct sk_buff *skb);
4235 bool skb_defer_rx_timestamp(struct sk_buff *skb);
4239 static inline void skb_clone_tx_timestamp(struct sk_buff *skb) in skb_clone_tx_timestamp() argument
4243 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb) in skb_defer_rx_timestamp() argument
4262 void skb_complete_tx_timestamp(struct sk_buff *skb,
4295 static inline void skb_tx_timestamp(struct sk_buff *skb) in skb_tx_timestamp() argument
4297 skb_clone_tx_timestamp(skb); in skb_tx_timestamp()
4298 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP) in skb_tx_timestamp()
4299 skb_tstamp_tx(skb, NULL); in skb_tx_timestamp()
4309 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
4311 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
4312 __sum16 __skb_checksum_complete(struct sk_buff *skb);
4314 static inline int skb_csum_unnecessary(const struct sk_buff *skb) in skb_csum_unnecessary() argument
4316 return ((skb->ip_summed == CHECKSUM_UNNECESSARY) || in skb_csum_unnecessary()
4317 skb->csum_valid || in skb_csum_unnecessary()
4318 (skb->ip_summed == CHECKSUM_PARTIAL && in skb_csum_unnecessary()
4319 skb_checksum_start_offset(skb) >= 0)); in skb_csum_unnecessary()
4338 static inline __sum16 skb_checksum_complete(struct sk_buff *skb) in skb_checksum_complete() argument
4340 return skb_csum_unnecessary(skb) ? in skb_checksum_complete()
4341 0 : __skb_checksum_complete(skb); in skb_checksum_complete()
4344 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb) in __skb_decr_checksum_unnecessary() argument
4346 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_decr_checksum_unnecessary()
4347 if (skb->csum_level == 0) in __skb_decr_checksum_unnecessary()
4348 skb->ip_summed = CHECKSUM_NONE; in __skb_decr_checksum_unnecessary()
4350 skb->csum_level--; in __skb_decr_checksum_unnecessary()
4354 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb) in __skb_incr_checksum_unnecessary() argument
4356 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_incr_checksum_unnecessary()
4357 if (skb->csum_level < SKB_MAX_CSUM_LEVEL) in __skb_incr_checksum_unnecessary()
4358 skb->csum_level++; in __skb_incr_checksum_unnecessary()
4359 } else if (skb->ip_summed == CHECKSUM_NONE) { in __skb_incr_checksum_unnecessary()
4360 skb->ip_summed = CHECKSUM_UNNECESSARY; in __skb_incr_checksum_unnecessary()
4361 skb->csum_level = 0; in __skb_incr_checksum_unnecessary()
4365 static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb) in __skb_reset_checksum_unnecessary() argument
4367 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in __skb_reset_checksum_unnecessary()
4368 skb->ip_summed = CHECKSUM_NONE; in __skb_reset_checksum_unnecessary()
4369 skb->csum_level = 0; in __skb_reset_checksum_unnecessary()
4378 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb, in __skb_checksum_validate_needed() argument
4382 if (skb_csum_unnecessary(skb) || (zero_okay && !check)) { in __skb_checksum_validate_needed()
4383 skb->csum_valid = 1; in __skb_checksum_validate_needed()
4384 __skb_decr_checksum_unnecessary(skb); in __skb_checksum_validate_needed()
4402 static inline void skb_checksum_complete_unset(struct sk_buff *skb) in skb_checksum_complete_unset() argument
4404 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_complete_unset()
4405 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_complete_unset()
4417 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb, in __skb_checksum_validate_complete() argument
4421 if (skb->ip_summed == CHECKSUM_COMPLETE) { in __skb_checksum_validate_complete()
4422 if (!csum_fold(csum_add(psum, skb->csum))) { in __skb_checksum_validate_complete()
4423 skb->csum_valid = 1; in __skb_checksum_validate_complete()
4428 skb->csum = psum; in __skb_checksum_validate_complete()
4430 if (complete || skb->len <= CHECKSUM_BREAK) { in __skb_checksum_validate_complete()
4433 csum = __skb_checksum_complete(skb); in __skb_checksum_validate_complete()
4434 skb->csum_valid = !csum; in __skb_checksum_validate_complete()
4441 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto) in null_compute_pseudo() argument
4456 #define __skb_checksum_validate(skb, proto, complete, \ argument
4460 skb->csum_valid = 0; \
4461 if (__skb_checksum_validate_needed(skb, zero_okay, check)) \
4462 __ret = __skb_checksum_validate_complete(skb, \
4463 complete, compute_pseudo(skb, proto)); \
4467 #define skb_checksum_init(skb, proto, compute_pseudo) \ argument
4468 __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
4470 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \ argument
4471 __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
4473 #define skb_checksum_validate(skb, proto, compute_pseudo) \ argument
4474 __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
4476 #define skb_checksum_validate_zero_check(skb, proto, check, \ argument
4478 __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
4480 #define skb_checksum_simple_validate(skb) \ argument
4481 __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
4483 static inline bool __skb_checksum_convert_check(struct sk_buff *skb) in __skb_checksum_convert_check() argument
4485 return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid); in __skb_checksum_convert_check()
4488 static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo) in __skb_checksum_convert() argument
4490 skb->csum = ~pseudo; in __skb_checksum_convert()
4491 skb->ip_summed = CHECKSUM_COMPLETE; in __skb_checksum_convert()
4494 #define skb_checksum_try_convert(skb, proto, compute_pseudo) \ argument
4496 if (__skb_checksum_convert_check(skb)) \
4497 __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
4500 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr, in skb_remcsum_adjust_partial() argument
4503 skb->ip_summed = CHECKSUM_PARTIAL; in skb_remcsum_adjust_partial()
4504 skb->csum_start = ((unsigned char *)ptr + start) - skb->head; in skb_remcsum_adjust_partial()
4505 skb->csum_offset = offset - start; in skb_remcsum_adjust_partial()
4513 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr, in skb_remcsum_process() argument
4519 skb_remcsum_adjust_partial(skb, ptr, start, offset); in skb_remcsum_process()
4523 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) { in skb_remcsum_process()
4524 __skb_checksum_complete(skb); in skb_remcsum_process()
4525 skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data); in skb_remcsum_process()
4528 delta = remcsum_adjust(ptr, skb->csum, start, offset); in skb_remcsum_process()
4531 skb->csum = csum_add(skb->csum, delta); in skb_remcsum_process()
4534 static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb) in skb_nfct() argument
4537 return (void *)(skb->_nfct & NFCT_PTRMASK); in skb_nfct()
4543 static inline unsigned long skb_get_nfct(const struct sk_buff *skb) in skb_get_nfct() argument
4546 return skb->_nfct; in skb_get_nfct()
4552 static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct) in skb_set_nfct() argument
4555 skb->slow_gro |= !!nfct; in skb_set_nfct()
4556 skb->_nfct = nfct; in skb_set_nfct()
4598 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
4600 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
4601 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
4604 static inline void skb_ext_put(struct sk_buff *skb) in skb_ext_put() argument
4606 if (skb->active_extensions) in skb_ext_put()
4607 __skb_ext_put(skb->extensions); in skb_ext_put()
4634 static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id) in skb_ext_exist() argument
4636 return skb->active_extensions & (1 << id); in skb_ext_exist()
4639 static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) in skb_ext_del() argument
4641 if (skb_ext_exist(skb, id)) in skb_ext_del()
4642 __skb_ext_del(skb, id); in skb_ext_del()
4645 static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id) in skb_ext_find() argument
4647 if (skb_ext_exist(skb, id)) { in skb_ext_find()
4648 struct skb_ext *ext = skb->extensions; in skb_ext_find()
4656 static inline void skb_ext_reset(struct sk_buff *skb) in skb_ext_reset() argument
4658 if (unlikely(skb->active_extensions)) { in skb_ext_reset()
4659 __skb_ext_put(skb->extensions); in skb_ext_reset()
4660 skb->active_extensions = 0; in skb_ext_reset()
4664 static inline bool skb_has_extensions(struct sk_buff *skb) in skb_has_extensions() argument
4666 return unlikely(skb->active_extensions); in skb_has_extensions()
4669 static inline void skb_ext_put(struct sk_buff *skb) {} in skb_ext_put() argument
4670 static inline void skb_ext_reset(struct sk_buff *skb) {} in skb_ext_reset() argument
4671 static inline void skb_ext_del(struct sk_buff *skb, int unused) {} in skb_ext_del() argument
4674 static inline bool skb_has_extensions(struct sk_buff *skb) { return false; } in skb_has_extensions() argument
4677 static inline void nf_reset_ct(struct sk_buff *skb) in nf_reset_ct() argument
4680 nf_conntrack_put(skb_nfct(skb)); in nf_reset_ct()
4681 skb->_nfct = 0; in nf_reset_ct()
4685 static inline void nf_reset_trace(struct sk_buff *skb) in nf_reset_trace() argument
4688 skb->nf_trace = 0; in nf_reset_trace()
4692 static inline void ipvs_reset(struct sk_buff *skb) in ipvs_reset() argument
4695 skb->ipvs_property = 0; in ipvs_reset()
4728 static inline void skb_init_secmark(struct sk_buff *skb) in skb_init_secmark() argument
4730 skb->secmark = 0; in skb_init_secmark()
4736 static inline void skb_init_secmark(struct sk_buff *skb) in skb_init_secmark() argument
4740 static inline int secpath_exists(const struct sk_buff *skb) in secpath_exists() argument
4743 return skb_ext_exist(skb, SKB_EXT_SEC_PATH); in secpath_exists()
4749 static inline bool skb_irq_freeable(const struct sk_buff *skb) in skb_irq_freeable() argument
4751 return !skb->destructor && in skb_irq_freeable()
4752 !secpath_exists(skb) && in skb_irq_freeable()
4753 !skb_nfct(skb) && in skb_irq_freeable()
4754 !skb->_skb_refdst && in skb_irq_freeable()
4755 !skb_has_frag_list(skb); in skb_irq_freeable()
4758 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping) in skb_set_queue_mapping() argument
4760 skb->queue_mapping = queue_mapping; in skb_set_queue_mapping()
4763 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb) in skb_get_queue_mapping() argument
4765 return skb->queue_mapping; in skb_get_queue_mapping()
4773 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue) in skb_record_rx_queue() argument
4775 skb->queue_mapping = rx_queue + 1; in skb_record_rx_queue()
4778 static inline u16 skb_get_rx_queue(const struct sk_buff *skb) in skb_get_rx_queue() argument
4780 return skb->queue_mapping - 1; in skb_get_rx_queue()
4783 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb) in skb_rx_queue_recorded() argument
4785 return skb->queue_mapping != 0; in skb_rx_queue_recorded()
4788 static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val) in skb_set_dst_pending_confirm() argument
4790 skb->dst_pending_confirm = val; in skb_set_dst_pending_confirm()
4793 static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb) in skb_get_dst_pending_confirm() argument
4795 return skb->dst_pending_confirm != 0; in skb_get_dst_pending_confirm()
4798 static inline struct sec_path *skb_sec_path(const struct sk_buff *skb) in skb_sec_path() argument
4801 return skb_ext_find(skb, SKB_EXT_SEC_PATH); in skb_sec_path()
4823 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET)) argument
4831 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra) in gso_pskb_expand_head() argument
4836 headroom = skb_headroom(skb); in gso_pskb_expand_head()
4837 ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC); in gso_pskb_expand_head()
4841 new_headroom = skb_headroom(skb); in gso_pskb_expand_head()
4842 SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom); in gso_pskb_expand_head()
4846 static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res) in gso_reset_checksum() argument
4849 if (skb->remcsum_offload) in gso_reset_checksum()
4852 SKB_GSO_CB(skb)->csum = res; in gso_reset_checksum()
4853 SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head; in gso_reset_checksum()
4864 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res) in gso_make_checksum() argument
4866 unsigned char *csum_start = skb_transport_header(skb); in gso_make_checksum()
4867 int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start; in gso_make_checksum()
4868 __wsum partial = SKB_GSO_CB(skb)->csum; in gso_make_checksum()
4870 SKB_GSO_CB(skb)->csum = res; in gso_make_checksum()
4871 SKB_GSO_CB(skb)->csum_start = csum_start - skb->head; in gso_make_checksum()
4876 static inline bool skb_is_gso(const struct sk_buff *skb) in skb_is_gso() argument
4878 return skb_shinfo(skb)->gso_size; in skb_is_gso()
4882 static inline bool skb_is_gso_v6(const struct sk_buff *skb) in skb_is_gso_v6() argument
4884 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; in skb_is_gso_v6()
4888 static inline bool skb_is_gso_sctp(const struct sk_buff *skb) in skb_is_gso_sctp() argument
4890 return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP; in skb_is_gso_sctp()
4894 static inline bool skb_is_gso_tcp(const struct sk_buff *skb) in skb_is_gso_tcp() argument
4896 return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6); in skb_is_gso_tcp()
4899 static inline void skb_gso_reset(struct sk_buff *skb) in skb_gso_reset() argument
4901 skb_shinfo(skb)->gso_size = 0; in skb_gso_reset()
4902 skb_shinfo(skb)->gso_segs = 0; in skb_gso_reset()
4903 skb_shinfo(skb)->gso_type = 0; in skb_gso_reset()
4922 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
4924 static inline bool skb_warn_if_lro(const struct sk_buff *skb) in skb_warn_if_lro() argument
4928 const struct skb_shared_info *shinfo = skb_shinfo(skb); in skb_warn_if_lro()
4930 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 && in skb_warn_if_lro()
4932 __skb_warn_lro_forwarding(skb); in skb_warn_if_lro()
4938 static inline void skb_forward_csum(struct sk_buff *skb) in skb_forward_csum() argument
4941 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_forward_csum()
4942 skb->ip_summed = CHECKSUM_NONE; in skb_forward_csum()
4953 static inline void skb_checksum_none_assert(const struct sk_buff *skb) in skb_checksum_none_assert() argument
4955 DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE); in skb_checksum_none_assert()
4958 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
4960 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
4961 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4963 __sum16(*skb_chkf)(struct sk_buff *skb));
4974 static inline bool skb_head_is_locked(const struct sk_buff *skb) in skb_head_is_locked() argument
4976 return !skb->head_frag || skb_cloned(skb); in skb_head_is_locked()
4988 static inline __wsum lco_csum(struct sk_buff *skb) in lco_csum() argument
4990 unsigned char *csum_start = skb_checksum_start(skb); in lco_csum()
4991 unsigned char *l4_hdr = skb_transport_header(skb); in lco_csum()
4996 skb->csum_offset)); in lco_csum()
5004 static inline bool skb_is_redirected(const struct sk_buff *skb) in skb_is_redirected() argument
5006 return skb->redirected; in skb_is_redirected()
5009 static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress) in skb_set_redirected() argument
5011 skb->redirected = 1; in skb_set_redirected()
5013 skb->from_ingress = from_ingress; in skb_set_redirected()
5014 if (skb->from_ingress) in skb_set_redirected()
5015 skb_clear_tstamp(skb); in skb_set_redirected()
5019 static inline void skb_reset_redirect(struct sk_buff *skb) in skb_reset_redirect() argument
5021 skb->redirected = 0; in skb_reset_redirect()
5024 static inline bool skb_csum_is_sctp(struct sk_buff *skb) in skb_csum_is_sctp() argument
5026 return skb->csum_not_inet; in skb_csum_is_sctp()
5029 static inline void skb_set_kcov_handle(struct sk_buff *skb, in skb_set_kcov_handle() argument
5033 skb->kcov_handle = kcov_handle; in skb_set_kcov_handle()
5037 static inline u64 skb_get_kcov_handle(struct sk_buff *skb) in skb_get_kcov_handle() argument
5040 return skb->kcov_handle; in skb_get_kcov_handle()
5047 static inline void skb_mark_for_recycle(struct sk_buff *skb) in skb_mark_for_recycle() argument
5049 skb->pp_recycle = 1; in skb_mark_for_recycle()
5053 static inline bool skb_pp_recycle(struct sk_buff *skb, void *data) in skb_pp_recycle() argument
5055 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) in skb_pp_recycle()