101 struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb,  in skb_eth_gso_segment()  argument
110 			segs = ptype->callbacks.gso_segment(skb, features);  in skb_eth_gso_segment()
125 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,  in skb_mac_gso_segment()  argument
130 	int vlan_depth = skb->mac_len;  in skb_mac_gso_segment()
131 	__be16 type = skb_network_protocol(skb, &vlan_depth);  in skb_mac_gso_segment()
136 	__skb_pull(skb, vlan_depth);  in skb_mac_gso_segment()
141 			segs = ptype->callbacks.gso_segment(skb, features);  in skb_mac_gso_segment()
147 	__skb_push(skb, skb->data - skb_mac_header(skb));  in skb_mac_gso_segment()
153 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)  in skb_gro_receive()  argument
155 	struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);  in skb_gro_receive()
156 	unsigned int offset = skb_gro_offset(skb);  in skb_gro_receive()
157 	unsigned int headlen = skb_headlen(skb);  in skb_gro_receive()
158 	unsigned int len = skb_gro_len(skb);  in skb_gro_receive()
168 	if (unlikely(p->len + len >= gro_max_size || NAPI_GRO_CB(skb)->flush))  in skb_gro_receive()
179 	segs = NAPI_GRO_CB(skb)->count;  in skb_gro_receive()
206 		new_truesize = SKB_TRUESIZE(skb_end_offset(skb));  in skb_gro_receive()
207 		delta_truesize = skb->truesize - new_truesize;  in skb_gro_receive()
209 		skb->truesize = new_truesize;  in skb_gro_receive()
210 		skb->len -= skb->data_len;  in skb_gro_receive()
211 		skb->data_len = 0;  in skb_gro_receive()
213 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;  in skb_gro_receive()
215 	} else if (skb->head_frag) {  in skb_gro_receive()
218 		struct page *page = virt_to_head_page(skb->head);  in skb_gro_receive()
225 		first_offset = skb->data -  in skb_gro_receive()
239 		delta_truesize = skb->truesize - new_truesize;  in skb_gro_receive()
240 		skb->truesize = new_truesize;  in skb_gro_receive()
241 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;  in skb_gro_receive()
247 	skb->destructor = NULL;  in skb_gro_receive()
248 	delta_truesize = skb->truesize;  in skb_gro_receive()
254 		skb->data_len -= eat;  in skb_gro_receive()
255 		skb->len -= eat;  in skb_gro_receive()
259 	__skb_pull(skb, offset);  in skb_gro_receive()
262 		skb_shinfo(p)->frag_list = skb;  in skb_gro_receive()
264 		NAPI_GRO_CB(p)->last->next = skb;  in skb_gro_receive()
265 	NAPI_GRO_CB(p)->last = skb;  in skb_gro_receive()
266 	__skb_header_release(skb);  in skb_gro_receive()
279 	NAPI_GRO_CB(skb)->same_flow = 1;  in skb_gro_receive()
284 static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)  in napi_gro_complete()  argument
287 	__be16 type = skb->protocol;  in napi_gro_complete()
291 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));  in napi_gro_complete()
293 	if (NAPI_GRO_CB(skb)->count == 1) {  in napi_gro_complete()
294 		skb_shinfo(skb)->gso_size = 0;  in napi_gro_complete()
305 					 skb, 0);  in napi_gro_complete()
312 		kfree_skb(skb);  in napi_gro_complete()
317 	gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);  in napi_gro_complete()
324 	struct sk_buff *skb, *p;  in __napi_gro_flush_chain()  local
326 	list_for_each_entry_safe_reverse(skb, p, head, list) {  in __napi_gro_flush_chain()
327 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)  in __napi_gro_flush_chain()
329 		skb_list_del_init(skb);  in __napi_gro_flush_chain()
330 		napi_gro_complete(napi, skb);  in __napi_gro_flush_chain()
356 			     const struct sk_buff *skb)  in gro_list_prepare()  argument
358 	unsigned int maclen = skb->dev->hard_header_len;  in gro_list_prepare()
359 	u32 hash = skb_get_hash_raw(skb);  in gro_list_prepare()
372 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;  in gro_list_prepare()
373 		diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);  in gro_list_prepare()
375 			diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);  in gro_list_prepare()
376 		diffs |= skb_metadata_differs(p, skb);  in gro_list_prepare()
379 						      skb_mac_header(skb));  in gro_list_prepare()
382 				       skb_mac_header(skb),  in gro_list_prepare()
390 		if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {  in gro_list_prepare()
396 			diffs |= p->sk != skb->sk;  in gro_list_prepare()
397 			diffs |= skb_metadata_dst_cmp(p, skb);  in gro_list_prepare()
398 			diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);  in gro_list_prepare()
401 			skb_ext = skb_ext_find(skb, TC_SKB_EXT);  in gro_list_prepare()
414 static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)  in skb_gro_reset_offset()  argument
416 	const struct skb_shared_info *pinfo = skb_shinfo(skb);  in skb_gro_reset_offset()
419 	NAPI_GRO_CB(skb)->data_offset = 0;  in skb_gro_reset_offset()
420 	NAPI_GRO_CB(skb)->frag0 = NULL;  in skb_gro_reset_offset()
421 	NAPI_GRO_CB(skb)->frag0_len = 0;  in skb_gro_reset_offset()
423 	if (!skb_headlen(skb) && pinfo->nr_frags &&  in skb_gro_reset_offset()
426 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);  in skb_gro_reset_offset()
427 		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,  in skb_gro_reset_offset()
429 						    skb->end - skb->tail);  in skb_gro_reset_offset()
433 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)  in gro_pull_from_frag0()  argument
435 	struct skb_shared_info *pinfo = skb_shinfo(skb);  in gro_pull_from_frag0()
437 	BUG_ON(skb->end - skb->tail < grow);  in gro_pull_from_frag0()
439 	memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);  in gro_pull_from_frag0()
441 	skb->data_len -= grow;  in gro_pull_from_frag0()
442 	skb->tail += grow;  in gro_pull_from_frag0()
448 		skb_frag_unref(skb, 0);  in gro_pull_from_frag0()
473 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)  in dev_gro_receive()  argument
475 	u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);  in dev_gro_receive()
479 	__be16 type = skb->protocol;  in dev_gro_receive()
485 	if (netif_elide_gro(skb->dev))  in dev_gro_receive()
488 	gro_list_prepare(&gro_list->list, skb);  in dev_gro_receive()
499 	skb_set_network_header(skb, skb_gro_offset(skb));  in dev_gro_receive()
500 	skb_reset_mac_len(skb);  in dev_gro_receive()
504 	*(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0;  in dev_gro_receive()
505 	NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb);  in dev_gro_receive()
506 	NAPI_GRO_CB(skb)->is_atomic = 1;  in dev_gro_receive()
507 	NAPI_GRO_CB(skb)->count = 1;  in dev_gro_receive()
508 	if (unlikely(skb_is_gso(skb))) {  in dev_gro_receive()
509 		NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs;  in dev_gro_receive()
511 		if (!skb_is_gso_tcp(skb) ||  in dev_gro_receive()
512 		    (skb_shinfo(skb)->gso_type & SKB_GSO_DODGY))  in dev_gro_receive()
513 			NAPI_GRO_CB(skb)->flush = 1;  in dev_gro_receive()
517 	switch (skb->ip_summed) {  in dev_gro_receive()
519 		NAPI_GRO_CB(skb)->csum = skb->csum;  in dev_gro_receive()
520 		NAPI_GRO_CB(skb)->csum_valid = 1;  in dev_gro_receive()
523 		NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;  in dev_gro_receive()
529 				&gro_list->list, skb);  in dev_gro_receive()
538 	same_flow = NAPI_GRO_CB(skb)->same_flow;  in dev_gro_receive()
539 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;  in dev_gro_receive()
550 	if (NAPI_GRO_CB(skb)->flush)  in dev_gro_receive()
558 	NAPI_GRO_CB(skb)->age = jiffies;  in dev_gro_receive()
559 	NAPI_GRO_CB(skb)->last = skb;  in dev_gro_receive()
560 	if (!skb_is_gso(skb))  in dev_gro_receive()
561 		skb_shinfo(skb)->gso_size = skb_gro_len(skb);  in dev_gro_receive()
562 	list_add(&skb->list, &gro_list->list);  in dev_gro_receive()
566 	grow = skb_gro_offset(skb) - skb_headlen(skb);  in dev_gro_receive()
568 		gro_pull_from_frag0(skb, grow);  in dev_gro_receive()
613 				    struct sk_buff *skb,  in napi_skb_finish()  argument
618 		gro_normal_one(napi, skb, 1);  in napi_skb_finish()
622 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)  in napi_skb_finish()
623 			napi_skb_free_stolen_head(skb);  in napi_skb_finish()
624 		else if (skb->fclone != SKB_FCLONE_UNAVAILABLE)  in napi_skb_finish()
625 			__kfree_skb(skb);  in napi_skb_finish()
627 			__kfree_skb_defer(skb);  in napi_skb_finish()
639 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)  in napi_gro_receive()  argument
643 	skb_mark_napi_id(skb, napi);  in napi_gro_receive()
644 	trace_napi_gro_receive_entry(skb);  in napi_gro_receive()
646 	skb_gro_reset_offset(skb, 0);  in napi_gro_receive()
648 	ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));  in napi_gro_receive()
655 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)  in napi_reuse_skb()  argument
657 	if (unlikely(skb->pfmemalloc)) {  in napi_reuse_skb()
658 		consume_skb(skb);  in napi_reuse_skb()
661 	__skb_pull(skb, skb_headlen(skb));  in napi_reuse_skb()
663 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));  in napi_reuse_skb()
664 	__vlan_hwaccel_clear_tag(skb);  in napi_reuse_skb()
665 	skb->dev = napi->dev;  in napi_reuse_skb()
666 	skb->skb_iif = 0;  in napi_reuse_skb()
669 	skb->pkt_type = PACKET_HOST;  in napi_reuse_skb()
671 	skb->encapsulation = 0;  in napi_reuse_skb()
672 	skb_shinfo(skb)->gso_type = 0;  in napi_reuse_skb()
673 	skb_shinfo(skb)->gso_size = 0;  in napi_reuse_skb()
674 	if (unlikely(skb->slow_gro)) {  in napi_reuse_skb()
675 		skb_orphan(skb);  in napi_reuse_skb()
676 		skb_ext_reset(skb);  in napi_reuse_skb()
677 		nf_reset_ct(skb);  in napi_reuse_skb()
678 		skb->slow_gro = 0;  in napi_reuse_skb()
681 	napi->skb = skb;  in napi_reuse_skb()
686 	struct sk_buff *skb = napi->skb;  in napi_get_frags()  local
688 	if (!skb) {  in napi_get_frags()
689 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);  in napi_get_frags()
690 		if (skb) {  in napi_get_frags()
691 			napi->skb = skb;  in napi_get_frags()
692 			skb_mark_napi_id(skb, napi);  in napi_get_frags()
695 	return skb;  in napi_get_frags()
700 				      struct sk_buff *skb,  in napi_frags_finish()  argument
706 		__skb_push(skb, ETH_HLEN);  in napi_frags_finish()
707 		skb->protocol = eth_type_trans(skb, skb->dev);  in napi_frags_finish()
709 			gro_normal_one(napi, skb, 1);  in napi_frags_finish()
713 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)  in napi_frags_finish()
714 			napi_skb_free_stolen_head(skb);  in napi_frags_finish()
716 			napi_reuse_skb(napi, skb);  in napi_frags_finish()
733 	struct sk_buff *skb = napi->skb;  in napi_frags_skb()  local
737 	napi->skb = NULL;  in napi_frags_skb()
739 	skb_reset_mac_header(skb);  in napi_frags_skb()
740 	skb_gro_reset_offset(skb, hlen);  in napi_frags_skb()
742 	if (unlikely(skb_gro_header_hard(skb, hlen))) {  in napi_frags_skb()
743 		eth = skb_gro_header_slow(skb, hlen, 0);  in napi_frags_skb()
747 			napi_reuse_skb(napi, skb);  in napi_frags_skb()
751 		eth = (const struct ethhdr *)skb->data;  in napi_frags_skb()
752 		gro_pull_from_frag0(skb, hlen);  in napi_frags_skb()
753 		NAPI_GRO_CB(skb)->frag0 += hlen;  in napi_frags_skb()
754 		NAPI_GRO_CB(skb)->frag0_len -= hlen;  in napi_frags_skb()
756 	__skb_pull(skb, hlen);  in napi_frags_skb()
763 	skb->protocol = eth->h_proto;  in napi_frags_skb()
765 	return skb;  in napi_frags_skb()
771 	struct sk_buff *skb = napi_frags_skb(napi);  in napi_gro_frags()  local
773 	trace_napi_gro_frags_entry(skb);  in napi_gro_frags()
775 	ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));  in napi_gro_frags()
785 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)  in __skb_gro_checksum_complete()  argument
790 	wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);  in __skb_gro_checksum_complete()
793 	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));  in __skb_gro_checksum_complete()
796 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&  in __skb_gro_checksum_complete()
797 		    !skb->csum_complete_sw)  in __skb_gro_checksum_complete()
798 			netdev_rx_csum_fault(skb->dev, skb);  in __skb_gro_checksum_complete()
801 	NAPI_GRO_CB(skb)->csum = wsum;  in __skb_gro_checksum_complete()
802 	NAPI_GRO_CB(skb)->csum_valid = 1;  in __skb_gro_checksum_complete()