1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <net/gro.h>
3 #include <net/dst_metadata.h>
4 #include <net/busy_poll.h>
5 #include <trace/events/net.h>
6 
7 #define MAX_GRO_SKBS 8
8 
9 /* This should be increased if a protocol with a bigger head is added. */
10 #define GRO_MAX_HEAD (MAX_HEADER + 128)
11 
12 static DEFINE_SPINLOCK(offload_lock);
13 static struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base);
14 /* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
15 int gro_normal_batch __read_mostly = 8;
16 
17 /**
18  *	dev_add_offload - register offload handlers
19  *	@po: protocol offload declaration
20  *
21  *	Add protocol offload handlers to the networking stack. The passed
22  *	&proto_offload is linked into kernel lists and may not be freed until
23  *	it has been removed from the kernel lists.
24  *
25  *	This call does not sleep therefore it can not
26  *	guarantee all CPU's that are in middle of receiving packets
27  *	will see the new offload handlers (until the next received packet).
28  */
dev_add_offload(struct packet_offload * po)29 void dev_add_offload(struct packet_offload *po)
30 {
31 	struct packet_offload *elem;
32 
33 	spin_lock(&offload_lock);
34 	list_for_each_entry(elem, &offload_base, list) {
35 		if (po->priority < elem->priority)
36 			break;
37 	}
38 	list_add_rcu(&po->list, elem->list.prev);
39 	spin_unlock(&offload_lock);
40 }
41 EXPORT_SYMBOL(dev_add_offload);
42 
43 /**
44  *	__dev_remove_offload	 - remove offload handler
45  *	@po: packet offload declaration
46  *
47  *	Remove a protocol offload handler that was previously added to the
48  *	kernel offload handlers by dev_add_offload(). The passed &offload_type
49  *	is removed from the kernel lists and can be freed or reused once this
50  *	function returns.
51  *
52  *      The packet type might still be in use by receivers
53  *	and must not be freed until after all the CPU's have gone
54  *	through a quiescent state.
55  */
__dev_remove_offload(struct packet_offload * po)56 static void __dev_remove_offload(struct packet_offload *po)
57 {
58 	struct list_head *head = &offload_base;
59 	struct packet_offload *po1;
60 
61 	spin_lock(&offload_lock);
62 
63 	list_for_each_entry(po1, head, list) {
64 		if (po == po1) {
65 			list_del_rcu(&po->list);
66 			goto out;
67 		}
68 	}
69 
70 	pr_warn("dev_remove_offload: %p not found\n", po);
71 out:
72 	spin_unlock(&offload_lock);
73 }
74 
75 /**
76  *	dev_remove_offload	 - remove packet offload handler
77  *	@po: packet offload declaration
78  *
79  *	Remove a packet offload handler that was previously added to the kernel
80  *	offload handlers by dev_add_offload(). The passed &offload_type is
81  *	removed from the kernel lists and can be freed or reused once this
82  *	function returns.
83  *
84  *	This call sleeps to guarantee that no CPU is looking at the packet
85  *	type after return.
86  */
dev_remove_offload(struct packet_offload * po)87 void dev_remove_offload(struct packet_offload *po)
88 {
89 	__dev_remove_offload(po);
90 
91 	synchronize_net();
92 }
93 EXPORT_SYMBOL(dev_remove_offload);
94 
95 /**
96  *	skb_eth_gso_segment - segmentation handler for ethernet protocols.
97  *	@skb: buffer to segment
98  *	@features: features for the output path (see dev->features)
99  *	@type: Ethernet Protocol ID
100  */
skb_eth_gso_segment(struct sk_buff * skb,netdev_features_t features,__be16 type)101 struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb,
102 				    netdev_features_t features, __be16 type)
103 {
104 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
105 	struct packet_offload *ptype;
106 
107 	rcu_read_lock();
108 	list_for_each_entry_rcu(ptype, &offload_base, list) {
109 		if (ptype->type == type && ptype->callbacks.gso_segment) {
110 			segs = ptype->callbacks.gso_segment(skb, features);
111 			break;
112 		}
113 	}
114 	rcu_read_unlock();
115 
116 	return segs;
117 }
118 EXPORT_SYMBOL(skb_eth_gso_segment);
119 
120 /**
121  *	skb_mac_gso_segment - mac layer segmentation handler.
122  *	@skb: buffer to segment
123  *	@features: features for the output path (see dev->features)
124  */
skb_mac_gso_segment(struct sk_buff * skb,netdev_features_t features)125 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
126 				    netdev_features_t features)
127 {
128 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
129 	struct packet_offload *ptype;
130 	int vlan_depth = skb->mac_len;
131 	__be16 type = skb_network_protocol(skb, &vlan_depth);
132 
133 	if (unlikely(!type))
134 		return ERR_PTR(-EINVAL);
135 
136 	__skb_pull(skb, vlan_depth);
137 
138 	rcu_read_lock();
139 	list_for_each_entry_rcu(ptype, &offload_base, list) {
140 		if (ptype->type == type && ptype->callbacks.gso_segment) {
141 			segs = ptype->callbacks.gso_segment(skb, features);
142 			break;
143 		}
144 	}
145 	rcu_read_unlock();
146 
147 	__skb_push(skb, skb->data - skb_mac_header(skb));
148 
149 	return segs;
150 }
151 EXPORT_SYMBOL(skb_mac_gso_segment);
152 
skb_gro_receive(struct sk_buff * p,struct sk_buff * skb)153 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
154 {
155 	struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
156 	unsigned int offset = skb_gro_offset(skb);
157 	unsigned int headlen = skb_headlen(skb);
158 	unsigned int len = skb_gro_len(skb);
159 	unsigned int delta_truesize;
160 	unsigned int gro_max_size;
161 	unsigned int new_truesize;
162 	struct sk_buff *lp;
163 
164 	/* pairs with WRITE_ONCE() in netif_set_gro_max_size() */
165 	gro_max_size = READ_ONCE(p->dev->gro_max_size);
166 
167 	if (unlikely(p->len + len >= gro_max_size || NAPI_GRO_CB(skb)->flush))
168 		return -E2BIG;
169 
170 	if (unlikely(p->len + len >= GRO_LEGACY_MAX_SIZE)) {
171 		if (p->protocol != htons(ETH_P_IPV6) ||
172 		    skb_headroom(p) < sizeof(struct hop_jumbo_hdr) ||
173 		    ipv6_hdr(p)->nexthdr != IPPROTO_TCP ||
174 		    p->encapsulation)
175 			return -E2BIG;
176 	}
177 
178 	lp = NAPI_GRO_CB(p)->last;
179 	pinfo = skb_shinfo(lp);
180 
181 	if (headlen <= offset) {
182 		skb_frag_t *frag;
183 		skb_frag_t *frag2;
184 		int i = skbinfo->nr_frags;
185 		int nr_frags = pinfo->nr_frags + i;
186 
187 		if (nr_frags > MAX_SKB_FRAGS)
188 			goto merge;
189 
190 		offset -= headlen;
191 		pinfo->nr_frags = nr_frags;
192 		skbinfo->nr_frags = 0;
193 
194 		frag = pinfo->frags + nr_frags;
195 		frag2 = skbinfo->frags + i;
196 		do {
197 			*--frag = *--frag2;
198 		} while (--i);
199 
200 		skb_frag_off_add(frag, offset);
201 		skb_frag_size_sub(frag, offset);
202 
203 		/* all fragments truesize : remove (head size + sk_buff) */
204 		new_truesize = SKB_TRUESIZE(skb_end_offset(skb));
205 		delta_truesize = skb->truesize - new_truesize;
206 
207 		skb->truesize = new_truesize;
208 		skb->len -= skb->data_len;
209 		skb->data_len = 0;
210 
211 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
212 		goto done;
213 	} else if (skb->head_frag) {
214 		int nr_frags = pinfo->nr_frags;
215 		skb_frag_t *frag = pinfo->frags + nr_frags;
216 		struct page *page = virt_to_head_page(skb->head);
217 		unsigned int first_size = headlen - offset;
218 		unsigned int first_offset;
219 
220 		if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
221 			goto merge;
222 
223 		first_offset = skb->data -
224 			       (unsigned char *)page_address(page) +
225 			       offset;
226 
227 		pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
228 
229 		__skb_frag_set_page(frag, page);
230 		skb_frag_off_set(frag, first_offset);
231 		skb_frag_size_set(frag, first_size);
232 
233 		memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
234 		/* We dont need to clear skbinfo->nr_frags here */
235 
236 		new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff));
237 		delta_truesize = skb->truesize - new_truesize;
238 		skb->truesize = new_truesize;
239 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
240 		goto done;
241 	}
242 
243 merge:
244 	/* sk owenrship - if any - completely transferred to the aggregated packet */
245 	skb->destructor = NULL;
246 	delta_truesize = skb->truesize;
247 	if (offset > headlen) {
248 		unsigned int eat = offset - headlen;
249 
250 		skb_frag_off_add(&skbinfo->frags[0], eat);
251 		skb_frag_size_sub(&skbinfo->frags[0], eat);
252 		skb->data_len -= eat;
253 		skb->len -= eat;
254 		offset = headlen;
255 	}
256 
257 	__skb_pull(skb, offset);
258 
259 	if (NAPI_GRO_CB(p)->last == p)
260 		skb_shinfo(p)->frag_list = skb;
261 	else
262 		NAPI_GRO_CB(p)->last->next = skb;
263 	NAPI_GRO_CB(p)->last = skb;
264 	__skb_header_release(skb);
265 	lp = p;
266 
267 done:
268 	NAPI_GRO_CB(p)->count++;
269 	p->data_len += len;
270 	p->truesize += delta_truesize;
271 	p->len += len;
272 	if (lp != p) {
273 		lp->data_len += len;
274 		lp->truesize += delta_truesize;
275 		lp->len += len;
276 	}
277 	NAPI_GRO_CB(skb)->same_flow = 1;
278 	return 0;
279 }
280 
281 
napi_gro_complete(struct napi_struct * napi,struct sk_buff * skb)282 static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
283 {
284 	struct packet_offload *ptype;
285 	__be16 type = skb->protocol;
286 	struct list_head *head = &offload_base;
287 	int err = -ENOENT;
288 
289 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
290 
291 	if (NAPI_GRO_CB(skb)->count == 1) {
292 		skb_shinfo(skb)->gso_size = 0;
293 		goto out;
294 	}
295 
296 	rcu_read_lock();
297 	list_for_each_entry_rcu(ptype, head, list) {
298 		if (ptype->type != type || !ptype->callbacks.gro_complete)
299 			continue;
300 
301 		err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
302 					 ipv6_gro_complete, inet_gro_complete,
303 					 skb, 0);
304 		break;
305 	}
306 	rcu_read_unlock();
307 
308 	if (err) {
309 		WARN_ON(&ptype->list == head);
310 		kfree_skb(skb);
311 		return;
312 	}
313 
314 out:
315 	gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
316 }
317 
__napi_gro_flush_chain(struct napi_struct * napi,u32 index,bool flush_old)318 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
319 				   bool flush_old)
320 {
321 	struct list_head *head = &napi->gro_hash[index].list;
322 	struct sk_buff *skb, *p;
323 
324 	list_for_each_entry_safe_reverse(skb, p, head, list) {
325 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
326 			return;
327 		skb_list_del_init(skb);
328 		napi_gro_complete(napi, skb);
329 		napi->gro_hash[index].count--;
330 	}
331 
332 	if (!napi->gro_hash[index].count)
333 		__clear_bit(index, &napi->gro_bitmask);
334 }
335 
336 /* napi->gro_hash[].list contains packets ordered by age.
337  * youngest packets at the head of it.
338  * Complete skbs in reverse order to reduce latencies.
339  */
napi_gro_flush(struct napi_struct * napi,bool flush_old)340 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
341 {
342 	unsigned long bitmask = napi->gro_bitmask;
343 	unsigned int i, base = ~0U;
344 
345 	while ((i = ffs(bitmask)) != 0) {
346 		bitmask >>= i;
347 		base += i;
348 		__napi_gro_flush_chain(napi, base, flush_old);
349 	}
350 }
351 EXPORT_SYMBOL(napi_gro_flush);
352 
gro_list_prepare(const struct list_head * head,const struct sk_buff * skb)353 static void gro_list_prepare(const struct list_head *head,
354 			     const struct sk_buff *skb)
355 {
356 	unsigned int maclen = skb->dev->hard_header_len;
357 	u32 hash = skb_get_hash_raw(skb);
358 	struct sk_buff *p;
359 
360 	list_for_each_entry(p, head, list) {
361 		unsigned long diffs;
362 
363 		NAPI_GRO_CB(p)->flush = 0;
364 
365 		if (hash != skb_get_hash_raw(p)) {
366 			NAPI_GRO_CB(p)->same_flow = 0;
367 			continue;
368 		}
369 
370 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
371 		diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
372 		if (skb_vlan_tag_present(p))
373 			diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
374 		diffs |= skb_metadata_differs(p, skb);
375 		if (maclen == ETH_HLEN)
376 			diffs |= compare_ether_header(skb_mac_header(p),
377 						      skb_mac_header(skb));
378 		else if (!diffs)
379 			diffs = memcmp(skb_mac_header(p),
380 				       skb_mac_header(skb),
381 				       maclen);
382 
383 		/* in most common scenarions 'slow_gro' is 0
384 		 * otherwise we are already on some slower paths
385 		 * either skip all the infrequent tests altogether or
386 		 * avoid trying too hard to skip each of them individually
387 		 */
388 		if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
389 #if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
390 			struct tc_skb_ext *skb_ext;
391 			struct tc_skb_ext *p_ext;
392 #endif
393 
394 			diffs |= p->sk != skb->sk;
395 			diffs |= skb_metadata_dst_cmp(p, skb);
396 			diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
397 
398 #if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
399 			skb_ext = skb_ext_find(skb, TC_SKB_EXT);
400 			p_ext = skb_ext_find(p, TC_SKB_EXT);
401 
402 			diffs |= (!!p_ext) ^ (!!skb_ext);
403 			if (!diffs && unlikely(skb_ext))
404 				diffs |= p_ext->chain ^ skb_ext->chain;
405 #endif
406 		}
407 
408 		NAPI_GRO_CB(p)->same_flow = !diffs;
409 	}
410 }
411 
skb_gro_reset_offset(struct sk_buff * skb,u32 nhoff)412 static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
413 {
414 	const struct skb_shared_info *pinfo = skb_shinfo(skb);
415 	const skb_frag_t *frag0 = &pinfo->frags[0];
416 
417 	NAPI_GRO_CB(skb)->data_offset = 0;
418 	NAPI_GRO_CB(skb)->frag0 = NULL;
419 	NAPI_GRO_CB(skb)->frag0_len = 0;
420 
421 	if (!skb_headlen(skb) && pinfo->nr_frags &&
422 	    !PageHighMem(skb_frag_page(frag0)) &&
423 	    (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
424 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
425 		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
426 						    skb_frag_size(frag0),
427 						    skb->end - skb->tail);
428 	}
429 }
430 
gro_pull_from_frag0(struct sk_buff * skb,int grow)431 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
432 {
433 	struct skb_shared_info *pinfo = skb_shinfo(skb);
434 
435 	BUG_ON(skb->end - skb->tail < grow);
436 
437 	memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
438 
439 	skb->data_len -= grow;
440 	skb->tail += grow;
441 
442 	skb_frag_off_add(&pinfo->frags[0], grow);
443 	skb_frag_size_sub(&pinfo->frags[0], grow);
444 
445 	if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
446 		skb_frag_unref(skb, 0);
447 		memmove(pinfo->frags, pinfo->frags + 1,
448 			--pinfo->nr_frags * sizeof(pinfo->frags[0]));
449 	}
450 }
451 
gro_flush_oldest(struct napi_struct * napi,struct list_head * head)452 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
453 {
454 	struct sk_buff *oldest;
455 
456 	oldest = list_last_entry(head, struct sk_buff, list);
457 
458 	/* We are called with head length >= MAX_GRO_SKBS, so this is
459 	 * impossible.
460 	 */
461 	if (WARN_ON_ONCE(!oldest))
462 		return;
463 
464 	/* Do not adjust napi->gro_hash[].count, caller is adding a new
465 	 * SKB to the chain.
466 	 */
467 	skb_list_del_init(oldest);
468 	napi_gro_complete(napi, oldest);
469 }
470 
dev_gro_receive(struct napi_struct * napi,struct sk_buff * skb)471 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
472 {
473 	u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
474 	struct gro_list *gro_list = &napi->gro_hash[bucket];
475 	struct list_head *head = &offload_base;
476 	struct packet_offload *ptype;
477 	__be16 type = skb->protocol;
478 	struct sk_buff *pp = NULL;
479 	enum gro_result ret;
480 	int same_flow;
481 	int grow;
482 
483 	if (netif_elide_gro(skb->dev))
484 		goto normal;
485 
486 	gro_list_prepare(&gro_list->list, skb);
487 
488 	rcu_read_lock();
489 	list_for_each_entry_rcu(ptype, head, list) {
490 		if (ptype->type != type || !ptype->callbacks.gro_receive)
491 			continue;
492 
493 		skb_set_network_header(skb, skb_gro_offset(skb));
494 		skb_reset_mac_len(skb);
495 		BUILD_BUG_ON(sizeof_field(struct napi_gro_cb, zeroed) != sizeof(u32));
496 		BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct napi_gro_cb, zeroed),
497 					 sizeof(u32))); /* Avoid slow unaligned acc */
498 		*(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0;
499 		NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
500 		NAPI_GRO_CB(skb)->is_atomic = 1;
501 
502 		/* Setup for GRO checksum validation */
503 		switch (skb->ip_summed) {
504 		case CHECKSUM_COMPLETE:
505 			NAPI_GRO_CB(skb)->csum = skb->csum;
506 			NAPI_GRO_CB(skb)->csum_valid = 1;
507 			break;
508 		case CHECKSUM_UNNECESSARY:
509 			NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
510 			break;
511 		}
512 
513 		pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
514 					ipv6_gro_receive, inet_gro_receive,
515 					&gro_list->list, skb);
516 		break;
517 	}
518 	rcu_read_unlock();
519 
520 	if (&ptype->list == head)
521 		goto normal;
522 
523 	if (PTR_ERR(pp) == -EINPROGRESS) {
524 		ret = GRO_CONSUMED;
525 		goto ok;
526 	}
527 
528 	same_flow = NAPI_GRO_CB(skb)->same_flow;
529 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
530 
531 	if (pp) {
532 		skb_list_del_init(pp);
533 		napi_gro_complete(napi, pp);
534 		gro_list->count--;
535 	}
536 
537 	if (same_flow)
538 		goto ok;
539 
540 	if (NAPI_GRO_CB(skb)->flush)
541 		goto normal;
542 
543 	if (unlikely(gro_list->count >= MAX_GRO_SKBS))
544 		gro_flush_oldest(napi, &gro_list->list);
545 	else
546 		gro_list->count++;
547 
548 	NAPI_GRO_CB(skb)->count = 1;
549 	NAPI_GRO_CB(skb)->age = jiffies;
550 	NAPI_GRO_CB(skb)->last = skb;
551 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
552 	list_add(&skb->list, &gro_list->list);
553 	ret = GRO_HELD;
554 
555 pull:
556 	grow = skb_gro_offset(skb) - skb_headlen(skb);
557 	if (grow > 0)
558 		gro_pull_from_frag0(skb, grow);
559 ok:
560 	if (gro_list->count) {
561 		if (!test_bit(bucket, &napi->gro_bitmask))
562 			__set_bit(bucket, &napi->gro_bitmask);
563 	} else if (test_bit(bucket, &napi->gro_bitmask)) {
564 		__clear_bit(bucket, &napi->gro_bitmask);
565 	}
566 
567 	return ret;
568 
569 normal:
570 	ret = GRO_NORMAL;
571 	goto pull;
572 }
573 
gro_find_receive_by_type(__be16 type)574 struct packet_offload *gro_find_receive_by_type(__be16 type)
575 {
576 	struct list_head *offload_head = &offload_base;
577 	struct packet_offload *ptype;
578 
579 	list_for_each_entry_rcu(ptype, offload_head, list) {
580 		if (ptype->type != type || !ptype->callbacks.gro_receive)
581 			continue;
582 		return ptype;
583 	}
584 	return NULL;
585 }
586 EXPORT_SYMBOL(gro_find_receive_by_type);
587 
gro_find_complete_by_type(__be16 type)588 struct packet_offload *gro_find_complete_by_type(__be16 type)
589 {
590 	struct list_head *offload_head = &offload_base;
591 	struct packet_offload *ptype;
592 
593 	list_for_each_entry_rcu(ptype, offload_head, list) {
594 		if (ptype->type != type || !ptype->callbacks.gro_complete)
595 			continue;
596 		return ptype;
597 	}
598 	return NULL;
599 }
600 EXPORT_SYMBOL(gro_find_complete_by_type);
601 
napi_skb_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)602 static gro_result_t napi_skb_finish(struct napi_struct *napi,
603 				    struct sk_buff *skb,
604 				    gro_result_t ret)
605 {
606 	switch (ret) {
607 	case GRO_NORMAL:
608 		gro_normal_one(napi, skb, 1);
609 		break;
610 
611 	case GRO_MERGED_FREE:
612 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
613 			napi_skb_free_stolen_head(skb);
614 		else if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
615 			__kfree_skb(skb);
616 		else
617 			__kfree_skb_defer(skb);
618 		break;
619 
620 	case GRO_HELD:
621 	case GRO_MERGED:
622 	case GRO_CONSUMED:
623 		break;
624 	}
625 
626 	return ret;
627 }
628 
napi_gro_receive(struct napi_struct * napi,struct sk_buff * skb)629 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
630 {
631 	gro_result_t ret;
632 
633 	skb_mark_napi_id(skb, napi);
634 	trace_napi_gro_receive_entry(skb);
635 
636 	skb_gro_reset_offset(skb, 0);
637 
638 	ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
639 	trace_napi_gro_receive_exit(ret);
640 
641 	return ret;
642 }
643 EXPORT_SYMBOL(napi_gro_receive);
644 
napi_reuse_skb(struct napi_struct * napi,struct sk_buff * skb)645 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
646 {
647 	if (unlikely(skb->pfmemalloc)) {
648 		consume_skb(skb);
649 		return;
650 	}
651 	__skb_pull(skb, skb_headlen(skb));
652 	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
653 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
654 	__vlan_hwaccel_clear_tag(skb);
655 	skb->dev = napi->dev;
656 	skb->skb_iif = 0;
657 
658 	/* eth_type_trans() assumes pkt_type is PACKET_HOST */
659 	skb->pkt_type = PACKET_HOST;
660 
661 	skb->encapsulation = 0;
662 	skb_shinfo(skb)->gso_type = 0;
663 	if (unlikely(skb->slow_gro)) {
664 		skb_orphan(skb);
665 		skb_ext_reset(skb);
666 		nf_reset_ct(skb);
667 		skb->slow_gro = 0;
668 	}
669 
670 	napi->skb = skb;
671 }
672 
napi_get_frags(struct napi_struct * napi)673 struct sk_buff *napi_get_frags(struct napi_struct *napi)
674 {
675 	struct sk_buff *skb = napi->skb;
676 
677 	if (!skb) {
678 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
679 		if (skb) {
680 			napi->skb = skb;
681 			skb_mark_napi_id(skb, napi);
682 		}
683 	}
684 	return skb;
685 }
686 EXPORT_SYMBOL(napi_get_frags);
687 
napi_frags_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)688 static gro_result_t napi_frags_finish(struct napi_struct *napi,
689 				      struct sk_buff *skb,
690 				      gro_result_t ret)
691 {
692 	switch (ret) {
693 	case GRO_NORMAL:
694 	case GRO_HELD:
695 		__skb_push(skb, ETH_HLEN);
696 		skb->protocol = eth_type_trans(skb, skb->dev);
697 		if (ret == GRO_NORMAL)
698 			gro_normal_one(napi, skb, 1);
699 		break;
700 
701 	case GRO_MERGED_FREE:
702 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
703 			napi_skb_free_stolen_head(skb);
704 		else
705 			napi_reuse_skb(napi, skb);
706 		break;
707 
708 	case GRO_MERGED:
709 	case GRO_CONSUMED:
710 		break;
711 	}
712 
713 	return ret;
714 }
715 
716 /* Upper GRO stack assumes network header starts at gro_offset=0
717  * Drivers could call both napi_gro_frags() and napi_gro_receive()
718  * We copy ethernet header into skb->data to have a common layout.
719  */
napi_frags_skb(struct napi_struct * napi)720 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
721 {
722 	struct sk_buff *skb = napi->skb;
723 	const struct ethhdr *eth;
724 	unsigned int hlen = sizeof(*eth);
725 
726 	napi->skb = NULL;
727 
728 	skb_reset_mac_header(skb);
729 	skb_gro_reset_offset(skb, hlen);
730 
731 	if (unlikely(skb_gro_header_hard(skb, hlen))) {
732 		eth = skb_gro_header_slow(skb, hlen, 0);
733 		if (unlikely(!eth)) {
734 			net_warn_ratelimited("%s: dropping impossible skb from %s\n",
735 					     __func__, napi->dev->name);
736 			napi_reuse_skb(napi, skb);
737 			return NULL;
738 		}
739 	} else {
740 		eth = (const struct ethhdr *)skb->data;
741 		gro_pull_from_frag0(skb, hlen);
742 		NAPI_GRO_CB(skb)->frag0 += hlen;
743 		NAPI_GRO_CB(skb)->frag0_len -= hlen;
744 	}
745 	__skb_pull(skb, hlen);
746 
747 	/*
748 	 * This works because the only protocols we care about don't require
749 	 * special handling.
750 	 * We'll fix it up properly in napi_frags_finish()
751 	 */
752 	skb->protocol = eth->h_proto;
753 
754 	return skb;
755 }
756 
napi_gro_frags(struct napi_struct * napi)757 gro_result_t napi_gro_frags(struct napi_struct *napi)
758 {
759 	gro_result_t ret;
760 	struct sk_buff *skb = napi_frags_skb(napi);
761 
762 	trace_napi_gro_frags_entry(skb);
763 
764 	ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
765 	trace_napi_gro_frags_exit(ret);
766 
767 	return ret;
768 }
769 EXPORT_SYMBOL(napi_gro_frags);
770 
771 /* Compute the checksum from gro_offset and return the folded value
772  * after adding in any pseudo checksum.
773  */
__skb_gro_checksum_complete(struct sk_buff * skb)774 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
775 {
776 	__wsum wsum;
777 	__sum16 sum;
778 
779 	wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
780 
781 	/* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
782 	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
783 	/* See comments in __skb_checksum_complete(). */
784 	if (likely(!sum)) {
785 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
786 		    !skb->csum_complete_sw)
787 			netdev_rx_csum_fault(skb->dev, skb);
788 	}
789 
790 	NAPI_GRO_CB(skb)->csum = wsum;
791 	NAPI_GRO_CB(skb)->csum_valid = 1;
792 
793 	return sum;
794 }
795 EXPORT_SYMBOL(__skb_gro_checksum_complete);
796