13 			       struct gve_rx_slot_page_info *page_info,  in gve_rx_free_buffer()  argument
19 	page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);  in gve_rx_free_buffer()
20 	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);  in gve_rx_free_buffer()
30 			gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],  in gve_rx_unfill_pages()
34 			page_ref_sub(rx->data.page_info[i].page,  in gve_rx_unfill_pages()
35 				     rx->data.page_info[i].pagecnt_bias - 1);  in gve_rx_unfill_pages()
39 	kvfree(rx->data.page_info);  in gve_rx_unfill_pages()
40 	rx->data.page_info = NULL;  in gve_rx_unfill_pages()
69 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,  in gve_setup_rx_buffer()  argument
72 	page_info->page = page;  in gve_setup_rx_buffer()
73 	page_info->page_offset = 0;  in gve_setup_rx_buffer()
74 	page_info->page_address = page_address(page);  in gve_setup_rx_buffer()
78 	page_info->pagecnt_bias = INT_MAX;  in gve_setup_rx_buffer()
82 			       struct gve_rx_slot_page_info *page_info,  in gve_rx_alloc_buffer()  argument
94 	gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);  in gve_rx_alloc_buffer()
110 	rx->data.page_info = kvzalloc(slots *  in gve_prefill_rx_pages()
111 				      sizeof(*rx->data.page_info), GFP_KERNEL);  in gve_prefill_rx_pages()
112 	if (!rx->data.page_info)  in gve_prefill_rx_pages()
118 			kvfree(rx->data.page_info);  in gve_prefill_rx_pages()
119 			rx->data.page_info = NULL;  in gve_prefill_rx_pages()
128 			gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,  in gve_prefill_rx_pages()
132 		err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],  in gve_prefill_rx_pages()
142 				   &rx->data.page_info[i],  in gve_prefill_rx_pages()
301 					struct gve_rx_slot_page_info *page_info,  in gve_rx_add_frags()  argument
305 	u32 offset = page_info->page_offset +  gve_rx_ctx_padding(ctx);  in gve_rx_add_frags()
315 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,  in gve_rx_add_frags()
321 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)  in gve_rx_flip_buff()  argument
326 	page_info->page_offset ^= PAGE_SIZE / 2;  in gve_rx_flip_buff()
330 static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)  in gve_rx_can_recycle_buffer()  argument
332 	int pagecount = page_count(page_info->page);  in gve_rx_can_recycle_buffer()
335 	if (pagecount == page_info->pagecnt_bias)  in gve_rx_can_recycle_buffer()
338 	else if (pagecount > page_info->pagecnt_bias)  in gve_rx_can_recycle_buffer()
340 	WARN(pagecount < page_info->pagecnt_bias,  in gve_rx_can_recycle_buffer()
347 		      struct gve_rx_slot_page_info *page_info, u16 len,  in gve_rx_raw_addressing()  argument
352 	struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);  in gve_rx_raw_addressing()
361 	gve_dec_pagecnt_bias(page_info);  in gve_rx_raw_addressing()
368 	   struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,  in gve_rx_qpl()  argument
381 		skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);  in gve_rx_qpl()
385 			gve_dec_pagecnt_bias(page_info);  in gve_rx_qpl()
386 			gve_rx_flip_buff(page_info, &data_slot->qpl_offset);  in gve_rx_qpl()
391 		skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);  in gve_rx_qpl()
411 	struct gve_rx_slot_page_info *page_info;  in gve_rx_ctx_init()  local
445 		page_info = &rx->data.page_info[idx];  in gve_rx_ctx_init()
447 			recycle = gve_rx_can_recycle_buffer(page_info);  in gve_rx_ctx_init()
489 				  struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,  in gve_rx_skb()  argument
498 		skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);  in gve_rx_skb()
508 			int recycle = gve_rx_can_recycle_buffer(page_info);  in gve_rx_skb()
514 			page_info->can_flip = recycle;  in gve_rx_skb()
515 			if (page_info->can_flip) {  in gve_rx_skb()
521 						    page_info, len, napi,  in gve_rx_skb()
531 					 page_info, len, napi, data_slot);  in gve_rx_skb()
540 	struct gve_rx_slot_page_info *page_info;  in gve_rx()  local
564 		page_info = &rx->data.page_info[(idx + 2) & rx->mask];  in gve_rx()
565 		va = page_info->page_address + page_info->page_offset;  in gve_rx()
567 		prefetch(page_info->page); /* Kernel page struct. */  in gve_rx()
573 		page_info = &rx->data.page_info[idx];  in gve_rx()
576 			   be64_to_cpu(data_slot->addr) - page_info->page_offset :  in gve_rx()
580 		skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);  in gve_rx()
654 		struct gve_rx_slot_page_info *page_info;  in gve_rx_refill_buffers()  local
657 		page_info = &rx->data.page_info[idx];  in gve_rx_refill_buffers()
658 		if (page_info->can_flip) {  in gve_rx_refill_buffers()
665 			gve_rx_flip_buff(page_info, &data_slot->addr);  in gve_rx_refill_buffers()
666 			page_info->can_flip = 0;  in gve_rx_refill_buffers()
675 			int recycle = gve_rx_can_recycle_buffer(page_info);  in gve_rx_refill_buffers()
687 				gve_rx_free_buffer(dev, page_info, data_slot);  in gve_rx_refill_buffers()
688 				page_info->page = NULL;  in gve_rx_refill_buffers()
689 				if (gve_rx_alloc_buffer(priv, dev, page_info,  in gve_rx_refill_buffers()