543 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)  in tsnep_rx_ring_cleanup()  argument
545 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_ring_cleanup()
550 		entry = &rx->entry[i];  in tsnep_rx_ring_cleanup()
559 	memset(rx->entry, 0, sizeof(rx->entry));  in tsnep_rx_ring_cleanup()
562 		if (rx->page[i]) {  in tsnep_rx_ring_cleanup()
563 			dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],  in tsnep_rx_ring_cleanup()
564 					  rx->page_dma[i]);  in tsnep_rx_ring_cleanup()
565 			rx->page[i] = NULL;  in tsnep_rx_ring_cleanup()
566 			rx->page_dma[i] = 0;  in tsnep_rx_ring_cleanup()
571 static int tsnep_rx_alloc_and_map_skb(struct tsnep_rx *rx,  in tsnep_rx_alloc_and_map_skb()  argument
574 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_alloc_and_map_skb()
578 	skb = __netdev_alloc_skb(rx->adapter->netdev, RX_SKB_ALLOC_LENGTH,  in tsnep_rx_alloc_and_map_skb()
595 	entry->desc->rx = __cpu_to_le64(dma);  in tsnep_rx_alloc_and_map_skb()
600 static int tsnep_rx_ring_init(struct tsnep_rx *rx)  in tsnep_rx_ring_init()  argument
602 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_ring_init()
609 		rx->page[i] =  in tsnep_rx_ring_init()
610 			dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],  in tsnep_rx_ring_init()
612 		if (!rx->page[i]) {  in tsnep_rx_ring_init()
617 			entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];  in tsnep_rx_ring_init()
619 				(((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);  in tsnep_rx_ring_init()
622 			entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;  in tsnep_rx_ring_init()
626 		entry = &rx->entry[i];  in tsnep_rx_ring_init()
627 		next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE];  in tsnep_rx_ring_init()
630 		retval = tsnep_rx_alloc_and_map_skb(rx, entry);  in tsnep_rx_ring_init()
638 	tsnep_rx_ring_cleanup(rx);  in tsnep_rx_ring_init()
642 static void tsnep_rx_activate(struct tsnep_rx *rx, int index)  in tsnep_rx_activate()  argument
644 	struct tsnep_rx_entry *entry = &rx->entry[index];  in tsnep_rx_activate()
649 	if (index == rx->increment_owner_counter) {  in tsnep_rx_activate()
650 		rx->owner_counter++;  in tsnep_rx_activate()
651 		if (rx->owner_counter == 4)  in tsnep_rx_activate()
652 			rx->owner_counter = 1;  in tsnep_rx_activate()
653 		rx->increment_owner_counter--;  in tsnep_rx_activate()
654 		if (rx->increment_owner_counter < 0)  in tsnep_rx_activate()
655 			rx->increment_owner_counter = TSNEP_RING_SIZE - 1;  in tsnep_rx_activate()
658 		(rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &  in tsnep_rx_activate()
669 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,  in tsnep_rx_poll()  argument
672 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_poll()
683 		entry = &rx->entry[rx->read];  in tsnep_rx_poll()
701 		retval = tsnep_rx_alloc_and_map_skb(rx, entry);  in tsnep_rx_poll()
708 			if (rx->adapter->hwtstamp_config.rx_filter ==  in tsnep_rx_poll()
722 						       rx->adapter->netdev);  in tsnep_rx_poll()
724 			rx->packets++;  in tsnep_rx_poll()
725 			rx->bytes += length - TSNEP_RX_INLINE_METADATA_SIZE;  in tsnep_rx_poll()
727 				rx->multicast++;  in tsnep_rx_poll()
732 			rx->dropped++;  in tsnep_rx_poll()
735 		tsnep_rx_activate(rx, rx->read);  in tsnep_rx_poll()
739 		rx->read = (rx->read + 1) % TSNEP_RING_SIZE;  in tsnep_rx_poll()
748 		iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);  in tsnep_rx_poll()
755 			 struct tsnep_rx *rx)  in tsnep_rx_open()  argument
761 	memset(rx, 0, sizeof(*rx));  in tsnep_rx_open()
762 	rx->adapter = adapter;  in tsnep_rx_open()
763 	rx->addr = addr;  in tsnep_rx_open()
765 	retval = tsnep_rx_ring_init(rx);  in tsnep_rx_open()
769 	dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;  in tsnep_rx_open()
770 	iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);  in tsnep_rx_open()
771 	iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);  in tsnep_rx_open()
772 	rx->owner_counter = 1;  in tsnep_rx_open()
773 	rx->increment_owner_counter = TSNEP_RING_SIZE - 1;  in tsnep_rx_open()
776 		tsnep_rx_activate(rx, i);  in tsnep_rx_open()
781 	iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);  in tsnep_rx_open()
786 static void tsnep_rx_close(struct tsnep_rx *rx)  in tsnep_rx_close()  argument
790 	iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);  in tsnep_rx_close()
791 	readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,  in tsnep_rx_close()
795 	tsnep_rx_ring_cleanup(rx);  in tsnep_rx_close()
808 	if (queue->rx) {  in tsnep_poll()
809 		done = tsnep_rx_poll(queue->rx, napi, budget);  in tsnep_poll()
847 		if (adapter->queue[i].rx) {  in tsnep_netdev_open()
850 					       adapter->queue[i].rx);  in tsnep_netdev_open()
878 		if (adapter->queue[i].rx)  in tsnep_netdev_open()
879 			tsnep_rx_close(adapter->queue[i].rx);  in tsnep_netdev_open()
898 		if (adapter->queue[i].rx)  in tsnep_netdev_close()
899 			tsnep_rx_close(adapter->queue[i].rx);  in tsnep_netdev_close()
961 		stats->rx_packets += adapter->rx[i].packets;  in tsnep_netdev_get_stats64()
962 		stats->rx_bytes += adapter->rx[i].bytes;  in tsnep_netdev_get_stats64()
963 		stats->rx_dropped += adapter->rx[i].dropped;  in tsnep_netdev_get_stats64()
964 		stats->multicast += adapter->rx[i].multicast;  in tsnep_netdev_get_stats64()
1192 	adapter->queue[0].rx = &adapter->rx[0];  in tsnep_probe()