639 	struct xilinx_vdma_tx_segment *segment;  in xilinx_vdma_alloc_tx_segment()  local
642 	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);  in xilinx_vdma_alloc_tx_segment()
643 	if (!segment)  in xilinx_vdma_alloc_tx_segment()
646 	segment->phys = phys;  in xilinx_vdma_alloc_tx_segment()
648 	return segment;  in xilinx_vdma_alloc_tx_segment()
660 	struct xilinx_cdma_tx_segment *segment;  in xilinx_cdma_alloc_tx_segment()  local
663 	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);  in xilinx_cdma_alloc_tx_segment()
664 	if (!segment)  in xilinx_cdma_alloc_tx_segment()
667 	segment->phys = phys;  in xilinx_cdma_alloc_tx_segment()
669 	return segment;  in xilinx_cdma_alloc_tx_segment()
681 	struct xilinx_axidma_tx_segment *segment = NULL;  in xilinx_axidma_alloc_tx_segment()  local
686 		segment = list_first_entry(&chan->free_seg_list,  in xilinx_axidma_alloc_tx_segment()
689 		list_del(&segment->node);  in xilinx_axidma_alloc_tx_segment()
693 	if (!segment)  in xilinx_axidma_alloc_tx_segment()
696 	return segment;  in xilinx_axidma_alloc_tx_segment()
708 	struct xilinx_aximcdma_tx_segment *segment = NULL;  in xilinx_aximcdma_alloc_tx_segment()  local
713 		segment = list_first_entry(&chan->free_seg_list,  in xilinx_aximcdma_alloc_tx_segment()
716 		list_del(&segment->node);  in xilinx_aximcdma_alloc_tx_segment()
720 	return segment;  in xilinx_aximcdma_alloc_tx_segment()
751 				struct xilinx_axidma_tx_segment *segment)  in xilinx_dma_free_tx_segment()  argument
753 	xilinx_dma_clean_hw_desc(&segment->hw);  in xilinx_dma_free_tx_segment()
755 	list_add_tail(&segment->node, &chan->free_seg_list);  in xilinx_dma_free_tx_segment()
765 					 segment)  in xilinx_mcdma_free_tx_segment()
767 	xilinx_mcdma_clean_hw_desc(&segment->hw);  in xilinx_mcdma_free_tx_segment()
769 	list_add_tail(&segment->node, &chan->free_seg_list);  in xilinx_mcdma_free_tx_segment()
778 				struct xilinx_cdma_tx_segment *segment)  in xilinx_cdma_free_tx_segment()  argument
780 	dma_pool_free(chan->desc_pool, segment, segment->phys);  in xilinx_cdma_free_tx_segment()
789 					struct xilinx_vdma_tx_segment *segment)  in xilinx_vdma_free_tx_segment()  argument
791 	dma_pool_free(chan->desc_pool, segment, segment->phys);  in xilinx_vdma_free_tx_segment()
823 	struct xilinx_vdma_tx_segment *segment, *next;  in xilinx_dma_free_tx_descriptor()  local
832 		list_for_each_entry_safe(segment, next, &desc->segments, node) {  in xilinx_dma_free_tx_descriptor()
833 			list_del(&segment->node);  in xilinx_dma_free_tx_descriptor()
834 			xilinx_vdma_free_tx_segment(chan, segment);  in xilinx_dma_free_tx_descriptor()
1335 	struct xilinx_vdma_tx_segment *segment, *last = NULL;  in xilinx_vdma_start_transfer()  local
1396 	list_for_each_entry(segment, &desc->segments, node) {  in xilinx_vdma_start_transfer()
1400 				   segment->hw.buf_addr,  in xilinx_vdma_start_transfer()
1401 				   segment->hw.buf_addr_msb);  in xilinx_vdma_start_transfer()
1405 					segment->hw.buf_addr);  in xilinx_vdma_start_transfer()
1407 		last = segment;  in xilinx_vdma_start_transfer()
1476 		struct xilinx_cdma_tx_segment *segment;  in xilinx_cdma_start_transfer()  local
1479 		segment = list_first_entry(&head_desc->segments,  in xilinx_cdma_start_transfer()
1483 		hw = &segment->hw;  in xilinx_cdma_start_transfer()
1553 		struct xilinx_axidma_tx_segment *segment;  in xilinx_dma_start_transfer()  local
1556 		segment = list_first_entry(&head_desc->segments,  in xilinx_dma_start_transfer()
1559 		hw = &segment->hw;  in xilinx_dma_start_transfer()
2008 	struct xilinx_vdma_tx_segment *segment;  in xilinx_vdma_dma_prep_interleaved()  local
2030 	segment = xilinx_vdma_alloc_tx_segment(chan);  in xilinx_vdma_dma_prep_interleaved()
2031 	if (!segment)  in xilinx_vdma_dma_prep_interleaved()
2035 	hw = &segment->hw;  in xilinx_vdma_dma_prep_interleaved()
2060 	list_add_tail(&segment->node, &desc->segments);  in xilinx_vdma_dma_prep_interleaved()
2063 	segment = list_first_entry(&desc->segments,  in xilinx_vdma_dma_prep_interleaved()
2065 	desc->async_tx.phys = segment->phys;  in xilinx_vdma_dma_prep_interleaved()
2090 	struct xilinx_cdma_tx_segment *segment;  in xilinx_cdma_prep_memcpy()  local
2104 	segment = xilinx_cdma_alloc_tx_segment(chan);  in xilinx_cdma_prep_memcpy()
2105 	if (!segment)  in xilinx_cdma_prep_memcpy()
2108 	hw = &segment->hw;  in xilinx_cdma_prep_memcpy()
2118 	list_add_tail(&segment->node, &desc->segments);  in xilinx_cdma_prep_memcpy()
2120 	desc->async_tx.phys = segment->phys;  in xilinx_cdma_prep_memcpy()
2121 	hw->next_desc = segment->phys;  in xilinx_cdma_prep_memcpy()
2148 	struct xilinx_cdma_tx_segment *segment, *prev = NULL;  in xilinx_cdma_prep_memcpy_sg()  local
2179 		segment = xilinx_cdma_alloc_tx_segment(chan);  in xilinx_cdma_prep_memcpy_sg()
2180 		if (!segment)  in xilinx_cdma_prep_memcpy_sg()
2187 		hw = &segment->hw;  in xilinx_cdma_prep_memcpy_sg()
2197 			prev->hw.next_desc = segment->phys;  in xilinx_cdma_prep_memcpy_sg()
2200 					upper_32_bits(segment->phys);  in xilinx_cdma_prep_memcpy_sg()
2203 		prev = segment;  in xilinx_cdma_prep_memcpy_sg()
2206 		list_add_tail(&segment->node, &desc->segments);  in xilinx_cdma_prep_memcpy_sg()
2238 	segment = list_first_entry(&desc->segments,  in xilinx_cdma_prep_memcpy_sg()
2240 	desc->async_tx.phys = segment->phys;  in xilinx_cdma_prep_memcpy_sg()
2241 	prev->hw.next_desc = segment->phys;  in xilinx_cdma_prep_memcpy_sg()
2268 	struct xilinx_axidma_tx_segment *segment = NULL;  in xilinx_dma_prep_slave_sg()  local
2295 			segment = xilinx_axidma_alloc_tx_segment(chan);  in xilinx_dma_prep_slave_sg()
2296 			if (!segment)  in xilinx_dma_prep_slave_sg()
2305 			hw = &segment->hw;  in xilinx_dma_prep_slave_sg()
2325 			list_add_tail(&segment->node, &desc->segments);  in xilinx_dma_prep_slave_sg()
2329 	segment = list_first_entry(&desc->segments,  in xilinx_dma_prep_slave_sg()
2331 	desc->async_tx.phys = segment->phys;  in xilinx_dma_prep_slave_sg()
2335 		segment->hw.control |= XILINX_DMA_BD_SOP;  in xilinx_dma_prep_slave_sg()
2336 		segment = list_last_entry(&desc->segments,  in xilinx_dma_prep_slave_sg()
2339 		segment->hw.control |= XILINX_DMA_BD_EOP;  in xilinx_dma_prep_slave_sg()
2367 	struct xilinx_axidma_tx_segment *segment, *head_segment, *prev = NULL;  in xilinx_dma_prep_dma_cyclic()  local
2400 			segment = xilinx_axidma_alloc_tx_segment(chan);  in xilinx_dma_prep_dma_cyclic()
2401 			if (!segment)  in xilinx_dma_prep_dma_cyclic()
2410 			hw = &segment->hw;  in xilinx_dma_prep_dma_cyclic()
2416 				prev->hw.next_desc = segment->phys;  in xilinx_dma_prep_dma_cyclic()
2418 			prev = segment;  in xilinx_dma_prep_dma_cyclic()
2425 			list_add_tail(&segment->node, &desc->segments);  in xilinx_dma_prep_dma_cyclic()
2438 	segment = list_last_entry(&desc->segments,  in xilinx_dma_prep_dma_cyclic()
2441 	segment->hw.next_desc = (u32) head_segment->phys;  in xilinx_dma_prep_dma_cyclic()
2446 		segment->hw.control |= XILINX_DMA_BD_EOP;  in xilinx_dma_prep_dma_cyclic()
2475 	struct xilinx_aximcdma_tx_segment *segment = NULL;  in xilinx_mcdma_prep_slave_sg()  local
2502 			segment = xilinx_aximcdma_alloc_tx_segment(chan);  in xilinx_mcdma_prep_slave_sg()
2503 			if (!segment)  in xilinx_mcdma_prep_slave_sg()
2512 			hw = &segment->hw;  in xilinx_mcdma_prep_slave_sg()
2529 			list_add_tail(&segment->node, &desc->segments);  in xilinx_mcdma_prep_slave_sg()
2533 	segment = list_first_entry(&desc->segments,  in xilinx_mcdma_prep_slave_sg()
2535 	desc->async_tx.phys = segment->phys;  in xilinx_mcdma_prep_slave_sg()
2539 		segment->hw.control |= XILINX_MCDMA_BD_SOP;  in xilinx_mcdma_prep_slave_sg()
2540 		segment = list_last_entry(&desc->segments,  in xilinx_mcdma_prep_slave_sg()
2543 		segment->hw.control |= XILINX_MCDMA_BD_EOP;  in xilinx_mcdma_prep_slave_sg()