50 			struct iova_domain	iovad;  member
139 		free_iova_fast(&cookie->iovad,  in fq_ring_free()
287 		return cookie->iovad.granule;  in cookie_msi_granule()
365 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {  in iommu_put_dma_cookie()
367 		put_iova_domain(&cookie->iovad);  in iommu_put_dma_cookie()
400 	struct iova_domain *iovad = &cookie->iovad;  in cookie_init_hw_msi_region()  local
404 	start -= iova_offset(iovad, start);  in cookie_init_hw_msi_region()
405 	num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);  in cookie_init_hw_msi_region()
416 		start += iovad->granule;  in cookie_init_hw_msi_region()
432 		struct iova_domain *iovad)  in iova_reserve_pci_windows()  argument
443 		lo = iova_pfn(iovad, window->res->start - window->offset);  in iova_reserve_pci_windows()
444 		hi = iova_pfn(iovad, window->res->end - window->offset);  in iova_reserve_pci_windows()
445 		reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
454 			lo = iova_pfn(iovad, start);  in iova_reserve_pci_windows()
455 			hi = iova_pfn(iovad, end);  in iova_reserve_pci_windows()
456 			reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
481 	struct iova_domain *iovad = &cookie->iovad;  in iova_reserve_iommu_regions()  local
487 		ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);  in iova_reserve_iommu_regions()
500 		lo = iova_pfn(iovad, region->start);  in iova_reserve_iommu_regions()
501 		hi = iova_pfn(iovad, region->start + region->length - 1);  in iova_reserve_iommu_regions()
502 		reserve_iova(iovad, lo, hi);  in iova_reserve_iommu_regions()
542 	struct iova_domain *iovad;  in iommu_dma_init_domain()  local
548 	iovad = &cookie->iovad;  in iommu_dma_init_domain()
568 	if (iovad->start_pfn) {  in iommu_dma_init_domain()
569 		if (1UL << order != iovad->granule ||  in iommu_dma_init_domain()
570 		    base_pfn != iovad->start_pfn) {  in iommu_dma_init_domain()
580 	init_iova_domain(iovad, 1UL << order, base_pfn);  in iommu_dma_init_domain()
581 	ret = iova_domain_init_rcaches(iovad);  in iommu_dma_init_domain()
629 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_iova()  local
637 	shift = iova_shift(iovad);  in iommu_dma_alloc_iova()
647 		iova = alloc_iova_fast(iovad, iova_len,  in iommu_dma_alloc_iova()
651 		iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,  in iommu_dma_alloc_iova()
660 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_free_iova()  local
666 		queue_iova(cookie, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
667 				size >> iova_shift(iovad),  in iommu_dma_free_iova()
670 		free_iova_fast(iovad, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
671 				size >> iova_shift(iovad));  in iommu_dma_free_iova()
679 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_unmap()  local
680 	size_t iova_off = iova_offset(iovad, dma_addr);  in __iommu_dma_unmap()
685 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_unmap()
702 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_map()  local
703 	size_t iova_off = iova_offset(iovad, phys);  in __iommu_dma_map()
710 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_map()
795 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_alloc_noncontiguous()  local
823 	size = iova_align(iovad, size);  in __iommu_dma_alloc_noncontiguous()
986 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_page()  local
993 	if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) {  in iommu_dma_map_page()
1002 		aligned_size = iova_align(iovad, size);  in iommu_dma_map_page()
1004 					      iova_mask(iovad), dir, attrs);  in iommu_dma_map_page()
1188 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_sg()  local
1218 		size_t s_iova_off = iova_offset(iovad, s->offset);  in iommu_dma_map_sg()
1249 		s_length = iova_align(iovad, s_length + s_iova_off);  in iommu_dma_map_sg()