52 			struct iova_domain	iovad;  member
141 		free_iova_fast(&cookie->iovad,  in fq_ring_free()
289 		return cookie->iovad.granule;  in cookie_msi_granule()
367 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {  in iommu_put_dma_cookie()
369 		put_iova_domain(&cookie->iovad);  in iommu_put_dma_cookie()
404 	struct iova_domain *iovad = &cookie->iovad;  in cookie_init_hw_msi_region()  local
408 	start -= iova_offset(iovad, start);  in cookie_init_hw_msi_region()
409 	num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);  in cookie_init_hw_msi_region()
420 		start += iovad->granule;  in cookie_init_hw_msi_region()
436 		struct iova_domain *iovad)  in iova_reserve_pci_windows()  argument
447 		lo = iova_pfn(iovad, window->res->start - window->offset);  in iova_reserve_pci_windows()
448 		hi = iova_pfn(iovad, window->res->end - window->offset);  in iova_reserve_pci_windows()
449 		reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
458 			lo = iova_pfn(iovad, start);  in iova_reserve_pci_windows()
459 			hi = iova_pfn(iovad, end);  in iova_reserve_pci_windows()
460 			reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
485 	struct iova_domain *iovad = &cookie->iovad;  in iova_reserve_iommu_regions()  local
491 		ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);  in iova_reserve_iommu_regions()
504 		lo = iova_pfn(iovad, region->start);  in iova_reserve_iommu_regions()
505 		hi = iova_pfn(iovad, region->start + region->length - 1);  in iova_reserve_iommu_regions()
506 		reserve_iova(iovad, lo, hi);  in iova_reserve_iommu_regions()
575 	struct iova_domain *iovad;  in iommu_dma_init_domain()  local
581 	iovad = &cookie->iovad;  in iommu_dma_init_domain()
601 	if (iovad->start_pfn) {  in iommu_dma_init_domain()
602 		if (1UL << order != iovad->granule ||  in iommu_dma_init_domain()
603 		    base_pfn != iovad->start_pfn) {  in iommu_dma_init_domain()
613 	init_iova_domain(iovad, 1UL << order, base_pfn);  in iommu_dma_init_domain()
614 	ret = iova_domain_init_rcaches(iovad);  in iommu_dma_init_domain()
663 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_iova()  local
671 	shift = iova_shift(iovad);  in iommu_dma_alloc_iova()
691 		iova = alloc_iova_fast(iovad, iova_len,  in iommu_dma_alloc_iova()
700 	iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true);  in iommu_dma_alloc_iova()
708 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_free_iova()  local
714 		queue_iova(cookie, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
715 				size >> iova_shift(iovad),  in iommu_dma_free_iova()
718 		free_iova_fast(iovad, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
719 				size >> iova_shift(iovad));  in iommu_dma_free_iova()
727 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_unmap()  local
728 	size_t iova_off = iova_offset(iovad, dma_addr);  in __iommu_dma_unmap()
733 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_unmap()
750 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_map()  local
751 	size_t iova_off = iova_offset(iovad, phys);  in __iommu_dma_map()
758 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_map()
840 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_alloc_noncontiguous()  local
868 	size = iova_align(iovad, size);  in __iommu_dma_alloc_noncontiguous()
1039 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_page()  local
1047 	    iova_offset(iovad, phys | size)) {  in iommu_dma_map_page()
1058 		aligned_size = iova_align(iovad, size);  in iommu_dma_map_page()
1060 					      iova_mask(iovad), dir, attrs);  in iommu_dma_map_page()
1246 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_sg()  local
1276 		size_t s_iova_off = iova_offset(iovad, s->offset);  in iommu_dma_map_sg()
1307 		s_length = iova_align(iovad, s_length + s_iova_off);  in iommu_dma_map_sg()