69 #define IOPTE_IS_DUMMY(iommu, iopte)	\  argument
70 	((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
72 static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)  in iopte_make_dummy()  argument
74 	unsigned long val = iopte_val(*iopte);  in iopte_make_dummy()
79 	iopte_val(*iopte) = val;  in iopte_make_dummy()
94 	iopte_t *iopte, *limit, *first;  in alloc_streaming_cluster()  local
100 	iopte  = (iommu->page_table +  in alloc_streaming_cluster()
107 		limit = (iopte +  in alloc_streaming_cluster()
110 	iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);  in alloc_streaming_cluster()
113 	first = iopte;  in alloc_streaming_cluster()
115 		if (IOPTE_IS_DUMMY(iommu, iopte)) {  in alloc_streaming_cluster()
116 			if ((iopte + (1 << cnum)) >= limit)  in alloc_streaming_cluster()
125 		iopte += (1 << cnum);  in alloc_streaming_cluster()
127 		if (iopte >= limit) {  in alloc_streaming_cluster()
128 			iopte = (iommu->page_table +  in alloc_streaming_cluster()
135 		if (iopte == first)  in alloc_streaming_cluster()
140 	return iopte;  in alloc_streaming_cluster()
173 	iopte_t *iopte;  in alloc_consistent_cluster()  local
175 	iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));  in alloc_consistent_cluster()
176 	while (iopte > iommu->page_table) {  in alloc_consistent_cluster()
177 		iopte--;  in alloc_consistent_cluster()
178 		if (IOPTE_IS_DUMMY(iommu, iopte)) {  in alloc_consistent_cluster()
182 				iopte--;  in alloc_consistent_cluster()
183 				if (!IOPTE_IS_DUMMY(iommu, iopte))  in alloc_consistent_cluster()
187 				u32 entry = (iopte - iommu->page_table);  in alloc_consistent_cluster()
191 				return iopte;  in alloc_consistent_cluster()
206 	iopte_t *iopte;  in pci_alloc_consistent()  local
225 	iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT);  in pci_alloc_consistent()
226 	if (iopte == NULL) {  in pci_alloc_consistent()
233 		      ((iopte - iommu->page_table) << IO_PAGE_SHIFT));  in pci_alloc_consistent()
241 		iopte_val(*iopte) = (IOPTE_CONSISTENT(ctx) |  in pci_alloc_consistent()
244 		iopte++;  in pci_alloc_consistent()
269 	iopte_t *iopte;  in pci_free_consistent()  local
275 	iopte = iommu->page_table +  in pci_free_consistent()
280 	if ((iopte - iommu->page_table) ==  in pci_free_consistent()
282 		iopte_t *walk = iopte + npages;  in pci_free_consistent()
304 		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;  in pci_free_consistent()
306 	for (i = 0; i < npages; i++, iopte++)  in pci_free_consistent()
307 		iopte_make_dummy(iommu, iopte);  in pci_free_consistent()
456 static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg,  in fill_sg()  argument
500 				*iopte++ = __iopte(pteval);  in fill_sg()
716 		iopte_t *iopte;  in pci_dma_sync_single()  local
718 		iopte = iommu->page_table +  in pci_dma_sync_single()
720 		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;  in pci_dma_sync_single()
773 		iopte_t *iopte;  in pci_dma_sync_sg()  local
775 		iopte = iommu->page_table +  in pci_dma_sync_sg()
777 		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;  in pci_dma_sync_sg()