209 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_find_table()  local
211 		if (tbl) {  in tce_iommu_find_table()
212 			unsigned long entry = ioba >> tbl->it_page_shift;  in tce_iommu_find_table()
213 			unsigned long start = tbl->it_offset;  in tce_iommu_find_table()
214 			unsigned long end = start + tbl->it_size;  in tce_iommu_find_table()
217 				*ptbl = tbl;  in tce_iommu_find_table()
339 		struct iommu_table *tbl,
342 		struct iommu_table *tbl);
362 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_release()  local
364 		if (!tbl)  in tce_iommu_release()
367 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_release()
368 		tce_iommu_free_table(container, tbl);  in tce_iommu_release()
411 		struct iommu_table *tbl, unsigned long entry)  in tce_iommu_unuse_page_v2()  argument
416 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);  in tce_iommu_unuse_page_v2()
422 			tbl->it_page_shift, &hpa, &mem);  in tce_iommu_unuse_page_v2()
433 		struct iommu_table *tbl,  in tce_iommu_clear()  argument
442 		if (tbl->it_indirect_levels && tbl->it_userspace) {  in tce_iommu_clear()
451 			__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,  in tce_iommu_clear()
455 				entry |= tbl->it_level_size - 1;  in tce_iommu_clear()
464 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,  in tce_iommu_clear()
473 			tce_iommu_unuse_page_v2(container, tbl, entry);  in tce_iommu_clear()
480 	iommu_tce_kill(tbl, firstentry, pages);  in tce_iommu_clear()
501 		struct iommu_table *tbl,  in tce_iommu_build()  argument
510 		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;  in tce_iommu_build()
517 				tbl->it_page_shift)) {  in tce_iommu_build()
524 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,  in tce_iommu_build()
529 					__func__, entry << tbl->it_page_shift,  in tce_iommu_build()
537 		tce += IOMMU_PAGE_SIZE(tbl);  in tce_iommu_build()
541 		tce_iommu_clear(container, tbl, entry, i);  in tce_iommu_build()
543 		iommu_tce_kill(tbl, entry, pages);  in tce_iommu_build()
549 		struct iommu_table *tbl,  in tce_iommu_build_v2()  argument
559 		__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);  in tce_iommu_build_v2()
562 				tce, tbl->it_page_shift, &hpa, &mem);  in tce_iommu_build_v2()
567 				tbl->it_page_shift)) {  in tce_iommu_build_v2()
573 		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;  in tce_iommu_build_v2()
580 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,  in tce_iommu_build_v2()
584 			tce_iommu_unuse_page_v2(container, tbl, entry + i);  in tce_iommu_build_v2()
586 					__func__, entry << tbl->it_page_shift,  in tce_iommu_build_v2()
592 			tce_iommu_unuse_page_v2(container, tbl, entry + i);  in tce_iommu_build_v2()
596 		tce += IOMMU_PAGE_SIZE(tbl);  in tce_iommu_build_v2()
600 		tce_iommu_clear(container, tbl, entry, i);  in tce_iommu_build_v2()
602 		iommu_tce_kill(tbl, entry, pages);  in tce_iommu_build_v2()
636 		struct iommu_table *tbl)  in tce_iommu_free_table()  argument
638 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;  in tce_iommu_free_table()
640 	iommu_tce_table_put(tbl);  in tce_iommu_free_table()
650 	struct iommu_table *tbl = NULL;  in tce_iommu_create_window()  local
674 			page_shift, window_size, levels, &tbl);  in tce_iommu_create_window()
678 	BUG_ON(!tbl->it_ops->free);  in tce_iommu_create_window()
687 		ret = table_group->ops->set_window(table_group, num, tbl);  in tce_iommu_create_window()
692 	container->tables[num] = tbl;  in tce_iommu_create_window()
695 	*start_addr = tbl->it_offset << tbl->it_page_shift;  in tce_iommu_create_window()
704 	tce_iommu_free_table(container, tbl);  in tce_iommu_create_window()
713 	struct iommu_table *tbl;  in tce_iommu_remove_window()  local
717 	num = tce_iommu_find_table(container, start_addr, &tbl);  in tce_iommu_remove_window()
721 	BUG_ON(!tbl->it_size);  in tce_iommu_remove_window()
741 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_remove_window()
742 	tce_iommu_free_table(container, tbl);  in tce_iommu_remove_window()
907 		struct iommu_table *tbl = NULL;  in tce_iommu_ioctl()  local
930 		num = tce_iommu_find_table(container, param.iova, &tbl);  in tce_iommu_ioctl()
934 		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||  in tce_iommu_ioctl()
935 				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))  in tce_iommu_ioctl()
951 		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);  in tce_iommu_ioctl()
956 			ret = tce_iommu_build_v2(container, tbl,  in tce_iommu_ioctl()
957 					param.iova >> tbl->it_page_shift,  in tce_iommu_ioctl()
959 					param.size >> tbl->it_page_shift,  in tce_iommu_ioctl()
962 			ret = tce_iommu_build(container, tbl,  in tce_iommu_ioctl()
963 					param.iova >> tbl->it_page_shift,  in tce_iommu_ioctl()
965 					param.size >> tbl->it_page_shift,  in tce_iommu_ioctl()
968 		iommu_flush_tce(tbl);  in tce_iommu_ioctl()
974 		struct iommu_table *tbl = NULL;  in tce_iommu_ioctl()  local
997 		num = tce_iommu_find_table(container, param.iova, &tbl);  in tce_iommu_ioctl()
1001 		if (param.size & ~IOMMU_PAGE_MASK(tbl))  in tce_iommu_ioctl()
1004 		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,  in tce_iommu_ioctl()
1005 				param.size >> tbl->it_page_shift);  in tce_iommu_ioctl()
1009 		ret = tce_iommu_clear(container, tbl,  in tce_iommu_ioctl()
1010 				param.iova >> tbl->it_page_shift,  in tce_iommu_ioctl()
1011 				param.size >> tbl->it_page_shift);  in tce_iommu_ioctl()
1012 		iommu_flush_tce(tbl);  in tce_iommu_ioctl()
1212 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_take_ownership()  local
1214 		if (!tbl)  in tce_iommu_take_ownership()
1217 		ret = table_group->ops->set_window(table_group, i, tbl);  in tce_iommu_take_ownership()