38 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_find_table()  local
40 	list_for_each_entry_lockless(stt, &kvm->arch.spapr_tce_tables, list)  in kvmppc_find_table()
41 		if (stt->liobn == liobn)  in kvmppc_find_table()
42 			return stt;  in kvmppc_find_table()
84 	struct kvmppc_spapr_tce_table *stt;  in kvm_spapr_tce_release_iommu_group()  local
89 	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {  in kvm_spapr_tce_release_iommu_group()
95 		list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {  in kvm_spapr_tce_release_iommu_group()
111 	struct kvmppc_spapr_tce_table *stt = NULL;  in kvm_spapr_tce_attach_iommu_group()  local
124 	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {  in kvm_spapr_tce_attach_iommu_group()
125 		if (stt == f.file->private_data) {  in kvm_spapr_tce_attach_iommu_group()
147 		if ((tbltmp->it_page_shift <= stt->page_shift) &&  in kvm_spapr_tce_attach_iommu_group()
149 				 stt->offset << stt->page_shift) &&  in kvm_spapr_tce_attach_iommu_group()
151 				 stt->size << stt->page_shift)) {  in kvm_spapr_tce_attach_iommu_group()
164 	list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {  in kvm_spapr_tce_attach_iommu_group()
192 	list_add_rcu(&stit->next, &stt->iommu_tables);  in kvm_spapr_tce_attach_iommu_group()
199 	struct kvmppc_spapr_tce_table *stt = container_of(head,  in release_spapr_tce_table()  local
201 	unsigned long i, npages = kvmppc_tce_pages(stt->size);  in release_spapr_tce_table()
204 		if (stt->pages[i])  in release_spapr_tce_table()
205 			__free_page(stt->pages[i]);  in release_spapr_tce_table()
207 	kfree(stt);  in release_spapr_tce_table()
210 static struct page *kvm_spapr_get_tce_page(struct kvmppc_spapr_tce_table *stt,  in kvm_spapr_get_tce_page()  argument
213 	struct page *page = stt->pages[sttpage];  in kvm_spapr_get_tce_page()
218 	mutex_lock(&stt->alloc_lock);  in kvm_spapr_get_tce_page()
219 	page = stt->pages[sttpage];  in kvm_spapr_get_tce_page()
224 			stt->pages[sttpage] = page;  in kvm_spapr_get_tce_page()
226 	mutex_unlock(&stt->alloc_lock);  in kvm_spapr_get_tce_page()
233 	struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data;  in kvm_spapr_tce_fault()  local
236 	if (vmf->pgoff >= kvmppc_tce_pages(stt->size))  in kvm_spapr_tce_fault()
239 	page = kvm_spapr_get_tce_page(stt, vmf->pgoff);  in kvm_spapr_tce_fault()
260 	struct kvmppc_spapr_tce_table *stt = filp->private_data;  in kvm_spapr_tce_release()  local
262 	struct kvm *kvm = stt->kvm;  in kvm_spapr_tce_release()
265 	list_del_rcu(&stt->list);  in kvm_spapr_tce_release()
268 	list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {  in kvm_spapr_tce_release()
277 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);  in kvm_spapr_tce_release()
279 	kvm_put_kvm(stt->kvm);  in kvm_spapr_tce_release()
281 	call_rcu(&stt->rcu, release_spapr_tce_table);  in kvm_spapr_tce_release()
294 	struct kvmppc_spapr_tce_table *stt = NULL;  in kvm_vm_ioctl_create_spapr_tce()  local
310 	stt = kzalloc(struct_size(stt, pages, npages), GFP_KERNEL | __GFP_NOWARN);  in kvm_vm_ioctl_create_spapr_tce()
311 	if (!stt)  in kvm_vm_ioctl_create_spapr_tce()
314 	stt->liobn = args->liobn;  in kvm_vm_ioctl_create_spapr_tce()
315 	stt->page_shift = args->page_shift;  in kvm_vm_ioctl_create_spapr_tce()
316 	stt->offset = args->offset;  in kvm_vm_ioctl_create_spapr_tce()
317 	stt->size = args->size;  in kvm_vm_ioctl_create_spapr_tce()
318 	stt->kvm = kvm;  in kvm_vm_ioctl_create_spapr_tce()
319 	mutex_init(&stt->alloc_lock);  in kvm_vm_ioctl_create_spapr_tce()
320 	INIT_LIST_HEAD_RCU(&stt->iommu_tables);  in kvm_vm_ioctl_create_spapr_tce()
336 				       stt, O_RDWR | O_CLOEXEC);  in kvm_vm_ioctl_create_spapr_tce()
339 		list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);  in kvm_vm_ioctl_create_spapr_tce()
348 	kfree(stt);  in kvm_vm_ioctl_create_spapr_tce()
370 static long kvmppc_tce_validate(struct kvmppc_spapr_tce_table *stt,  in kvmppc_tce_validate()  argument
382 	if (iommu_tce_check_gpa(stt->page_shift, gpa))  in kvmppc_tce_validate()
385 	if (kvmppc_tce_to_ua(stt->kvm, tce, &ua))  in kvmppc_tce_validate()
389 	list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {  in kvmppc_tce_validate()
394 		mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift);  in kvmppc_tce_validate()
410 static void kvmppc_tce_put(struct kvmppc_spapr_tce_table *stt,  in kvmppc_tce_put()  argument
417 	idx -= stt->offset;  in kvmppc_tce_put()
419 	page = stt->pages[sttpage];  in kvmppc_tce_put()
426 		page = kvm_spapr_get_tce_page(stt, sttpage);  in kvmppc_tce_put()
435 static void kvmppc_clear_tce(struct mm_struct *mm, struct kvmppc_spapr_tce_table *stt,  in kvmppc_clear_tce()  argument
439 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_clear_tce()
440 	unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift);  in kvmppc_clear_tce()
493 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_unmap()  argument
497 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_unmap()
550 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_map()  argument
555 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_map()
575 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_put_tce()  local
584 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_put_tce()
585 	if (!stt)  in kvmppc_h_put_tce()
588 	ret = kvmppc_ioba_validate(stt, ioba, 1);  in kvmppc_h_put_tce()
594 	ret = kvmppc_tce_validate(stt, tce);  in kvmppc_h_put_tce()
605 	entry = ioba >> stt->page_shift;  in kvmppc_h_put_tce()
607 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {  in kvmppc_h_put_tce()
609 			ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,  in kvmppc_h_put_tce()
612 			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,  in kvmppc_h_put_tce()
617 			kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry);  in kvmppc_h_put_tce()
622 	kvmppc_tce_put(stt, entry, tce);  in kvmppc_h_put_tce()
635 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_put_tce_indirect()  local
642 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_put_tce_indirect()
643 	if (!stt)  in kvmppc_h_put_tce_indirect()
646 	entry = ioba >> stt->page_shift;  in kvmppc_h_put_tce_indirect()
657 	ret = kvmppc_ioba_validate(stt, ioba, npages);  in kvmppc_h_put_tce_indirect()
675 		ret = kvmppc_tce_validate(stt, tce);  in kvmppc_h_put_tce_indirect()
702 		list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {  in kvmppc_h_put_tce_indirect()
703 			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt,  in kvmppc_h_put_tce_indirect()
708 				kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl,  in kvmppc_h_put_tce_indirect()
714 		kvmppc_tce_put(stt, entry + i, tce);  in kvmppc_h_put_tce_indirect()
728 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_stuff_tce()  local
732 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_stuff_tce()
733 	if (!stt)  in kvmppc_h_stuff_tce()
736 	ret = kvmppc_ioba_validate(stt, ioba, npages);  in kvmppc_h_stuff_tce()
744 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {  in kvmppc_h_stuff_tce()
745 		unsigned long entry = ioba >> stt->page_shift;  in kvmppc_h_stuff_tce()
748 			ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,  in kvmppc_h_stuff_tce()
758 			kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i);  in kvmppc_h_stuff_tce()
762 	for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))  in kvmppc_h_stuff_tce()
763 		kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);  in kvmppc_h_stuff_tce()
772 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_get_tce()  local
778 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_get_tce()
779 	if (!stt)  in kvmppc_h_get_tce()
782 	ret = kvmppc_ioba_validate(stt, ioba, 1);  in kvmppc_h_get_tce()
786 	idx = (ioba >> stt->page_shift) - stt->offset;  in kvmppc_h_get_tce()
787 	page = stt->pages[idx / TCES_PER_PAGE];  in kvmppc_h_get_tce()