219 svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)  in svm_migrate_get_vram_page()  argument
224 	svm_range_bo_ref(prange->svm_bo);  in svm_migrate_get_vram_page()
225 	page->zone_device_data = prange->svm_bo;  in svm_migrate_get_vram_page()
296 svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange,  in svm_migrate_copy_to_vram()  argument
308 	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,  in svm_migrate_copy_to_vram()
309 		 prange->last);  in svm_migrate_copy_to_vram()
314 	r = svm_range_vram_node_new(adev, prange, true);  in svm_migrate_copy_to_vram()
320 	amdgpu_res_first(prange->ttm_res, prange->offset << PAGE_SHIFT,  in svm_migrate_copy_to_vram()
327 		svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
405 svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,  in svm_migrate_vma_to_vram()  argument
409 	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);  in svm_migrate_vma_to_vram()
438 				      0, adev->kfd.dev->id, prange->prefetch_loc,  in svm_migrate_vma_to_vram()
439 				      prange->preferred_loc, trigger);  in svm_migrate_vma_to_vram()
444 			__func__, r, prange->start, prange->last);  in svm_migrate_vma_to_vram()
451 			 prange->start, prange->last);  in svm_migrate_vma_to_vram()
460 	r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence, scratch);  in svm_migrate_vma_to_vram()
474 	svm_range_free_dma_mappings(prange);  in svm_migrate_vma_to_vram()
480 		pdd = svm_range_get_pdd_by_adev(prange, adev);  in svm_migrate_vma_to_vram()
502 svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,  in svm_migrate_ram_to_vram()  argument
511 	if (prange->actual_loc == best_loc) {  in svm_migrate_ram_to_vram()
513 			 prange->svms, prange->start, prange->last, best_loc);  in svm_migrate_ram_to_vram()
517 	adev = svm_range_get_adev_by_id(prange, best_loc);  in svm_migrate_ram_to_vram()
523 	pr_debug("svms 0x%p [0x%lx 0x%lx] to gpu 0x%x\n", prange->svms,  in svm_migrate_ram_to_vram()
524 		 prange->start, prange->last, best_loc);  in svm_migrate_ram_to_vram()
526 	start = prange->start << PAGE_SHIFT;  in svm_migrate_ram_to_vram()
527 	end = (prange->last + 1) << PAGE_SHIFT;  in svm_migrate_ram_to_vram()
537 		r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next, trigger);  in svm_migrate_ram_to_vram()
548 		prange->actual_loc = best_loc;  in svm_migrate_ram_to_vram()
564 svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,  in svm_migrate_copy_to_ram()  argument
576 	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,  in svm_migrate_copy_to_ram()
577 		 prange->last);  in svm_migrate_copy_to_ram()
579 	addr = prange->start << PAGE_SHIFT;  in svm_migrate_copy_to_ram()
590 				 prange->svms, prange->start, prange->last);  in svm_migrate_copy_to_ram()
616 				 prange->svms, prange->start, prange->last);  in svm_migrate_copy_to_ram()
667 svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,  in svm_migrate_vma_to_ram()  argument
671 	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);  in svm_migrate_vma_to_ram()
705 				      adev->kfd.dev->id, 0, prange->prefetch_loc,  in svm_migrate_vma_to_ram()
706 				      prange->preferred_loc, trigger);  in svm_migrate_vma_to_ram()
711 			__func__, r, prange->start, prange->last);  in svm_migrate_vma_to_ram()
718 			 prange->start, prange->last);  in svm_migrate_vma_to_ram()
728 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
749 		pdd = svm_range_get_pdd_by_adev(prange, adev);  in svm_migrate_vma_to_ram()
767 int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,  in svm_migrate_vram_to_ram()  argument
778 	if (!prange->actual_loc) {  in svm_migrate_vram_to_ram()
780 			 prange->start, prange->last);  in svm_migrate_vram_to_ram()
784 	adev = svm_range_get_adev_by_id(prange, prange->actual_loc);  in svm_migrate_vram_to_ram()
787 			 prange->actual_loc);  in svm_migrate_vram_to_ram()
792 		 prange->svms, prange, prange->start, prange->last,  in svm_migrate_vram_to_ram()
793 		 prange->actual_loc);  in svm_migrate_vram_to_ram()
795 	start = prange->start << PAGE_SHIFT;  in svm_migrate_vram_to_ram()
796 	end = (prange->last + 1) << PAGE_SHIFT;  in svm_migrate_vram_to_ram()
803 			pr_debug("failed to find vma for prange %p\n", prange);  in svm_migrate_vram_to_ram()
809 		r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next, trigger,  in svm_migrate_vram_to_ram()
812 			pr_debug("failed %ld to migrate prange %p\n", r, prange);  in svm_migrate_vram_to_ram()
821 		svm_range_vram_node_free(prange);  in svm_migrate_vram_to_ram()
822 		prange->actual_loc = 0;  in svm_migrate_vram_to_ram()
841 svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,  in svm_migrate_vram_to_vram()  argument
851 	pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);  in svm_migrate_vram_to_vram()
854 		r = svm_migrate_vram_to_ram(prange, mm, trigger, NULL);  in svm_migrate_vram_to_vram()
857 	} while (prange->actual_loc && --retries);  in svm_migrate_vram_to_vram()
859 	if (prange->actual_loc)  in svm_migrate_vram_to_vram()
862 	return svm_migrate_ram_to_vram(prange, best_loc, mm, trigger);  in svm_migrate_vram_to_vram()
866 svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,  in svm_migrate_to_vram()  argument
869 	if  (!prange->actual_loc)  in svm_migrate_to_vram()
870 		return svm_migrate_ram_to_vram(prange, best_loc, mm, trigger);  in svm_migrate_to_vram()
872 		return svm_migrate_vram_to_vram(prange, best_loc, mm, trigger);  in svm_migrate_to_vram()
892 	struct svm_range *prange;  in svm_migrate_to_ram()  local
928 	prange = svm_range_from_addr(&p->svms, addr, &parent);  in svm_migrate_to_ram()
929 	if (!prange) {  in svm_migrate_to_ram()
936 	if (prange != parent)  in svm_migrate_to_ram()
937 		mutex_lock_nested(&prange->migrate_mutex, 1);  in svm_migrate_to_ram()
939 	if (!prange->actual_loc)  in svm_migrate_to_ram()
943 	if (prange != parent)  in svm_migrate_to_ram()
944 		mutex_lock_nested(&prange->lock, 1);  in svm_migrate_to_ram()
945 	r = svm_range_split_by_granularity(p, mm, addr, parent, prange);  in svm_migrate_to_ram()
946 	if (prange != parent)  in svm_migrate_to_ram()
947 		mutex_unlock(&prange->lock);  in svm_migrate_to_ram()
954 	r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm,  in svm_migrate_to_ram()
959 			 r, prange->svms, prange, prange->start, prange->last);  in svm_migrate_to_ram()
962 	if (p->xnack_enabled && parent == prange)  in svm_migrate_to_ram()
970 	if (prange != parent)  in svm_migrate_to_ram()
971 		mutex_unlock(&prange->migrate_mutex);  in svm_migrate_to_ram()