385 	info = vm->process_info;  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
965 				struct amdkfd_process_info *process_info,  in add_kgd_mem_to_kfd_bo_list()  argument
968 	mutex_lock(&process_info->lock);  in add_kgd_mem_to_kfd_bo_list()
971 			      &process_info->userptr_valid_list);  in add_kgd_mem_to_kfd_bo_list()
973 		list_add_tail(&mem->validate_list, &process_info->kfd_bo_list);  in add_kgd_mem_to_kfd_bo_list()
974 	mutex_unlock(&process_info->lock);  in add_kgd_mem_to_kfd_bo_list()
978 		struct amdkfd_process_info *process_info)  in remove_kgd_mem_from_kfd_bo_list()  argument
980 	mutex_lock(&process_info->lock);  in remove_kgd_mem_from_kfd_bo_list()
982 	mutex_unlock(&process_info->lock);  in remove_kgd_mem_from_kfd_bo_list()
1000 	struct amdkfd_process_info *process_info = mem->process_info;  in init_user_pages()  local
1006 	mutex_lock(&process_info->lock);  in init_user_pages()
1028 		mutex_lock(&process_info->notifier_lock);  in init_user_pages()
1030 		mutex_unlock(&process_info->notifier_lock);  in init_user_pages()
1031 		mutex_unlock(&process_info->lock);  in init_user_pages()
1058 	mutex_unlock(&process_info->lock);  in init_user_pages()
1263 static int process_validate_vms(struct amdkfd_process_info *process_info)  in process_validate_vms()  argument
1268 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in process_validate_vms()
1278 static int process_sync_pds_resv(struct amdkfd_process_info *process_info,  in process_sync_pds_resv()  argument
1284 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in process_sync_pds_resv()
1298 static int process_update_pds(struct amdkfd_process_info *process_info,  in process_update_pds()  argument
1304 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in process_update_pds()
1314 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,  in init_kfd_vm()  argument
1320 	if (!*process_info) {  in init_kfd_vm()
1346 		*process_info = info;  in init_kfd_vm()
1350 	vm->process_info = *process_info;  in init_kfd_vm()
1369 			   &vm->process_info->eviction_fence->base,  in init_kfd_vm()
1374 	mutex_lock(&vm->process_info->lock);  in init_kfd_vm()
1376 			&(vm->process_info->vm_list_head));  in init_kfd_vm()
1377 	vm->process_info->n_vms++;  in init_kfd_vm()
1378 	mutex_unlock(&vm->process_info->lock);  in init_kfd_vm()
1387 	vm->process_info = NULL;  in init_kfd_vm()
1393 		*process_info = NULL;  in init_kfd_vm()
1475 					   void **process_info,  in amdgpu_amdkfd_gpuvm_acquire_process_vm()  argument
1481 	if (avm->process_info)  in amdgpu_amdkfd_gpuvm_acquire_process_vm()
1490 	ret = init_kfd_vm(avm, process_info, ef);  in amdgpu_amdkfd_gpuvm_acquire_process_vm()
1502 	struct amdkfd_process_info *process_info = vm->process_info;  in amdgpu_amdkfd_gpuvm_destroy_cb()  local
1504 	if (!process_info)  in amdgpu_amdkfd_gpuvm_destroy_cb()
1508 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1509 	process_info->n_vms--;  in amdgpu_amdkfd_gpuvm_destroy_cb()
1511 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1513 	vm->process_info = NULL;  in amdgpu_amdkfd_gpuvm_destroy_cb()
1516 	if (!process_info->n_vms) {  in amdgpu_amdkfd_gpuvm_destroy_cb()
1517 		WARN_ON(!list_empty(&process_info->kfd_bo_list));  in amdgpu_amdkfd_gpuvm_destroy_cb()
1518 		WARN_ON(!list_empty(&process_info->userptr_valid_list));  in amdgpu_amdkfd_gpuvm_destroy_cb()
1519 		WARN_ON(!list_empty(&process_info->userptr_inval_list));  in amdgpu_amdkfd_gpuvm_destroy_cb()
1521 		dma_fence_put(&process_info->eviction_fence->base);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1522 		cancel_delayed_work_sync(&process_info->restore_userptr_work);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1523 		put_pid(process_info->pid);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1524 		mutex_destroy(&process_info->lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1525 		mutex_destroy(&process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1526 		kfree(process_info);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1753 	(*mem)->process_info = avm->process_info;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1755 	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1780 	remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1805 	struct amdkfd_process_info *process_info = mem->process_info;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()  local
1837 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1839 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1844 		mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1846 		mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1858 					process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1938 	mutex_lock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1945 		mutex_lock(&mem->process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1947 		mutex_unlock(&mem->process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2024 				   &avm->process_info->eviction_fence->base,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2033 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2042 	struct amdkfd_process_info *process_info = avm->process_info;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()  local
2090 						process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2147 		bo, bo->vm_bo->vm->process_info->eviction_fence);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2188 	mutex_lock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2209 		bo, mem->process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2216 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2224 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2314 	(*mem)->process_info = avm->process_info;  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2315 	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2358 	struct amdkfd_process_info *process_info = mem->process_info;  in amdgpu_amdkfd_evict_userptr()  local
2364 	if (READ_ONCE(process_info->block_mmu_notifications))  in amdgpu_amdkfd_evict_userptr()
2367 	mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_evict_userptr()
2371 	if (++process_info->evicted_bos == 1) {  in amdgpu_amdkfd_evict_userptr()
2377 		schedule_delayed_work(&process_info->restore_userptr_work,  in amdgpu_amdkfd_evict_userptr()
2380 	mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_evict_userptr()
2391 static int update_invalid_user_pages(struct amdkfd_process_info *process_info,  in update_invalid_user_pages()  argument
2400 	mutex_lock(&process_info->notifier_lock);  in update_invalid_user_pages()
2404 				 &process_info->userptr_valid_list,  in update_invalid_user_pages()
2408 				       &process_info->userptr_inval_list);  in update_invalid_user_pages()
2411 	list_for_each_entry(mem, &process_info->userptr_inval_list,  in update_invalid_user_pages()
2428 		mutex_unlock(&process_info->notifier_lock);  in update_invalid_user_pages()
2465 		mutex_lock(&process_info->notifier_lock);  in update_invalid_user_pages()
2480 	mutex_unlock(&process_info->notifier_lock);  in update_invalid_user_pages()
2490 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)  in validate_invalid_user_pages()  argument
2507 		list_for_each_entry(peer_vm, &process_info->vm_list_head,  in validate_invalid_user_pages()
2516 		list_for_each_entry(mem, &process_info->userptr_inval_list,  in validate_invalid_user_pages()
2528 	ret = process_validate_vms(process_info);  in validate_invalid_user_pages()
2534 				 &process_info->userptr_inval_list,  in validate_invalid_user_pages()
2565 				mutex_lock(&process_info->notifier_lock);  in validate_invalid_user_pages()
2567 				mutex_unlock(&process_info->notifier_lock);  in validate_invalid_user_pages()
2574 	ret = process_update_pds(process_info, &sync);  in validate_invalid_user_pages()
2588 static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)  in confirm_valid_user_pages_locked()  argument
2594 				 &process_info->userptr_inval_list,  in confirm_valid_user_pages_locked()
2620 			       &process_info->userptr_valid_list);  in confirm_valid_user_pages_locked()
2635 	struct amdkfd_process_info *process_info =  in amdgpu_amdkfd_restore_userptr_worker()  local
2642 	mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2643 	evicted_bos = process_info->evicted_bos;  in amdgpu_amdkfd_restore_userptr_worker()
2644 	mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2649 	usertask = get_pid_task(process_info->pid, PIDTYPE_PID);  in amdgpu_amdkfd_restore_userptr_worker()
2658 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_restore_userptr_worker()
2660 	if (update_invalid_user_pages(process_info, mm))  in amdgpu_amdkfd_restore_userptr_worker()
2666 	if (!list_empty(&process_info->userptr_inval_list)) {  in amdgpu_amdkfd_restore_userptr_worker()
2667 		if (validate_invalid_user_pages(process_info))  in amdgpu_amdkfd_restore_userptr_worker()
2675 	mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2676 	if (process_info->evicted_bos != evicted_bos)  in amdgpu_amdkfd_restore_userptr_worker()
2679 	if (confirm_valid_user_pages_locked(process_info)) {  in amdgpu_amdkfd_restore_userptr_worker()
2684 	process_info->evicted_bos = evicted_bos = 0;  in amdgpu_amdkfd_restore_userptr_worker()
2694 	mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2696 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_restore_userptr_worker()
2700 		schedule_delayed_work(&process_info->restore_userptr_work,  in amdgpu_amdkfd_restore_userptr_worker()
2729 	struct amdkfd_process_info *process_info = info;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2742 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2746 		list_for_each_entry(peer_vm, &process_info->vm_list_head,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2757 		list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2772 	ret = process_validate_vms(process_info);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2776 	ret = process_sync_pds_resv(process_info, &sync_obj);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2783 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2833 	ret = process_update_pds(process_info, &sync_obj);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2847 				process_info->eviction_fence->base.context,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2848 				process_info->eviction_fence->mm,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2855 	dma_fence_put(&process_info->eviction_fence->base);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2856 	process_info->eviction_fence = new_fence;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2860 	list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2865 				   &process_info->eviction_fence->base,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2869 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2874 				   &process_info->eviction_fence->base,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2882 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2888 	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;  in amdgpu_amdkfd_add_gws_to_process()  local
2903 	(*mem)->process_info = process_info;  in amdgpu_amdkfd_add_gws_to_process()
2904 	add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);  in amdgpu_amdkfd_add_gws_to_process()
2909 	mutex_lock(&(*mem)->process_info->lock);  in amdgpu_amdkfd_add_gws_to_process()
2929 			   &process_info->eviction_fence->base,  in amdgpu_amdkfd_add_gws_to_process()
2932 	mutex_unlock(&(*mem)->process_info->lock);  in amdgpu_amdkfd_add_gws_to_process()
2940 	mutex_unlock(&(*mem)->process_info->lock);  in amdgpu_amdkfd_add_gws_to_process()
2942 	remove_kgd_mem_from_kfd_bo_list(*mem, process_info);  in amdgpu_amdkfd_add_gws_to_process()
2953 	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;  in amdgpu_amdkfd_remove_gws_from_process()  local
2960 	remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);  in amdgpu_amdkfd_remove_gws_from_process()
2969 			process_info->eviction_fence);  in amdgpu_amdkfd_remove_gws_from_process()