1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/dma-fence-array.h>
30 #include <linux/interval_tree_generic.h>
31 #include <linux/idr.h>
32 #include <linux/dma-buf.h>
33 
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include "amdgpu.h"
37 #include "amdgpu_trace.h"
38 #include "amdgpu_amdkfd.h"
39 #include "amdgpu_gmc.h"
40 #include "amdgpu_xgmi.h"
41 #include "amdgpu_dma_buf.h"
42 #include "amdgpu_res_cursor.h"
43 #include "kfd_svm.h"
44 
45 /**
46  * DOC: GPUVM
47  *
48  * GPUVM is similar to the legacy gart on older asics, however
49  * rather than there being a single global gart table
50  * for the entire GPU, there are multiple VM page tables active
51  * at any given time.  The VM page tables can contain a mix
52  * vram pages and system memory pages and system memory pages
53  * can be mapped as snooped (cached system pages) or unsnooped
54  * (uncached system pages).
55  * Each VM has an ID associated with it and there is a page table
56  * associated with each VMID.  When executing a command buffer,
57  * the kernel tells the ring what VMID to use for that command
58  * buffer.  VMIDs are allocated dynamically as commands are submitted.
59  * The userspace drivers maintain their own address space and the kernel
60  * sets up their pages tables accordingly when they submit their
61  * command buffers and a VMID is assigned.
62  * Cayman/Trinity support up to 8 active VMs at any given time;
63  * SI supports 16.
64  */
65 
66 #define START(node) ((node)->start)
67 #define LAST(node) ((node)->last)
68 
69 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
70 		     START, LAST, static, amdgpu_vm_it)
71 
72 #undef START
73 #undef LAST
74 
75 /**
76  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
77  */
78 struct amdgpu_prt_cb {
79 
80 	/**
81 	 * @adev: amdgpu device
82 	 */
83 	struct amdgpu_device *adev;
84 
85 	/**
86 	 * @cb: callback
87 	 */
88 	struct dma_fence_cb cb;
89 };
90 
91 /**
92  * struct amdgpu_vm_tlb_seq_cb - Helper to increment the TLB flush sequence
93  */
94 struct amdgpu_vm_tlb_seq_cb {
95 	/**
96 	 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on
97 	 */
98 	struct amdgpu_vm *vm;
99 
100 	/**
101 	 * @cb: callback
102 	 */
103 	struct dma_fence_cb cb;
104 };
105 
106 /**
107  * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping
108  *
109  * @adev: amdgpu_device pointer
110  * @vm: amdgpu_vm pointer
111  * @pasid: the pasid the VM is using on this GPU
112  *
113  * Set the pasid this VM is using on this GPU, can also be used to remove the
114  * pasid by passing in zero.
115  *
116  */
amdgpu_vm_set_pasid(struct amdgpu_device * adev,struct amdgpu_vm * vm,u32 pasid)117 int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
118 			u32 pasid)
119 {
120 	int r;
121 
122 	if (vm->pasid == pasid)
123 		return 0;
124 
125 	if (vm->pasid) {
126 		r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid));
127 		if (r < 0)
128 			return r;
129 
130 		vm->pasid = 0;
131 	}
132 
133 	if (pasid) {
134 		r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm,
135 					GFP_KERNEL));
136 		if (r < 0)
137 			return r;
138 
139 		vm->pasid = pasid;
140 	}
141 
142 
143 	return 0;
144 }
145 
146 /**
147  * amdgpu_vm_bo_evicted - vm_bo is evicted
148  *
149  * @vm_bo: vm_bo which is evicted
150  *
151  * State for PDs/PTs and per VM BOs which are not at the location they should
152  * be.
153  */
amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base * vm_bo)154 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
155 {
156 	struct amdgpu_vm *vm = vm_bo->vm;
157 	struct amdgpu_bo *bo = vm_bo->bo;
158 
159 	vm_bo->moved = true;
160 	spin_lock(&vm_bo->vm->status_lock);
161 	if (bo->tbo.type == ttm_bo_type_kernel)
162 		list_move(&vm_bo->vm_status, &vm->evicted);
163 	else
164 		list_move_tail(&vm_bo->vm_status, &vm->evicted);
165 	spin_unlock(&vm_bo->vm->status_lock);
166 }
167 /**
168  * amdgpu_vm_bo_moved - vm_bo is moved
169  *
170  * @vm_bo: vm_bo which is moved
171  *
172  * State for per VM BOs which are moved, but that change is not yet reflected
173  * in the page tables.
174  */
amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base * vm_bo)175 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
176 {
177 	spin_lock(&vm_bo->vm->status_lock);
178 	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
179 	spin_unlock(&vm_bo->vm->status_lock);
180 }
181 
182 /**
183  * amdgpu_vm_bo_idle - vm_bo is idle
184  *
185  * @vm_bo: vm_bo which is now idle
186  *
187  * State for PDs/PTs and per VM BOs which have gone through the state machine
188  * and are now idle.
189  */
amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base * vm_bo)190 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
191 {
192 	spin_lock(&vm_bo->vm->status_lock);
193 	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
194 	spin_unlock(&vm_bo->vm->status_lock);
195 	vm_bo->moved = false;
196 }
197 
198 /**
199  * amdgpu_vm_bo_invalidated - vm_bo is invalidated
200  *
201  * @vm_bo: vm_bo which is now invalidated
202  *
203  * State for normal BOs which are invalidated and that change not yet reflected
204  * in the PTs.
205  */
amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base * vm_bo)206 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
207 {
208 	spin_lock(&vm_bo->vm->status_lock);
209 	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
210 	spin_unlock(&vm_bo->vm->status_lock);
211 }
212 
213 /**
214  * amdgpu_vm_bo_relocated - vm_bo is reloacted
215  *
216  * @vm_bo: vm_bo which is relocated
217  *
218  * State for PDs/PTs which needs to update their parent PD.
219  * For the root PD, just move to idle state.
220  */
amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base * vm_bo)221 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
222 {
223 	if (vm_bo->bo->parent) {
224 		spin_lock(&vm_bo->vm->status_lock);
225 		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
226 		spin_unlock(&vm_bo->vm->status_lock);
227 	} else {
228 		amdgpu_vm_bo_idle(vm_bo);
229 	}
230 }
231 
232 /**
233  * amdgpu_vm_bo_done - vm_bo is done
234  *
235  * @vm_bo: vm_bo which is now done
236  *
237  * State for normal BOs which are invalidated and that change has been updated
238  * in the PTs.
239  */
amdgpu_vm_bo_done(struct amdgpu_vm_bo_base * vm_bo)240 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
241 {
242 	spin_lock(&vm_bo->vm->status_lock);
243 	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
244 	spin_unlock(&vm_bo->vm->status_lock);
245 }
246 
247 /**
248  * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
249  *
250  * @base: base structure for tracking BO usage in a VM
251  * @vm: vm to which bo is to be added
252  * @bo: amdgpu buffer object
253  *
254  * Initialize a bo_va_base structure and add it to the appropriate lists
255  *
256  */
amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base * base,struct amdgpu_vm * vm,struct amdgpu_bo * bo)257 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
258 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo)
259 {
260 	base->vm = vm;
261 	base->bo = bo;
262 	base->next = NULL;
263 	INIT_LIST_HEAD(&base->vm_status);
264 
265 	if (!bo)
266 		return;
267 	base->next = bo->vm_bo;
268 	bo->vm_bo = base;
269 
270 	if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
271 		return;
272 
273 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
274 
275 	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
276 	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
277 		amdgpu_vm_bo_relocated(base);
278 	else
279 		amdgpu_vm_bo_idle(base);
280 
281 	if (bo->preferred_domains &
282 	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
283 		return;
284 
285 	/*
286 	 * we checked all the prerequisites, but it looks like this per vm bo
287 	 * is currently evicted. add the bo to the evicted list to make sure it
288 	 * is validated on next vm use to avoid fault.
289 	 * */
290 	amdgpu_vm_bo_evicted(base);
291 }
292 
293 /**
294  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
295  *
296  * @vm: vm providing the BOs
297  * @validated: head of validation list
298  * @entry: entry to add
299  *
300  * Add the page directory to the list of BOs to
301  * validate for command submission.
302  */
amdgpu_vm_get_pd_bo(struct amdgpu_vm * vm,struct list_head * validated,struct amdgpu_bo_list_entry * entry)303 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
304 			 struct list_head *validated,
305 			 struct amdgpu_bo_list_entry *entry)
306 {
307 	entry->priority = 0;
308 	entry->tv.bo = &vm->root.bo->tbo;
309 	/* Two for VM updates, one for TTM and one for the CS job */
310 	entry->tv.num_shared = 4;
311 	entry->user_pages = NULL;
312 	list_add(&entry->tv.head, validated);
313 }
314 
315 /**
316  * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
317  *
318  * @adev: amdgpu device pointer
319  * @vm: vm providing the BOs
320  *
321  * Move all BOs to the end of LRU and remember their positions to put them
322  * together.
323  */
amdgpu_vm_move_to_lru_tail(struct amdgpu_device * adev,struct amdgpu_vm * vm)324 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
325 				struct amdgpu_vm *vm)
326 {
327 	spin_lock(&adev->mman.bdev.lru_lock);
328 	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
329 	spin_unlock(&adev->mman.bdev.lru_lock);
330 }
331 
332 /**
333  * amdgpu_vm_validate_pt_bos - validate the page table BOs
334  *
335  * @adev: amdgpu device pointer
336  * @vm: vm providing the BOs
337  * @validate: callback to do the validation
338  * @param: parameter for the validation callback
339  *
340  * Validate the page table BOs on command submission if neccessary.
341  *
342  * Returns:
343  * Validation result.
344  */
amdgpu_vm_validate_pt_bos(struct amdgpu_device * adev,struct amdgpu_vm * vm,int (* validate)(void * p,struct amdgpu_bo * bo),void * param)345 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
346 			      int (*validate)(void *p, struct amdgpu_bo *bo),
347 			      void *param)
348 {
349 	struct amdgpu_vm_bo_base *bo_base;
350 	struct amdgpu_bo *shadow;
351 	struct amdgpu_bo *bo;
352 	int r;
353 
354 	spin_lock(&vm->status_lock);
355 	while (!list_empty(&vm->evicted)) {
356 		bo_base = list_first_entry(&vm->evicted,
357 					   struct amdgpu_vm_bo_base,
358 					   vm_status);
359 		spin_unlock(&vm->status_lock);
360 
361 		bo = bo_base->bo;
362 		shadow = amdgpu_bo_shadowed(bo);
363 
364 		r = validate(param, bo);
365 		if (r)
366 			return r;
367 		if (shadow) {
368 			r = validate(param, shadow);
369 			if (r)
370 				return r;
371 		}
372 
373 		if (bo->tbo.type != ttm_bo_type_kernel) {
374 			amdgpu_vm_bo_moved(bo_base);
375 		} else {
376 			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
377 			amdgpu_vm_bo_relocated(bo_base);
378 		}
379 		spin_lock(&vm->status_lock);
380 	}
381 	spin_unlock(&vm->status_lock);
382 
383 	amdgpu_vm_eviction_lock(vm);
384 	vm->evicting = false;
385 	amdgpu_vm_eviction_unlock(vm);
386 
387 	return 0;
388 }
389 
390 /**
391  * amdgpu_vm_ready - check VM is ready for updates
392  *
393  * @vm: VM to check
394  *
395  * Check if all VM PDs/PTs are ready for updates
396  *
397  * Returns:
398  * True if VM is not evicting.
399  */
amdgpu_vm_ready(struct amdgpu_vm * vm)400 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
401 {
402 	bool empty;
403 	bool ret;
404 
405 	amdgpu_vm_eviction_lock(vm);
406 	ret = !vm->evicting;
407 	amdgpu_vm_eviction_unlock(vm);
408 
409 	spin_lock(&vm->status_lock);
410 	empty = list_empty(&vm->evicted);
411 	spin_unlock(&vm->status_lock);
412 
413 	return ret && empty;
414 }
415 
416 /**
417  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
418  *
419  * @adev: amdgpu_device pointer
420  */
amdgpu_vm_check_compute_bug(struct amdgpu_device * adev)421 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
422 {
423 	const struct amdgpu_ip_block *ip_block;
424 	bool has_compute_vm_bug;
425 	struct amdgpu_ring *ring;
426 	int i;
427 
428 	has_compute_vm_bug = false;
429 
430 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
431 	if (ip_block) {
432 		/* Compute has a VM bug for GFX version < 7.
433 		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
434 		if (ip_block->version->major <= 7)
435 			has_compute_vm_bug = true;
436 		else if (ip_block->version->major == 8)
437 			if (adev->gfx.mec_fw_version < 673)
438 				has_compute_vm_bug = true;
439 	}
440 
441 	for (i = 0; i < adev->num_rings; i++) {
442 		ring = adev->rings[i];
443 		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
444 			/* only compute rings */
445 			ring->has_compute_vm_bug = has_compute_vm_bug;
446 		else
447 			ring->has_compute_vm_bug = false;
448 	}
449 }
450 
451 /**
452  * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
453  *
454  * @ring: ring on which the job will be submitted
455  * @job: job to submit
456  *
457  * Returns:
458  * True if sync is needed.
459  */
amdgpu_vm_need_pipeline_sync(struct amdgpu_ring * ring,struct amdgpu_job * job)460 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
461 				  struct amdgpu_job *job)
462 {
463 	struct amdgpu_device *adev = ring->adev;
464 	unsigned vmhub = ring->funcs->vmhub;
465 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
466 	struct amdgpu_vmid *id;
467 	bool gds_switch_needed;
468 	bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
469 
470 	if (job->vmid == 0)
471 		return false;
472 	id = &id_mgr->ids[job->vmid];
473 	gds_switch_needed = ring->funcs->emit_gds_switch && (
474 		id->gds_base != job->gds_base ||
475 		id->gds_size != job->gds_size ||
476 		id->gws_base != job->gws_base ||
477 		id->gws_size != job->gws_size ||
478 		id->oa_base != job->oa_base ||
479 		id->oa_size != job->oa_size);
480 
481 	if (amdgpu_vmid_had_gpu_reset(adev, id))
482 		return true;
483 
484 	return vm_flush_needed || gds_switch_needed;
485 }
486 
487 /**
488  * amdgpu_vm_flush - hardware flush the vm
489  *
490  * @ring: ring to use for flush
491  * @job:  related job
492  * @need_pipe_sync: is pipe sync needed
493  *
494  * Emit a VM flush when it is necessary.
495  *
496  * Returns:
497  * 0 on success, errno otherwise.
498  */
amdgpu_vm_flush(struct amdgpu_ring * ring,struct amdgpu_job * job,bool need_pipe_sync)499 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
500 		    bool need_pipe_sync)
501 {
502 	struct amdgpu_device *adev = ring->adev;
503 	unsigned vmhub = ring->funcs->vmhub;
504 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
505 	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
506 	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
507 		id->gds_base != job->gds_base ||
508 		id->gds_size != job->gds_size ||
509 		id->gws_base != job->gws_base ||
510 		id->gws_size != job->gws_size ||
511 		id->oa_base != job->oa_base ||
512 		id->oa_size != job->oa_size);
513 	bool vm_flush_needed = job->vm_needs_flush;
514 	struct dma_fence *fence = NULL;
515 	bool pasid_mapping_needed = false;
516 	unsigned patch_offset = 0;
517 	bool update_spm_vmid_needed = (job->vm && (job->vm->reserved_vmid[vmhub] != NULL));
518 	int r;
519 
520 	if (update_spm_vmid_needed && adev->gfx.rlc.funcs->update_spm_vmid)
521 		adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid);
522 
523 	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
524 		gds_switch_needed = true;
525 		vm_flush_needed = true;
526 		pasid_mapping_needed = true;
527 	}
528 
529 	mutex_lock(&id_mgr->lock);
530 	if (id->pasid != job->pasid || !id->pasid_mapping ||
531 	    !dma_fence_is_signaled(id->pasid_mapping))
532 		pasid_mapping_needed = true;
533 	mutex_unlock(&id_mgr->lock);
534 
535 	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
536 	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
537 			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
538 	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
539 		ring->funcs->emit_wreg;
540 
541 	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
542 		return 0;
543 
544 	if (ring->funcs->init_cond_exec)
545 		patch_offset = amdgpu_ring_init_cond_exec(ring);
546 
547 	if (need_pipe_sync)
548 		amdgpu_ring_emit_pipeline_sync(ring);
549 
550 	if (vm_flush_needed) {
551 		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
552 		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
553 	}
554 
555 	if (pasid_mapping_needed)
556 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
557 
558 	if (vm_flush_needed || pasid_mapping_needed) {
559 		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
560 		if (r)
561 			return r;
562 	}
563 
564 	if (vm_flush_needed) {
565 		mutex_lock(&id_mgr->lock);
566 		dma_fence_put(id->last_flush);
567 		id->last_flush = dma_fence_get(fence);
568 		id->current_gpu_reset_count =
569 			atomic_read(&adev->gpu_reset_counter);
570 		mutex_unlock(&id_mgr->lock);
571 	}
572 
573 	if (pasid_mapping_needed) {
574 		mutex_lock(&id_mgr->lock);
575 		id->pasid = job->pasid;
576 		dma_fence_put(id->pasid_mapping);
577 		id->pasid_mapping = dma_fence_get(fence);
578 		mutex_unlock(&id_mgr->lock);
579 	}
580 	dma_fence_put(fence);
581 
582 	if (!ring->is_mes_queue && ring->funcs->emit_gds_switch &&
583 	    gds_switch_needed) {
584 		id->gds_base = job->gds_base;
585 		id->gds_size = job->gds_size;
586 		id->gws_base = job->gws_base;
587 		id->gws_size = job->gws_size;
588 		id->oa_base = job->oa_base;
589 		id->oa_size = job->oa_size;
590 		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
591 					    job->gds_size, job->gws_base,
592 					    job->gws_size, job->oa_base,
593 					    job->oa_size);
594 	}
595 
596 	if (ring->funcs->patch_cond_exec)
597 		amdgpu_ring_patch_cond_exec(ring, patch_offset);
598 
599 	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
600 	if (ring->funcs->emit_switch_buffer) {
601 		amdgpu_ring_emit_switch_buffer(ring);
602 		amdgpu_ring_emit_switch_buffer(ring);
603 	}
604 	return 0;
605 }
606 
607 /**
608  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
609  *
610  * @vm: requested vm
611  * @bo: requested buffer object
612  *
613  * Find @bo inside the requested vm.
614  * Search inside the @bos vm list for the requested vm
615  * Returns the found bo_va or NULL if none is found
616  *
617  * Object has to be reserved!
618  *
619  * Returns:
620  * Found bo_va or NULL.
621  */
amdgpu_vm_bo_find(struct amdgpu_vm * vm,struct amdgpu_bo * bo)622 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
623 				       struct amdgpu_bo *bo)
624 {
625 	struct amdgpu_vm_bo_base *base;
626 
627 	for (base = bo->vm_bo; base; base = base->next) {
628 		if (base->vm != vm)
629 			continue;
630 
631 		return container_of(base, struct amdgpu_bo_va, base);
632 	}
633 	return NULL;
634 }
635 
636 /**
637  * amdgpu_vm_map_gart - Resolve gart mapping of addr
638  *
639  * @pages_addr: optional DMA address to use for lookup
640  * @addr: the unmapped addr
641  *
642  * Look up the physical address of the page that the pte resolves
643  * to.
644  *
645  * Returns:
646  * The pointer for the page table entry.
647  */
amdgpu_vm_map_gart(const dma_addr_t * pages_addr,uint64_t addr)648 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
649 {
650 	uint64_t result;
651 
652 	/* page table offset */
653 	result = pages_addr[addr >> PAGE_SHIFT];
654 
655 	/* in case cpu page size != gpu page size*/
656 	result |= addr & (~PAGE_MASK);
657 
658 	result &= 0xFFFFFFFFFFFFF000ULL;
659 
660 	return result;
661 }
662 
663 /**
664  * amdgpu_vm_update_pdes - make sure that all directories are valid
665  *
666  * @adev: amdgpu_device pointer
667  * @vm: requested vm
668  * @immediate: submit immediately to the paging queue
669  *
670  * Makes sure all directories are up to date.
671  *
672  * Returns:
673  * 0 for success, error for failure.
674  */
amdgpu_vm_update_pdes(struct amdgpu_device * adev,struct amdgpu_vm * vm,bool immediate)675 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
676 			  struct amdgpu_vm *vm, bool immediate)
677 {
678 	struct amdgpu_vm_update_params params;
679 	struct amdgpu_vm_bo_base *entry;
680 	bool flush_tlb_needed = false;
681 	LIST_HEAD(relocated);
682 	int r, idx;
683 
684 	spin_lock(&vm->status_lock);
685 	list_splice_init(&vm->relocated, &relocated);
686 	spin_unlock(&vm->status_lock);
687 
688 	if (list_empty(&relocated))
689 		return 0;
690 
691 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
692 		return -ENODEV;
693 
694 	memset(&params, 0, sizeof(params));
695 	params.adev = adev;
696 	params.vm = vm;
697 	params.immediate = immediate;
698 
699 	r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT);
700 	if (r)
701 		goto error;
702 
703 	list_for_each_entry(entry, &relocated, vm_status) {
704 		/* vm_flush_needed after updating moved PDEs */
705 		flush_tlb_needed |= entry->moved;
706 
707 		r = amdgpu_vm_pde_update(&params, entry);
708 		if (r)
709 			goto error;
710 	}
711 
712 	r = vm->update_funcs->commit(&params, &vm->last_update);
713 	if (r)
714 		goto error;
715 
716 	if (flush_tlb_needed)
717 		atomic64_inc(&vm->tlb_seq);
718 
719 	while (!list_empty(&relocated)) {
720 		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
721 					 vm_status);
722 		amdgpu_vm_bo_idle(entry);
723 	}
724 
725 error:
726 	drm_dev_exit(idx);
727 	return r;
728 }
729 
730 /**
731  * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
732  * @fence: unused
733  * @cb: the callback structure
734  *
735  * Increments the tlb sequence to make sure that future CS execute a VM flush.
736  */
amdgpu_vm_tlb_seq_cb(struct dma_fence * fence,struct dma_fence_cb * cb)737 static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
738 				 struct dma_fence_cb *cb)
739 {
740 	struct amdgpu_vm_tlb_seq_cb *tlb_cb;
741 
742 	tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
743 	atomic64_inc(&tlb_cb->vm->tlb_seq);
744 	kfree(tlb_cb);
745 }
746 
747 /**
748  * amdgpu_vm_update_range - update a range in the vm page table
749  *
750  * @adev: amdgpu_device pointer to use for commands
751  * @vm: the VM to update the range
752  * @immediate: immediate submission in a page fault
753  * @unlocked: unlocked invalidation during MM callback
754  * @flush_tlb: trigger tlb invalidation after update completed
755  * @resv: fences we need to sync to
756  * @start: start of mapped range
757  * @last: last mapped entry
758  * @flags: flags for the entries
759  * @offset: offset into nodes and pages_addr
760  * @vram_base: base for vram mappings
761  * @res: ttm_resource to map
762  * @pages_addr: DMA addresses to use for mapping
763  * @fence: optional resulting fence
764  *
765  * Fill in the page table entries between @start and @last.
766  *
767  * Returns:
768  * 0 for success, negative erro code for failure.
769  */
amdgpu_vm_update_range(struct amdgpu_device * adev,struct amdgpu_vm * vm,bool immediate,bool unlocked,bool flush_tlb,struct dma_resv * resv,uint64_t start,uint64_t last,uint64_t flags,uint64_t offset,uint64_t vram_base,struct ttm_resource * res,dma_addr_t * pages_addr,struct dma_fence ** fence)770 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
771 			   bool immediate, bool unlocked, bool flush_tlb,
772 			   struct dma_resv *resv, uint64_t start, uint64_t last,
773 			   uint64_t flags, uint64_t offset, uint64_t vram_base,
774 			   struct ttm_resource *res, dma_addr_t *pages_addr,
775 			   struct dma_fence **fence)
776 {
777 	struct amdgpu_vm_update_params params;
778 	struct amdgpu_vm_tlb_seq_cb *tlb_cb;
779 	struct amdgpu_res_cursor cursor;
780 	enum amdgpu_sync_mode sync_mode;
781 	int r, idx;
782 
783 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
784 		return -ENODEV;
785 
786 	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
787 	if (!tlb_cb) {
788 		r = -ENOMEM;
789 		goto error_unlock;
790 	}
791 
792 	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
793 	 * heavy-weight flush TLB unconditionally.
794 	 */
795 	flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
796 		     adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0);
797 
798 	/*
799 	 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
800 	 */
801 	flush_tlb |= adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 0, 0);
802 
803 	memset(&params, 0, sizeof(params));
804 	params.adev = adev;
805 	params.vm = vm;
806 	params.immediate = immediate;
807 	params.pages_addr = pages_addr;
808 	params.unlocked = unlocked;
809 
810 	/* Implicitly sync to command submissions in the same VM before
811 	 * unmapping. Sync to moving fences before mapping.
812 	 */
813 	if (!(flags & AMDGPU_PTE_VALID))
814 		sync_mode = AMDGPU_SYNC_EQ_OWNER;
815 	else
816 		sync_mode = AMDGPU_SYNC_EXPLICIT;
817 
818 	amdgpu_vm_eviction_lock(vm);
819 	if (vm->evicting) {
820 		r = -EBUSY;
821 		goto error_free;
822 	}
823 
824 	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
825 		struct dma_fence *tmp = dma_fence_get_stub();
826 
827 		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
828 		swap(vm->last_unlocked, tmp);
829 		dma_fence_put(tmp);
830 	}
831 
832 	r = vm->update_funcs->prepare(&params, resv, sync_mode);
833 	if (r)
834 		goto error_free;
835 
836 	amdgpu_res_first(pages_addr ? NULL : res, offset,
837 			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
838 	while (cursor.remaining) {
839 		uint64_t tmp, num_entries, addr;
840 
841 		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
842 		if (pages_addr) {
843 			bool contiguous = true;
844 
845 			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
846 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
847 				uint64_t count;
848 
849 				contiguous = pages_addr[pfn + 1] ==
850 					pages_addr[pfn] + PAGE_SIZE;
851 
852 				tmp = num_entries /
853 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
854 				for (count = 2; count < tmp; ++count) {
855 					uint64_t idx = pfn + count;
856 
857 					if (contiguous != (pages_addr[idx] ==
858 					    pages_addr[idx - 1] + PAGE_SIZE))
859 						break;
860 				}
861 				num_entries = count *
862 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
863 			}
864 
865 			if (!contiguous) {
866 				addr = cursor.start;
867 				params.pages_addr = pages_addr;
868 			} else {
869 				addr = pages_addr[cursor.start >> PAGE_SHIFT];
870 				params.pages_addr = NULL;
871 			}
872 
873 		} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
874 			addr = vram_base + cursor.start;
875 		} else {
876 			addr = 0;
877 		}
878 
879 		tmp = start + num_entries;
880 		r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags);
881 		if (r)
882 			goto error_free;
883 
884 		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
885 		start = tmp;
886 	}
887 
888 	r = vm->update_funcs->commit(&params, fence);
889 
890 	if (flush_tlb || params.table_freed) {
891 		tlb_cb->vm = vm;
892 		if (fence && *fence &&
893 		    !dma_fence_add_callback(*fence, &tlb_cb->cb,
894 					   amdgpu_vm_tlb_seq_cb)) {
895 			dma_fence_put(vm->last_tlb_flush);
896 			vm->last_tlb_flush = dma_fence_get(*fence);
897 		} else {
898 			amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
899 		}
900 		tlb_cb = NULL;
901 	}
902 
903 error_free:
904 	kfree(tlb_cb);
905 
906 error_unlock:
907 	amdgpu_vm_eviction_unlock(vm);
908 	drm_dev_exit(idx);
909 	return r;
910 }
911 
amdgpu_vm_get_memory(struct amdgpu_vm * vm,uint64_t * vram_mem,uint64_t * gtt_mem,uint64_t * cpu_mem)912 void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
913 				uint64_t *gtt_mem, uint64_t *cpu_mem)
914 {
915 	struct amdgpu_bo_va *bo_va, *tmp;
916 
917 	spin_lock(&vm->status_lock);
918 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
919 		if (!bo_va->base.bo)
920 			continue;
921 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
922 				gtt_mem, cpu_mem);
923 	}
924 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
925 		if (!bo_va->base.bo)
926 			continue;
927 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
928 				gtt_mem, cpu_mem);
929 	}
930 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
931 		if (!bo_va->base.bo)
932 			continue;
933 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
934 				gtt_mem, cpu_mem);
935 	}
936 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
937 		if (!bo_va->base.bo)
938 			continue;
939 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
940 				gtt_mem, cpu_mem);
941 	}
942 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
943 		if (!bo_va->base.bo)
944 			continue;
945 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
946 				gtt_mem, cpu_mem);
947 	}
948 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
949 		if (!bo_va->base.bo)
950 			continue;
951 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
952 				gtt_mem, cpu_mem);
953 	}
954 	spin_unlock(&vm->status_lock);
955 }
956 /**
957  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
958  *
959  * @adev: amdgpu_device pointer
960  * @bo_va: requested BO and VM object
961  * @clear: if true clear the entries
962  *
963  * Fill in the page table entries for @bo_va.
964  *
965  * Returns:
966  * 0 for success, -EINVAL for failure.
967  */
amdgpu_vm_bo_update(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,bool clear)968 int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
969 			bool clear)
970 {
971 	struct amdgpu_bo *bo = bo_va->base.bo;
972 	struct amdgpu_vm *vm = bo_va->base.vm;
973 	struct amdgpu_bo_va_mapping *mapping;
974 	dma_addr_t *pages_addr = NULL;
975 	struct ttm_resource *mem;
976 	struct dma_fence **last_update;
977 	bool flush_tlb = clear;
978 	struct dma_resv *resv;
979 	uint64_t vram_base;
980 	uint64_t flags;
981 	int r;
982 
983 	if (clear || !bo) {
984 		mem = NULL;
985 		resv = vm->root.bo->tbo.base.resv;
986 	} else {
987 		struct drm_gem_object *obj = &bo->tbo.base;
988 
989 		resv = bo->tbo.base.resv;
990 		if (obj->import_attach && bo_va->is_xgmi) {
991 			struct dma_buf *dma_buf = obj->import_attach->dmabuf;
992 			struct drm_gem_object *gobj = dma_buf->priv;
993 			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
994 
995 			if (abo->tbo.resource->mem_type == TTM_PL_VRAM)
996 				bo = gem_to_amdgpu_bo(gobj);
997 		}
998 		mem = bo->tbo.resource;
999 		if (mem->mem_type == TTM_PL_TT ||
1000 		    mem->mem_type == AMDGPU_PL_PREEMPT)
1001 			pages_addr = bo->tbo.ttm->dma_address;
1002 	}
1003 
1004 	if (bo) {
1005 		struct amdgpu_device *bo_adev;
1006 
1007 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1008 
1009 		if (amdgpu_bo_encrypted(bo))
1010 			flags |= AMDGPU_PTE_TMZ;
1011 
1012 		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1013 		vram_base = bo_adev->vm_manager.vram_base_offset;
1014 	} else {
1015 		flags = 0x0;
1016 		vram_base = 0;
1017 	}
1018 
1019 	if (clear || (bo && bo->tbo.base.resv ==
1020 		      vm->root.bo->tbo.base.resv))
1021 		last_update = &vm->last_update;
1022 	else
1023 		last_update = &bo_va->last_pt_update;
1024 
1025 	if (!clear && bo_va->base.moved) {
1026 		flush_tlb = true;
1027 		list_splice_init(&bo_va->valids, &bo_va->invalids);
1028 
1029 	} else if (bo_va->cleared != clear) {
1030 		list_splice_init(&bo_va->valids, &bo_va->invalids);
1031 	}
1032 
1033 	list_for_each_entry(mapping, &bo_va->invalids, list) {
1034 		uint64_t update_flags = flags;
1035 
1036 		/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1037 		 * but in case of something, we filter the flags in first place
1038 		 */
1039 		if (!(mapping->flags & AMDGPU_PTE_READABLE))
1040 			update_flags &= ~AMDGPU_PTE_READABLE;
1041 		if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1042 			update_flags &= ~AMDGPU_PTE_WRITEABLE;
1043 
1044 		/* Apply ASIC specific mapping flags */
1045 		amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags);
1046 
1047 		trace_amdgpu_vm_bo_update(mapping);
1048 
1049 		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
1050 					   resv, mapping->start, mapping->last,
1051 					   update_flags, mapping->offset,
1052 					   vram_base, mem, pages_addr,
1053 					   last_update);
1054 		if (r)
1055 			return r;
1056 	}
1057 
1058 	/* If the BO is not in its preferred location add it back to
1059 	 * the evicted list so that it gets validated again on the
1060 	 * next command submission.
1061 	 */
1062 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1063 		uint32_t mem_type = bo->tbo.resource->mem_type;
1064 
1065 		if (!(bo->preferred_domains &
1066 		      amdgpu_mem_type_to_domain(mem_type)))
1067 			amdgpu_vm_bo_evicted(&bo_va->base);
1068 		else
1069 			amdgpu_vm_bo_idle(&bo_va->base);
1070 	} else {
1071 		amdgpu_vm_bo_done(&bo_va->base);
1072 	}
1073 
1074 	list_splice_init(&bo_va->invalids, &bo_va->valids);
1075 	bo_va->cleared = clear;
1076 	bo_va->base.moved = false;
1077 
1078 	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1079 		list_for_each_entry(mapping, &bo_va->valids, list)
1080 			trace_amdgpu_vm_bo_mapping(mapping);
1081 	}
1082 
1083 	return 0;
1084 }
1085 
1086 /**
1087  * amdgpu_vm_update_prt_state - update the global PRT state
1088  *
1089  * @adev: amdgpu_device pointer
1090  */
amdgpu_vm_update_prt_state(struct amdgpu_device * adev)1091 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1092 {
1093 	unsigned long flags;
1094 	bool enable;
1095 
1096 	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1097 	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1098 	adev->gmc.gmc_funcs->set_prt(adev, enable);
1099 	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1100 }
1101 
1102 /**
1103  * amdgpu_vm_prt_get - add a PRT user
1104  *
1105  * @adev: amdgpu_device pointer
1106  */
amdgpu_vm_prt_get(struct amdgpu_device * adev)1107 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1108 {
1109 	if (!adev->gmc.gmc_funcs->set_prt)
1110 		return;
1111 
1112 	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1113 		amdgpu_vm_update_prt_state(adev);
1114 }
1115 
1116 /**
1117  * amdgpu_vm_prt_put - drop a PRT user
1118  *
1119  * @adev: amdgpu_device pointer
1120  */
amdgpu_vm_prt_put(struct amdgpu_device * adev)1121 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1122 {
1123 	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1124 		amdgpu_vm_update_prt_state(adev);
1125 }
1126 
1127 /**
1128  * amdgpu_vm_prt_cb - callback for updating the PRT status
1129  *
1130  * @fence: fence for the callback
1131  * @_cb: the callback function
1132  */
amdgpu_vm_prt_cb(struct dma_fence * fence,struct dma_fence_cb * _cb)1133 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1134 {
1135 	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1136 
1137 	amdgpu_vm_prt_put(cb->adev);
1138 	kfree(cb);
1139 }
1140 
1141 /**
1142  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1143  *
1144  * @adev: amdgpu_device pointer
1145  * @fence: fence for the callback
1146  */
amdgpu_vm_add_prt_cb(struct amdgpu_device * adev,struct dma_fence * fence)1147 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1148 				 struct dma_fence *fence)
1149 {
1150 	struct amdgpu_prt_cb *cb;
1151 
1152 	if (!adev->gmc.gmc_funcs->set_prt)
1153 		return;
1154 
1155 	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1156 	if (!cb) {
1157 		/* Last resort when we are OOM */
1158 		if (fence)
1159 			dma_fence_wait(fence, false);
1160 
1161 		amdgpu_vm_prt_put(adev);
1162 	} else {
1163 		cb->adev = adev;
1164 		if (!fence || dma_fence_add_callback(fence, &cb->cb,
1165 						     amdgpu_vm_prt_cb))
1166 			amdgpu_vm_prt_cb(fence, &cb->cb);
1167 	}
1168 }
1169 
1170 /**
1171  * amdgpu_vm_free_mapping - free a mapping
1172  *
1173  * @adev: amdgpu_device pointer
1174  * @vm: requested vm
1175  * @mapping: mapping to be freed
1176  * @fence: fence of the unmap operation
1177  *
1178  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1179  */
amdgpu_vm_free_mapping(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo_va_mapping * mapping,struct dma_fence * fence)1180 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1181 				   struct amdgpu_vm *vm,
1182 				   struct amdgpu_bo_va_mapping *mapping,
1183 				   struct dma_fence *fence)
1184 {
1185 	if (mapping->flags & AMDGPU_PTE_PRT)
1186 		amdgpu_vm_add_prt_cb(adev, fence);
1187 	kfree(mapping);
1188 }
1189 
1190 /**
1191  * amdgpu_vm_prt_fini - finish all prt mappings
1192  *
1193  * @adev: amdgpu_device pointer
1194  * @vm: requested vm
1195  *
1196  * Register a cleanup callback to disable PRT support after VM dies.
1197  */
amdgpu_vm_prt_fini(struct amdgpu_device * adev,struct amdgpu_vm * vm)1198 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1199 {
1200 	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1201 	struct dma_resv_iter cursor;
1202 	struct dma_fence *fence;
1203 
1204 	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1205 		/* Add a callback for each fence in the reservation object */
1206 		amdgpu_vm_prt_get(adev);
1207 		amdgpu_vm_add_prt_cb(adev, fence);
1208 	}
1209 }
1210 
1211 /**
1212  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1213  *
1214  * @adev: amdgpu_device pointer
1215  * @vm: requested vm
1216  * @fence: optional resulting fence (unchanged if no work needed to be done
1217  * or if an error occurred)
1218  *
1219  * Make sure all freed BOs are cleared in the PT.
1220  * PTs have to be reserved and mutex must be locked!
1221  *
1222  * Returns:
1223  * 0 for success.
1224  *
1225  */
amdgpu_vm_clear_freed(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct dma_fence ** fence)1226 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1227 			  struct amdgpu_vm *vm,
1228 			  struct dma_fence **fence)
1229 {
1230 	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1231 	struct amdgpu_bo_va_mapping *mapping;
1232 	uint64_t init_pte_value = 0;
1233 	struct dma_fence *f = NULL;
1234 	int r;
1235 
1236 	while (!list_empty(&vm->freed)) {
1237 		mapping = list_first_entry(&vm->freed,
1238 			struct amdgpu_bo_va_mapping, list);
1239 		list_del(&mapping->list);
1240 
1241 		if (vm->pte_support_ats &&
1242 		    mapping->start < AMDGPU_GMC_HOLE_START)
1243 			init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1244 
1245 		r = amdgpu_vm_update_range(adev, vm, false, false, true, resv,
1246 					   mapping->start, mapping->last,
1247 					   init_pte_value, 0, 0, NULL, NULL,
1248 					   &f);
1249 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
1250 		if (r) {
1251 			dma_fence_put(f);
1252 			return r;
1253 		}
1254 	}
1255 
1256 	if (fence && f) {
1257 		dma_fence_put(*fence);
1258 		*fence = f;
1259 	} else {
1260 		dma_fence_put(f);
1261 	}
1262 
1263 	return 0;
1264 
1265 }
1266 
1267 /**
1268  * amdgpu_vm_handle_moved - handle moved BOs in the PT
1269  *
1270  * @adev: amdgpu_device pointer
1271  * @vm: requested vm
1272  *
1273  * Make sure all BOs which are moved are updated in the PTs.
1274  *
1275  * Returns:
1276  * 0 for success.
1277  *
1278  * PTs have to be reserved!
1279  */
amdgpu_vm_handle_moved(struct amdgpu_device * adev,struct amdgpu_vm * vm)1280 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1281 			   struct amdgpu_vm *vm)
1282 {
1283 	struct amdgpu_bo_va *bo_va;
1284 	struct dma_resv *resv;
1285 	bool clear;
1286 	int r;
1287 
1288 	spin_lock(&vm->status_lock);
1289 	while (!list_empty(&vm->moved)) {
1290 		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
1291 					 base.vm_status);
1292 		spin_unlock(&vm->status_lock);
1293 
1294 		/* Per VM BOs never need to bo cleared in the page tables */
1295 		r = amdgpu_vm_bo_update(adev, bo_va, false);
1296 		if (r)
1297 			return r;
1298 		spin_lock(&vm->status_lock);
1299 	}
1300 
1301 	while (!list_empty(&vm->invalidated)) {
1302 		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1303 					 base.vm_status);
1304 		resv = bo_va->base.bo->tbo.base.resv;
1305 		spin_unlock(&vm->status_lock);
1306 
1307 		/* Try to reserve the BO to avoid clearing its ptes */
1308 		if (!amdgpu_vm_debug && dma_resv_trylock(resv))
1309 			clear = false;
1310 		/* Somebody else is using the BO right now */
1311 		else
1312 			clear = true;
1313 
1314 		r = amdgpu_vm_bo_update(adev, bo_va, clear);
1315 		if (r)
1316 			return r;
1317 
1318 		if (!clear)
1319 			dma_resv_unlock(resv);
1320 		spin_lock(&vm->status_lock);
1321 	}
1322 	spin_unlock(&vm->status_lock);
1323 
1324 	return 0;
1325 }
1326 
1327 /**
1328  * amdgpu_vm_bo_add - add a bo to a specific vm
1329  *
1330  * @adev: amdgpu_device pointer
1331  * @vm: requested vm
1332  * @bo: amdgpu buffer object
1333  *
1334  * Add @bo into the requested vm.
1335  * Add @bo to the list of bos associated with the vm
1336  *
1337  * Returns:
1338  * Newly added bo_va or NULL for failure
1339  *
1340  * Object has to be reserved!
1341  */
amdgpu_vm_bo_add(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo * bo)1342 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1343 				      struct amdgpu_vm *vm,
1344 				      struct amdgpu_bo *bo)
1345 {
1346 	struct amdgpu_bo_va *bo_va;
1347 
1348 	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1349 	if (bo_va == NULL) {
1350 		return NULL;
1351 	}
1352 	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
1353 
1354 	bo_va->ref_count = 1;
1355 	INIT_LIST_HEAD(&bo_va->valids);
1356 	INIT_LIST_HEAD(&bo_va->invalids);
1357 
1358 	if (!bo)
1359 		return bo_va;
1360 
1361 	dma_resv_assert_held(bo->tbo.base.resv);
1362 	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1363 		bo_va->is_xgmi = true;
1364 		/* Power up XGMI if it can be potentially used */
1365 		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1366 	}
1367 
1368 	return bo_va;
1369 }
1370 
1371 
1372 /**
1373  * amdgpu_vm_bo_insert_map - insert a new mapping
1374  *
1375  * @adev: amdgpu_device pointer
1376  * @bo_va: bo_va to store the address
1377  * @mapping: the mapping to insert
1378  *
1379  * Insert a new mapping into all structures.
1380  */
amdgpu_vm_bo_insert_map(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,struct amdgpu_bo_va_mapping * mapping)1381 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1382 				    struct amdgpu_bo_va *bo_va,
1383 				    struct amdgpu_bo_va_mapping *mapping)
1384 {
1385 	struct amdgpu_vm *vm = bo_va->base.vm;
1386 	struct amdgpu_bo *bo = bo_va->base.bo;
1387 
1388 	mapping->bo_va = bo_va;
1389 	list_add(&mapping->list, &bo_va->invalids);
1390 	amdgpu_vm_it_insert(mapping, &vm->va);
1391 
1392 	if (mapping->flags & AMDGPU_PTE_PRT)
1393 		amdgpu_vm_prt_get(adev);
1394 
1395 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
1396 	    !bo_va->base.moved) {
1397 		amdgpu_vm_bo_moved(&bo_va->base);
1398 	}
1399 	trace_amdgpu_vm_bo_map(bo_va, mapping);
1400 }
1401 
1402 /**
1403  * amdgpu_vm_bo_map - map bo inside a vm
1404  *
1405  * @adev: amdgpu_device pointer
1406  * @bo_va: bo_va to store the address
1407  * @saddr: where to map the BO
1408  * @offset: requested offset in the BO
1409  * @size: BO size in bytes
1410  * @flags: attributes of pages (read/write/valid/etc.)
1411  *
1412  * Add a mapping of the BO at the specefied addr into the VM.
1413  *
1414  * Returns:
1415  * 0 for success, error for failure.
1416  *
1417  * Object has to be reserved and unreserved outside!
1418  */
amdgpu_vm_bo_map(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,uint64_t saddr,uint64_t offset,uint64_t size,uint64_t flags)1419 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1420 		     struct amdgpu_bo_va *bo_va,
1421 		     uint64_t saddr, uint64_t offset,
1422 		     uint64_t size, uint64_t flags)
1423 {
1424 	struct amdgpu_bo_va_mapping *mapping, *tmp;
1425 	struct amdgpu_bo *bo = bo_va->base.bo;
1426 	struct amdgpu_vm *vm = bo_va->base.vm;
1427 	uint64_t eaddr;
1428 
1429 	/* validate the parameters */
1430 	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
1431 	    size == 0 || size & ~PAGE_MASK)
1432 		return -EINVAL;
1433 
1434 	/* make sure object fit at this offset */
1435 	eaddr = saddr + size - 1;
1436 	if (saddr >= eaddr ||
1437 	    (bo && offset + size > amdgpu_bo_size(bo)) ||
1438 	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
1439 		return -EINVAL;
1440 
1441 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1442 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1443 
1444 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1445 	if (tmp) {
1446 		/* bo and tmp overlap, invalid addr */
1447 		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1448 			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1449 			tmp->start, tmp->last + 1);
1450 		return -EINVAL;
1451 	}
1452 
1453 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1454 	if (!mapping)
1455 		return -ENOMEM;
1456 
1457 	mapping->start = saddr;
1458 	mapping->last = eaddr;
1459 	mapping->offset = offset;
1460 	mapping->flags = flags;
1461 
1462 	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1463 
1464 	return 0;
1465 }
1466 
1467 /**
1468  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1469  *
1470  * @adev: amdgpu_device pointer
1471  * @bo_va: bo_va to store the address
1472  * @saddr: where to map the BO
1473  * @offset: requested offset in the BO
1474  * @size: BO size in bytes
1475  * @flags: attributes of pages (read/write/valid/etc.)
1476  *
1477  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1478  * mappings as we do so.
1479  *
1480  * Returns:
1481  * 0 for success, error for failure.
1482  *
1483  * Object has to be reserved and unreserved outside!
1484  */
amdgpu_vm_bo_replace_map(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,uint64_t saddr,uint64_t offset,uint64_t size,uint64_t flags)1485 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1486 			     struct amdgpu_bo_va *bo_va,
1487 			     uint64_t saddr, uint64_t offset,
1488 			     uint64_t size, uint64_t flags)
1489 {
1490 	struct amdgpu_bo_va_mapping *mapping;
1491 	struct amdgpu_bo *bo = bo_va->base.bo;
1492 	uint64_t eaddr;
1493 	int r;
1494 
1495 	/* validate the parameters */
1496 	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
1497 	    size == 0 || size & ~PAGE_MASK)
1498 		return -EINVAL;
1499 
1500 	/* make sure object fit at this offset */
1501 	eaddr = saddr + size - 1;
1502 	if (saddr >= eaddr ||
1503 	    (bo && offset + size > amdgpu_bo_size(bo)) ||
1504 	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
1505 		return -EINVAL;
1506 
1507 	/* Allocate all the needed memory */
1508 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1509 	if (!mapping)
1510 		return -ENOMEM;
1511 
1512 	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1513 	if (r) {
1514 		kfree(mapping);
1515 		return r;
1516 	}
1517 
1518 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1519 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1520 
1521 	mapping->start = saddr;
1522 	mapping->last = eaddr;
1523 	mapping->offset = offset;
1524 	mapping->flags = flags;
1525 
1526 	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1527 
1528 	return 0;
1529 }
1530 
1531 /**
1532  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1533  *
1534  * @adev: amdgpu_device pointer
1535  * @bo_va: bo_va to remove the address from
1536  * @saddr: where to the BO is mapped
1537  *
1538  * Remove a mapping of the BO at the specefied addr from the VM.
1539  *
1540  * Returns:
1541  * 0 for success, error for failure.
1542  *
1543  * Object has to be reserved and unreserved outside!
1544  */
amdgpu_vm_bo_unmap(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,uint64_t saddr)1545 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1546 		       struct amdgpu_bo_va *bo_va,
1547 		       uint64_t saddr)
1548 {
1549 	struct amdgpu_bo_va_mapping *mapping;
1550 	struct amdgpu_vm *vm = bo_va->base.vm;
1551 	bool valid = true;
1552 
1553 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1554 
1555 	list_for_each_entry(mapping, &bo_va->valids, list) {
1556 		if (mapping->start == saddr)
1557 			break;
1558 	}
1559 
1560 	if (&mapping->list == &bo_va->valids) {
1561 		valid = false;
1562 
1563 		list_for_each_entry(mapping, &bo_va->invalids, list) {
1564 			if (mapping->start == saddr)
1565 				break;
1566 		}
1567 
1568 		if (&mapping->list == &bo_va->invalids)
1569 			return -ENOENT;
1570 	}
1571 
1572 	list_del(&mapping->list);
1573 	amdgpu_vm_it_remove(mapping, &vm->va);
1574 	mapping->bo_va = NULL;
1575 	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1576 
1577 	if (valid)
1578 		list_add(&mapping->list, &vm->freed);
1579 	else
1580 		amdgpu_vm_free_mapping(adev, vm, mapping,
1581 				       bo_va->last_pt_update);
1582 
1583 	return 0;
1584 }
1585 
1586 /**
1587  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
1588  *
1589  * @adev: amdgpu_device pointer
1590  * @vm: VM structure to use
1591  * @saddr: start of the range
1592  * @size: size of the range
1593  *
1594  * Remove all mappings in a range, split them as appropriate.
1595  *
1596  * Returns:
1597  * 0 for success, error for failure.
1598  */
amdgpu_vm_bo_clear_mappings(struct amdgpu_device * adev,struct amdgpu_vm * vm,uint64_t saddr,uint64_t size)1599 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1600 				struct amdgpu_vm *vm,
1601 				uint64_t saddr, uint64_t size)
1602 {
1603 	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
1604 	LIST_HEAD(removed);
1605 	uint64_t eaddr;
1606 
1607 	eaddr = saddr + size - 1;
1608 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1609 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1610 
1611 	/* Allocate all the needed memory */
1612 	before = kzalloc(sizeof(*before), GFP_KERNEL);
1613 	if (!before)
1614 		return -ENOMEM;
1615 	INIT_LIST_HEAD(&before->list);
1616 
1617 	after = kzalloc(sizeof(*after), GFP_KERNEL);
1618 	if (!after) {
1619 		kfree(before);
1620 		return -ENOMEM;
1621 	}
1622 	INIT_LIST_HEAD(&after->list);
1623 
1624 	/* Now gather all removed mappings */
1625 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1626 	while (tmp) {
1627 		/* Remember mapping split at the start */
1628 		if (tmp->start < saddr) {
1629 			before->start = tmp->start;
1630 			before->last = saddr - 1;
1631 			before->offset = tmp->offset;
1632 			before->flags = tmp->flags;
1633 			before->bo_va = tmp->bo_va;
1634 			list_add(&before->list, &tmp->bo_va->invalids);
1635 		}
1636 
1637 		/* Remember mapping split at the end */
1638 		if (tmp->last > eaddr) {
1639 			after->start = eaddr + 1;
1640 			after->last = tmp->last;
1641 			after->offset = tmp->offset;
1642 			after->offset += (after->start - tmp->start) << PAGE_SHIFT;
1643 			after->flags = tmp->flags;
1644 			after->bo_va = tmp->bo_va;
1645 			list_add(&after->list, &tmp->bo_va->invalids);
1646 		}
1647 
1648 		list_del(&tmp->list);
1649 		list_add(&tmp->list, &removed);
1650 
1651 		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
1652 	}
1653 
1654 	/* And free them up */
1655 	list_for_each_entry_safe(tmp, next, &removed, list) {
1656 		amdgpu_vm_it_remove(tmp, &vm->va);
1657 		list_del(&tmp->list);
1658 
1659 		if (tmp->start < saddr)
1660 		    tmp->start = saddr;
1661 		if (tmp->last > eaddr)
1662 		    tmp->last = eaddr;
1663 
1664 		tmp->bo_va = NULL;
1665 		list_add(&tmp->list, &vm->freed);
1666 		trace_amdgpu_vm_bo_unmap(NULL, tmp);
1667 	}
1668 
1669 	/* Insert partial mapping before the range */
1670 	if (!list_empty(&before->list)) {
1671 		amdgpu_vm_it_insert(before, &vm->va);
1672 		if (before->flags & AMDGPU_PTE_PRT)
1673 			amdgpu_vm_prt_get(adev);
1674 	} else {
1675 		kfree(before);
1676 	}
1677 
1678 	/* Insert partial mapping after the range */
1679 	if (!list_empty(&after->list)) {
1680 		amdgpu_vm_it_insert(after, &vm->va);
1681 		if (after->flags & AMDGPU_PTE_PRT)
1682 			amdgpu_vm_prt_get(adev);
1683 	} else {
1684 		kfree(after);
1685 	}
1686 
1687 	return 0;
1688 }
1689 
1690 /**
1691  * amdgpu_vm_bo_lookup_mapping - find mapping by address
1692  *
1693  * @vm: the requested VM
1694  * @addr: the address
1695  *
1696  * Find a mapping by it's address.
1697  *
1698  * Returns:
1699  * The amdgpu_bo_va_mapping matching for addr or NULL
1700  *
1701  */
amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm * vm,uint64_t addr)1702 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
1703 							 uint64_t addr)
1704 {
1705 	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
1706 }
1707 
1708 /**
1709  * amdgpu_vm_bo_trace_cs - trace all reserved mappings
1710  *
1711  * @vm: the requested vm
1712  * @ticket: CS ticket
1713  *
1714  * Trace all mappings of BOs reserved during a command submission.
1715  */
amdgpu_vm_bo_trace_cs(struct amdgpu_vm * vm,struct ww_acquire_ctx * ticket)1716 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
1717 {
1718 	struct amdgpu_bo_va_mapping *mapping;
1719 
1720 	if (!trace_amdgpu_vm_bo_cs_enabled())
1721 		return;
1722 
1723 	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
1724 	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
1725 		if (mapping->bo_va && mapping->bo_va->base.bo) {
1726 			struct amdgpu_bo *bo;
1727 
1728 			bo = mapping->bo_va->base.bo;
1729 			if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
1730 			    ticket)
1731 				continue;
1732 		}
1733 
1734 		trace_amdgpu_vm_bo_cs(mapping);
1735 	}
1736 }
1737 
1738 /**
1739  * amdgpu_vm_bo_del - remove a bo from a specific vm
1740  *
1741  * @adev: amdgpu_device pointer
1742  * @bo_va: requested bo_va
1743  *
1744  * Remove @bo_va->bo from the requested vm.
1745  *
1746  * Object have to be reserved!
1747  */
amdgpu_vm_bo_del(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va)1748 void amdgpu_vm_bo_del(struct amdgpu_device *adev,
1749 		      struct amdgpu_bo_va *bo_va)
1750 {
1751 	struct amdgpu_bo_va_mapping *mapping, *next;
1752 	struct amdgpu_bo *bo = bo_va->base.bo;
1753 	struct amdgpu_vm *vm = bo_va->base.vm;
1754 	struct amdgpu_vm_bo_base **base;
1755 
1756 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
1757 
1758 	if (bo) {
1759 		dma_resv_assert_held(bo->tbo.base.resv);
1760 		if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1761 			ttm_bo_set_bulk_move(&bo->tbo, NULL);
1762 
1763 		for (base = &bo_va->base.bo->vm_bo; *base;
1764 		     base = &(*base)->next) {
1765 			if (*base != &bo_va->base)
1766 				continue;
1767 
1768 			*base = bo_va->base.next;
1769 			break;
1770 		}
1771 	}
1772 
1773 	spin_lock(&vm->status_lock);
1774 	list_del(&bo_va->base.vm_status);
1775 	spin_unlock(&vm->status_lock);
1776 
1777 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1778 		list_del(&mapping->list);
1779 		amdgpu_vm_it_remove(mapping, &vm->va);
1780 		mapping->bo_va = NULL;
1781 		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1782 		list_add(&mapping->list, &vm->freed);
1783 	}
1784 	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1785 		list_del(&mapping->list);
1786 		amdgpu_vm_it_remove(mapping, &vm->va);
1787 		amdgpu_vm_free_mapping(adev, vm, mapping,
1788 				       bo_va->last_pt_update);
1789 	}
1790 
1791 	dma_fence_put(bo_va->last_pt_update);
1792 
1793 	if (bo && bo_va->is_xgmi)
1794 		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
1795 
1796 	kfree(bo_va);
1797 }
1798 
1799 /**
1800  * amdgpu_vm_evictable - check if we can evict a VM
1801  *
1802  * @bo: A page table of the VM.
1803  *
1804  * Check if it is possible to evict a VM.
1805  */
amdgpu_vm_evictable(struct amdgpu_bo * bo)1806 bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
1807 {
1808 	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
1809 
1810 	/* Page tables of a destroyed VM can go away immediately */
1811 	if (!bo_base || !bo_base->vm)
1812 		return true;
1813 
1814 	/* Don't evict VM page tables while they are busy */
1815 	if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))
1816 		return false;
1817 
1818 	/* Try to block ongoing updates */
1819 	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
1820 		return false;
1821 
1822 	/* Don't evict VM page tables while they are updated */
1823 	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
1824 		amdgpu_vm_eviction_unlock(bo_base->vm);
1825 		return false;
1826 	}
1827 
1828 	bo_base->vm->evicting = true;
1829 	amdgpu_vm_eviction_unlock(bo_base->vm);
1830 	return true;
1831 }
1832 
1833 /**
1834  * amdgpu_vm_bo_invalidate - mark the bo as invalid
1835  *
1836  * @adev: amdgpu_device pointer
1837  * @bo: amdgpu buffer object
1838  * @evicted: is the BO evicted
1839  *
1840  * Mark @bo as invalid.
1841  */
amdgpu_vm_bo_invalidate(struct amdgpu_device * adev,struct amdgpu_bo * bo,bool evicted)1842 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1843 			     struct amdgpu_bo *bo, bool evicted)
1844 {
1845 	struct amdgpu_vm_bo_base *bo_base;
1846 
1847 	/* shadow bo doesn't have bo base, its validation needs its parent */
1848 	if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo))
1849 		bo = bo->parent;
1850 
1851 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
1852 		struct amdgpu_vm *vm = bo_base->vm;
1853 
1854 		if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1855 			amdgpu_vm_bo_evicted(bo_base);
1856 			continue;
1857 		}
1858 
1859 		if (bo_base->moved)
1860 			continue;
1861 		bo_base->moved = true;
1862 
1863 		if (bo->tbo.type == ttm_bo_type_kernel)
1864 			amdgpu_vm_bo_relocated(bo_base);
1865 		else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1866 			amdgpu_vm_bo_moved(bo_base);
1867 		else
1868 			amdgpu_vm_bo_invalidated(bo_base);
1869 	}
1870 }
1871 
1872 /**
1873  * amdgpu_vm_get_block_size - calculate VM page table size as power of two
1874  *
1875  * @vm_size: VM size
1876  *
1877  * Returns:
1878  * VM page table as power of two
1879  */
amdgpu_vm_get_block_size(uint64_t vm_size)1880 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
1881 {
1882 	/* Total bits covered by PD + PTs */
1883 	unsigned bits = ilog2(vm_size) + 18;
1884 
1885 	/* Make sure the PD is 4K in size up to 8GB address space.
1886 	   Above that split equal between PD and PTs */
1887 	if (vm_size <= 8)
1888 		return (bits - 9);
1889 	else
1890 		return ((bits + 3) / 2);
1891 }
1892 
1893 /**
1894  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
1895  *
1896  * @adev: amdgpu_device pointer
1897  * @min_vm_size: the minimum vm size in GB if it's set auto
1898  * @fragment_size_default: Default PTE fragment size
1899  * @max_level: max VMPT level
1900  * @max_bits: max address space size in bits
1901  *
1902  */
amdgpu_vm_adjust_size(struct amdgpu_device * adev,uint32_t min_vm_size,uint32_t fragment_size_default,unsigned max_level,unsigned max_bits)1903 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
1904 			   uint32_t fragment_size_default, unsigned max_level,
1905 			   unsigned max_bits)
1906 {
1907 	unsigned int max_size = 1 << (max_bits - 30);
1908 	unsigned int vm_size;
1909 	uint64_t tmp;
1910 
1911 	/* adjust vm size first */
1912 	if (amdgpu_vm_size != -1) {
1913 		vm_size = amdgpu_vm_size;
1914 		if (vm_size > max_size) {
1915 			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
1916 				 amdgpu_vm_size, max_size);
1917 			vm_size = max_size;
1918 		}
1919 	} else {
1920 		struct sysinfo si;
1921 		unsigned int phys_ram_gb;
1922 
1923 		/* Optimal VM size depends on the amount of physical
1924 		 * RAM available. Underlying requirements and
1925 		 * assumptions:
1926 		 *
1927 		 *  - Need to map system memory and VRAM from all GPUs
1928 		 *     - VRAM from other GPUs not known here
1929 		 *     - Assume VRAM <= system memory
1930 		 *  - On GFX8 and older, VM space can be segmented for
1931 		 *    different MTYPEs
1932 		 *  - Need to allow room for fragmentation, guard pages etc.
1933 		 *
1934 		 * This adds up to a rough guess of system memory x3.
1935 		 * Round up to power of two to maximize the available
1936 		 * VM size with the given page table size.
1937 		 */
1938 		si_meminfo(&si);
1939 		phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
1940 			       (1 << 30) - 1) >> 30;
1941 		vm_size = roundup_pow_of_two(
1942 			min(max(phys_ram_gb * 3, min_vm_size), max_size));
1943 	}
1944 
1945 	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
1946 
1947 	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
1948 	if (amdgpu_vm_block_size != -1)
1949 		tmp >>= amdgpu_vm_block_size - 9;
1950 	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
1951 	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
1952 	switch (adev->vm_manager.num_level) {
1953 	case 3:
1954 		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
1955 		break;
1956 	case 2:
1957 		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
1958 		break;
1959 	case 1:
1960 		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
1961 		break;
1962 	default:
1963 		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
1964 	}
1965 	/* block size depends on vm size and hw setup*/
1966 	if (amdgpu_vm_block_size != -1)
1967 		adev->vm_manager.block_size =
1968 			min((unsigned)amdgpu_vm_block_size, max_bits
1969 			    - AMDGPU_GPU_PAGE_SHIFT
1970 			    - 9 * adev->vm_manager.num_level);
1971 	else if (adev->vm_manager.num_level > 1)
1972 		adev->vm_manager.block_size = 9;
1973 	else
1974 		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
1975 
1976 	if (amdgpu_vm_fragment_size == -1)
1977 		adev->vm_manager.fragment_size = fragment_size_default;
1978 	else
1979 		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
1980 
1981 	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
1982 		 vm_size, adev->vm_manager.num_level + 1,
1983 		 adev->vm_manager.block_size,
1984 		 adev->vm_manager.fragment_size);
1985 }
1986 
1987 /**
1988  * amdgpu_vm_wait_idle - wait for the VM to become idle
1989  *
1990  * @vm: VM object to wait for
1991  * @timeout: timeout to wait for VM to become idle
1992  */
amdgpu_vm_wait_idle(struct amdgpu_vm * vm,long timeout)1993 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
1994 {
1995 	timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
1996 					DMA_RESV_USAGE_BOOKKEEP,
1997 					true, timeout);
1998 	if (timeout <= 0)
1999 		return timeout;
2000 
2001 	return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
2002 }
2003 
2004 /**
2005  * amdgpu_vm_init - initialize a vm instance
2006  *
2007  * @adev: amdgpu_device pointer
2008  * @vm: requested vm
2009  *
2010  * Init @vm fields.
2011  *
2012  * Returns:
2013  * 0 for success, error for failure.
2014  */
amdgpu_vm_init(struct amdgpu_device * adev,struct amdgpu_vm * vm)2015 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2016 {
2017 	struct amdgpu_bo *root_bo;
2018 	struct amdgpu_bo_vm *root;
2019 	int r, i;
2020 
2021 	vm->va = RB_ROOT_CACHED;
2022 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2023 		vm->reserved_vmid[i] = NULL;
2024 	INIT_LIST_HEAD(&vm->evicted);
2025 	INIT_LIST_HEAD(&vm->relocated);
2026 	INIT_LIST_HEAD(&vm->moved);
2027 	INIT_LIST_HEAD(&vm->idle);
2028 	INIT_LIST_HEAD(&vm->invalidated);
2029 	spin_lock_init(&vm->status_lock);
2030 	INIT_LIST_HEAD(&vm->freed);
2031 	INIT_LIST_HEAD(&vm->done);
2032 	INIT_LIST_HEAD(&vm->pt_freed);
2033 	INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work);
2034 
2035 	/* create scheduler entities for page table updates */
2036 	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
2037 				  adev->vm_manager.vm_pte_scheds,
2038 				  adev->vm_manager.vm_pte_num_scheds, NULL);
2039 	if (r)
2040 		return r;
2041 
2042 	r = drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
2043 				  adev->vm_manager.vm_pte_scheds,
2044 				  adev->vm_manager.vm_pte_num_scheds, NULL);
2045 	if (r)
2046 		goto error_free_immediate;
2047 
2048 	vm->pte_support_ats = false;
2049 	vm->is_compute_context = false;
2050 
2051 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2052 				    AMDGPU_VM_USE_CPU_FOR_GFX);
2053 
2054 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2055 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2056 	WARN_ONCE((vm->use_cpu_for_update &&
2057 		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2058 		  "CPU update of VM recommended only for large BAR system\n");
2059 
2060 	if (vm->use_cpu_for_update)
2061 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2062 	else
2063 		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2064 	vm->last_update = NULL;
2065 	vm->last_unlocked = dma_fence_get_stub();
2066 	vm->last_tlb_flush = dma_fence_get_stub();
2067 
2068 	mutex_init(&vm->eviction_lock);
2069 	vm->evicting = false;
2070 
2071 	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2072 				false, &root);
2073 	if (r)
2074 		goto error_free_delayed;
2075 	root_bo = &root->bo;
2076 	r = amdgpu_bo_reserve(root_bo, true);
2077 	if (r)
2078 		goto error_free_root;
2079 
2080 	r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);
2081 	if (r)
2082 		goto error_unreserve;
2083 
2084 	amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
2085 
2086 	r = amdgpu_vm_pt_clear(adev, vm, root, false);
2087 	if (r)
2088 		goto error_unreserve;
2089 
2090 	amdgpu_bo_unreserve(vm->root.bo);
2091 
2092 	INIT_KFIFO(vm->faults);
2093 
2094 	return 0;
2095 
2096 error_unreserve:
2097 	amdgpu_bo_unreserve(vm->root.bo);
2098 
2099 error_free_root:
2100 	amdgpu_bo_unref(&root->shadow);
2101 	amdgpu_bo_unref(&root_bo);
2102 	vm->root.bo = NULL;
2103 
2104 error_free_delayed:
2105 	dma_fence_put(vm->last_tlb_flush);
2106 	dma_fence_put(vm->last_unlocked);
2107 	drm_sched_entity_destroy(&vm->delayed);
2108 
2109 error_free_immediate:
2110 	drm_sched_entity_destroy(&vm->immediate);
2111 
2112 	return r;
2113 }
2114 
2115 /**
2116  * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2117  *
2118  * @adev: amdgpu_device pointer
2119  * @vm: requested vm
2120  *
2121  * This only works on GFX VMs that don't have any BOs added and no
2122  * page tables allocated yet.
2123  *
2124  * Changes the following VM parameters:
2125  * - use_cpu_for_update
2126  * - pte_supports_ats
2127  *
2128  * Reinitializes the page directory to reflect the changed ATS
2129  * setting.
2130  *
2131  * Returns:
2132  * 0 for success, -errno for errors.
2133  */
amdgpu_vm_make_compute(struct amdgpu_device * adev,struct amdgpu_vm * vm)2134 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2135 {
2136 	bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2137 	int r;
2138 
2139 	r = amdgpu_bo_reserve(vm->root.bo, true);
2140 	if (r)
2141 		return r;
2142 
2143 	/* Sanity checks */
2144 	if (!amdgpu_vm_pt_is_root_clean(adev, vm)) {
2145 		r = -EINVAL;
2146 		goto unreserve_bo;
2147 	}
2148 
2149 	/* Check if PD needs to be reinitialized and do it before
2150 	 * changing any other state, in case it fails.
2151 	 */
2152 	if (pte_support_ats != vm->pte_support_ats) {
2153 		vm->pte_support_ats = pte_support_ats;
2154 		r = amdgpu_vm_pt_clear(adev, vm, to_amdgpu_bo_vm(vm->root.bo),
2155 				       false);
2156 		if (r)
2157 			goto unreserve_bo;
2158 	}
2159 
2160 	/* Update VM state */
2161 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2162 				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2163 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2164 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2165 	WARN_ONCE((vm->use_cpu_for_update &&
2166 		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2167 		  "CPU update of VM recommended only for large BAR system\n");
2168 
2169 	if (vm->use_cpu_for_update) {
2170 		/* Sync with last SDMA update/clear before switching to CPU */
2171 		r = amdgpu_bo_sync_wait(vm->root.bo,
2172 					AMDGPU_FENCE_OWNER_UNDEFINED, true);
2173 		if (r)
2174 			goto unreserve_bo;
2175 
2176 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2177 	} else {
2178 		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2179 	}
2180 	/*
2181 	 * Make sure root PD gets mapped. As vm_update_mode could be changed
2182 	 * when turning a GFX VM into a compute VM.
2183 	 */
2184 	r = vm->update_funcs->map_table(to_amdgpu_bo_vm(vm->root.bo));
2185 	if (r)
2186 		goto unreserve_bo;
2187 
2188 	dma_fence_put(vm->last_update);
2189 	vm->last_update = NULL;
2190 	vm->is_compute_context = true;
2191 
2192 	/* Free the shadow bo for compute VM */
2193 	amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow);
2194 
2195 	goto unreserve_bo;
2196 
2197 unreserve_bo:
2198 	amdgpu_bo_unreserve(vm->root.bo);
2199 	return r;
2200 }
2201 
2202 /**
2203  * amdgpu_vm_release_compute - release a compute vm
2204  * @adev: amdgpu_device pointer
2205  * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
2206  *
2207  * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2208  * pasid from vm. Compute should stop use of vm after this call.
2209  */
amdgpu_vm_release_compute(struct amdgpu_device * adev,struct amdgpu_vm * vm)2210 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2211 {
2212 	amdgpu_vm_set_pasid(adev, vm, 0);
2213 	vm->is_compute_context = false;
2214 }
2215 
2216 /**
2217  * amdgpu_vm_fini - tear down a vm instance
2218  *
2219  * @adev: amdgpu_device pointer
2220  * @vm: requested vm
2221  *
2222  * Tear down @vm.
2223  * Unbind the VM and remove all bos from the vm bo list
2224  */
amdgpu_vm_fini(struct amdgpu_device * adev,struct amdgpu_vm * vm)2225 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2226 {
2227 	struct amdgpu_bo_va_mapping *mapping, *tmp;
2228 	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2229 	struct amdgpu_bo *root;
2230 	unsigned long flags;
2231 	int i;
2232 
2233 	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2234 
2235 	flush_work(&vm->pt_free_work);
2236 
2237 	root = amdgpu_bo_ref(vm->root.bo);
2238 	amdgpu_bo_reserve(root, true);
2239 	amdgpu_vm_set_pasid(adev, vm, 0);
2240 	dma_fence_wait(vm->last_unlocked, false);
2241 	dma_fence_put(vm->last_unlocked);
2242 	dma_fence_wait(vm->last_tlb_flush, false);
2243 	/* Make sure that all fence callbacks have completed */
2244 	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
2245 	spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);
2246 	dma_fence_put(vm->last_tlb_flush);
2247 
2248 	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2249 		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2250 			amdgpu_vm_prt_fini(adev, vm);
2251 			prt_fini_needed = false;
2252 		}
2253 
2254 		list_del(&mapping->list);
2255 		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2256 	}
2257 
2258 	amdgpu_vm_pt_free_root(adev, vm);
2259 	amdgpu_bo_unreserve(root);
2260 	amdgpu_bo_unref(&root);
2261 	WARN_ON(vm->root.bo);
2262 
2263 	drm_sched_entity_destroy(&vm->immediate);
2264 	drm_sched_entity_destroy(&vm->delayed);
2265 
2266 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2267 		dev_err(adev->dev, "still active bo inside vm\n");
2268 	}
2269 	rbtree_postorder_for_each_entry_safe(mapping, tmp,
2270 					     &vm->va.rb_root, rb) {
2271 		/* Don't remove the mapping here, we don't want to trigger a
2272 		 * rebalance and the tree is about to be destroyed anyway.
2273 		 */
2274 		list_del(&mapping->list);
2275 		kfree(mapping);
2276 	}
2277 
2278 	dma_fence_put(vm->last_update);
2279 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2280 		amdgpu_vmid_free_reserved(adev, vm, i);
2281 }
2282 
2283 /**
2284  * amdgpu_vm_manager_init - init the VM manager
2285  *
2286  * @adev: amdgpu_device pointer
2287  *
2288  * Initialize the VM manager structures
2289  */
amdgpu_vm_manager_init(struct amdgpu_device * adev)2290 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2291 {
2292 	unsigned i;
2293 
2294 	/* Concurrent flushes are only possible starting with Vega10 and
2295 	 * are broken on Navi10 and Navi14.
2296 	 */
2297 	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
2298 					      adev->asic_type == CHIP_NAVI10 ||
2299 					      adev->asic_type == CHIP_NAVI14);
2300 	amdgpu_vmid_mgr_init(adev);
2301 
2302 	adev->vm_manager.fence_context =
2303 		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2304 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2305 		adev->vm_manager.seqno[i] = 0;
2306 
2307 	spin_lock_init(&adev->vm_manager.prt_lock);
2308 	atomic_set(&adev->vm_manager.num_prt_users, 0);
2309 
2310 	/* If not overridden by the user, by default, only in large BAR systems
2311 	 * Compute VM tables will be updated by CPU
2312 	 */
2313 #ifdef CONFIG_X86_64
2314 	if (amdgpu_vm_update_mode == -1) {
2315 		/* For asic with VF MMIO access protection
2316 		 * avoid using CPU for VM table updates
2317 		 */
2318 		if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&
2319 		    !amdgpu_sriov_vf_mmio_access_protection(adev))
2320 			adev->vm_manager.vm_update_mode =
2321 				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2322 		else
2323 			adev->vm_manager.vm_update_mode = 0;
2324 	} else
2325 		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2326 #else
2327 	adev->vm_manager.vm_update_mode = 0;
2328 #endif
2329 
2330 	xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
2331 }
2332 
2333 /**
2334  * amdgpu_vm_manager_fini - cleanup VM manager
2335  *
2336  * @adev: amdgpu_device pointer
2337  *
2338  * Cleanup the VM manager and free resources.
2339  */
amdgpu_vm_manager_fini(struct amdgpu_device * adev)2340 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2341 {
2342 	WARN_ON(!xa_empty(&adev->vm_manager.pasids));
2343 	xa_destroy(&adev->vm_manager.pasids);
2344 
2345 	amdgpu_vmid_mgr_fini(adev);
2346 }
2347 
2348 /**
2349  * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
2350  *
2351  * @dev: drm device pointer
2352  * @data: drm_amdgpu_vm
2353  * @filp: drm file pointer
2354  *
2355  * Returns:
2356  * 0 for success, -errno for errors.
2357  */
amdgpu_vm_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)2358 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2359 {
2360 	union drm_amdgpu_vm *args = data;
2361 	struct amdgpu_device *adev = drm_to_adev(dev);
2362 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
2363 	long timeout = msecs_to_jiffies(2000);
2364 	int r;
2365 
2366 	switch (args->in.op) {
2367 	case AMDGPU_VM_OP_RESERVE_VMID:
2368 		/* We only have requirement to reserve vmid from gfxhub */
2369 		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm,
2370 					       AMDGPU_GFXHUB_0);
2371 		if (r)
2372 			return r;
2373 		break;
2374 	case AMDGPU_VM_OP_UNRESERVE_VMID:
2375 		if (amdgpu_sriov_runtime(adev))
2376 			timeout = 8 * timeout;
2377 
2378 		/* Wait vm idle to make sure the vmid set in SPM_VMID is
2379 		 * not referenced anymore.
2380 		 */
2381 		r = amdgpu_bo_reserve(fpriv->vm.root.bo, true);
2382 		if (r)
2383 			return r;
2384 
2385 		r = amdgpu_vm_wait_idle(&fpriv->vm, timeout);
2386 		if (r < 0)
2387 			return r;
2388 
2389 		amdgpu_bo_unreserve(fpriv->vm.root.bo);
2390 		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
2391 		break;
2392 	default:
2393 		return -EINVAL;
2394 	}
2395 
2396 	return 0;
2397 }
2398 
2399 /**
2400  * amdgpu_vm_get_task_info - Extracts task info for a PASID.
2401  *
2402  * @adev: drm device pointer
2403  * @pasid: PASID identifier for VM
2404  * @task_info: task_info to fill.
2405  */
amdgpu_vm_get_task_info(struct amdgpu_device * adev,u32 pasid,struct amdgpu_task_info * task_info)2406 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
2407 			 struct amdgpu_task_info *task_info)
2408 {
2409 	struct amdgpu_vm *vm;
2410 	unsigned long flags;
2411 
2412 	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
2413 
2414 	vm = xa_load(&adev->vm_manager.pasids, pasid);
2415 	if (vm)
2416 		*task_info = vm->task_info;
2417 
2418 	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
2419 }
2420 
2421 /**
2422  * amdgpu_vm_set_task_info - Sets VMs task info.
2423  *
2424  * @vm: vm for which to set the info
2425  */
amdgpu_vm_set_task_info(struct amdgpu_vm * vm)2426 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
2427 {
2428 	if (vm->task_info.pid)
2429 		return;
2430 
2431 	vm->task_info.pid = current->pid;
2432 	get_task_comm(vm->task_info.task_name, current);
2433 
2434 	if (current->group_leader->mm != current->mm)
2435 		return;
2436 
2437 	vm->task_info.tgid = current->group_leader->pid;
2438 	get_task_comm(vm->task_info.process_name, current->group_leader);
2439 }
2440 
2441 /**
2442  * amdgpu_vm_handle_fault - graceful handling of VM faults.
2443  * @adev: amdgpu device pointer
2444  * @pasid: PASID of the VM
2445  * @addr: Address of the fault
2446  * @write_fault: true is write fault, false is read fault
2447  *
2448  * Try to gracefully handle a VM fault. Return true if the fault was handled and
2449  * shouldn't be reported any more.
2450  */
amdgpu_vm_handle_fault(struct amdgpu_device * adev,u32 pasid,uint64_t addr,bool write_fault)2451 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
2452 			    uint64_t addr, bool write_fault)
2453 {
2454 	bool is_compute_context = false;
2455 	struct amdgpu_bo *root;
2456 	unsigned long irqflags;
2457 	uint64_t value, flags;
2458 	struct amdgpu_vm *vm;
2459 	int r;
2460 
2461 	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2462 	vm = xa_load(&adev->vm_manager.pasids, pasid);
2463 	if (vm) {
2464 		root = amdgpu_bo_ref(vm->root.bo);
2465 		is_compute_context = vm->is_compute_context;
2466 	} else {
2467 		root = NULL;
2468 	}
2469 	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2470 
2471 	if (!root)
2472 		return false;
2473 
2474 	addr /= AMDGPU_GPU_PAGE_SIZE;
2475 
2476 	if (is_compute_context &&
2477 	    !svm_range_restore_pages(adev, pasid, addr, write_fault)) {
2478 		amdgpu_bo_unref(&root);
2479 		return true;
2480 	}
2481 
2482 	r = amdgpu_bo_reserve(root, true);
2483 	if (r)
2484 		goto error_unref;
2485 
2486 	/* Double check that the VM still exists */
2487 	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2488 	vm = xa_load(&adev->vm_manager.pasids, pasid);
2489 	if (vm && vm->root.bo != root)
2490 		vm = NULL;
2491 	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2492 	if (!vm)
2493 		goto error_unlock;
2494 
2495 	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
2496 		AMDGPU_PTE_SYSTEM;
2497 
2498 	if (is_compute_context) {
2499 		/* Intentionally setting invalid PTE flag
2500 		 * combination to force a no-retry-fault
2501 		 */
2502 		flags = AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT;
2503 		value = 0;
2504 	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
2505 		/* Redirect the access to the dummy page */
2506 		value = adev->dummy_page_addr;
2507 		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
2508 			AMDGPU_PTE_WRITEABLE;
2509 
2510 	} else {
2511 		/* Let the hw retry silently on the PTE */
2512 		value = 0;
2513 	}
2514 
2515 	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
2516 	if (r) {
2517 		pr_debug("failed %d to reserve fence slot\n", r);
2518 		goto error_unlock;
2519 	}
2520 
2521 	r = amdgpu_vm_update_range(adev, vm, true, false, false, NULL, addr,
2522 				   addr, flags, value, 0, NULL, NULL, NULL);
2523 	if (r)
2524 		goto error_unlock;
2525 
2526 	r = amdgpu_vm_update_pdes(adev, vm, true);
2527 
2528 error_unlock:
2529 	amdgpu_bo_unreserve(root);
2530 	if (r < 0)
2531 		DRM_ERROR("Can't handle page fault (%d)\n", r);
2532 
2533 error_unref:
2534 	amdgpu_bo_unref(&root);
2535 
2536 	return false;
2537 }
2538 
2539 #if defined(CONFIG_DEBUG_FS)
2540 /**
2541  * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
2542  *
2543  * @vm: Requested VM for printing BO info
2544  * @m: debugfs file
2545  *
2546  * Print BO information in debugfs file for the VM
2547  */
amdgpu_debugfs_vm_bo_info(struct amdgpu_vm * vm,struct seq_file * m)2548 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
2549 {
2550 	struct amdgpu_bo_va *bo_va, *tmp;
2551 	u64 total_idle = 0;
2552 	u64 total_evicted = 0;
2553 	u64 total_relocated = 0;
2554 	u64 total_moved = 0;
2555 	u64 total_invalidated = 0;
2556 	u64 total_done = 0;
2557 	unsigned int total_idle_objs = 0;
2558 	unsigned int total_evicted_objs = 0;
2559 	unsigned int total_relocated_objs = 0;
2560 	unsigned int total_moved_objs = 0;
2561 	unsigned int total_invalidated_objs = 0;
2562 	unsigned int total_done_objs = 0;
2563 	unsigned int id = 0;
2564 
2565 	spin_lock(&vm->status_lock);
2566 	seq_puts(m, "\tIdle BOs:\n");
2567 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
2568 		if (!bo_va->base.bo)
2569 			continue;
2570 		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2571 	}
2572 	total_idle_objs = id;
2573 	id = 0;
2574 
2575 	seq_puts(m, "\tEvicted BOs:\n");
2576 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
2577 		if (!bo_va->base.bo)
2578 			continue;
2579 		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2580 	}
2581 	total_evicted_objs = id;
2582 	id = 0;
2583 
2584 	seq_puts(m, "\tRelocated BOs:\n");
2585 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
2586 		if (!bo_va->base.bo)
2587 			continue;
2588 		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2589 	}
2590 	total_relocated_objs = id;
2591 	id = 0;
2592 
2593 	seq_puts(m, "\tMoved BOs:\n");
2594 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
2595 		if (!bo_va->base.bo)
2596 			continue;
2597 		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2598 	}
2599 	total_moved_objs = id;
2600 	id = 0;
2601 
2602 	seq_puts(m, "\tInvalidated BOs:\n");
2603 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
2604 		if (!bo_va->base.bo)
2605 			continue;
2606 		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
2607 	}
2608 	total_invalidated_objs = id;
2609 	id = 0;
2610 
2611 	seq_puts(m, "\tDone BOs:\n");
2612 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
2613 		if (!bo_va->base.bo)
2614 			continue;
2615 		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2616 	}
2617 	spin_unlock(&vm->status_lock);
2618 	total_done_objs = id;
2619 
2620 	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
2621 		   total_idle_objs);
2622 	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
2623 		   total_evicted_objs);
2624 	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
2625 		   total_relocated_objs);
2626 	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
2627 		   total_moved_objs);
2628 	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
2629 		   total_invalidated_objs);
2630 	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
2631 		   total_done_objs);
2632 }
2633 #endif
2634