1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24 #ifndef __AMDGPU_VM_H__
25 #define __AMDGPU_VM_H__
26
27 #include <linux/idr.h>
28 #include <linux/kfifo.h>
29 #include <linux/rbtree.h>
30 #include <drm/gpu_scheduler.h>
31 #include <drm/drm_file.h>
32 #include <drm/ttm/ttm_bo.h>
33 #include <linux/sched/mm.h>
34
35 #include "amdgpu_sync.h"
36 #include "amdgpu_ring.h"
37 #include "amdgpu_ids.h"
38
39 struct drm_exec;
40
41 struct amdgpu_bo_va;
42 struct amdgpu_job;
43 struct amdgpu_bo_list_entry;
44 struct amdgpu_bo_vm;
45 struct amdgpu_mem_stats;
46
47 /*
48 * GPUVM handling
49 */
50
51 /* Maximum number of PTEs the hardware can write with one command */
52 #define AMDGPU_VM_MAX_UPDATE_SIZE 0x3FFFF
53
54 /* number of entries in page table */
55 #define AMDGPU_VM_PTE_COUNT(adev) (1 << (adev)->vm_manager.block_size)
56
57 #define AMDGPU_PTE_VALID (1ULL << 0)
58 #define AMDGPU_PTE_SYSTEM (1ULL << 1)
59 #define AMDGPU_PTE_SNOOPED (1ULL << 2)
60
61 /* RV+ */
62 #define AMDGPU_PTE_TMZ (1ULL << 3)
63
64 /* VI only */
65 #define AMDGPU_PTE_EXECUTABLE (1ULL << 4)
66
67 #define AMDGPU_PTE_READABLE (1ULL << 5)
68 #define AMDGPU_PTE_WRITEABLE (1ULL << 6)
69
70 #define AMDGPU_PTE_FRAG(x) ((x & 0x1fULL) << 7)
71
72 /* TILED for VEGA10, reserved for older ASICs */
73 #define AMDGPU_PTE_PRT (1ULL << 51)
74
75 /* PDE is handled as PTE for VEGA10 */
76 #define AMDGPU_PDE_PTE (1ULL << 54)
77
78 #define AMDGPU_PTE_LOG (1ULL << 55)
79
80 /* PTE is handled as PDE for VEGA10 (Translate Further) */
81 #define AMDGPU_PTE_TF (1ULL << 56)
82
83 /* MALL noalloc for sienna_cichlid, reserved for older ASICs */
84 #define AMDGPU_PTE_NOALLOC (1ULL << 58)
85
86 /* PDE Block Fragment Size for VEGA10 */
87 #define AMDGPU_PDE_BFS(a) ((uint64_t)a << 59)
88
89 /* Flag combination to set no-retry with TF disabled */
90 #define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \
91 AMDGPU_PTE_TF)
92
93 /* Flag combination to set no-retry with TF enabled */
94 #define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \
95 AMDGPU_PTE_PRT)
96 /* For GFX9 */
97 #define AMDGPU_PTE_MTYPE_VG10(a) ((uint64_t)(a) << 57)
98 #define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)
99
100 #define AMDGPU_MTYPE_NC 0
101 #define AMDGPU_MTYPE_CC 2
102
103 #define AMDGPU_PTE_DEFAULT_ATC (AMDGPU_PTE_SYSTEM \
104 | AMDGPU_PTE_SNOOPED \
105 | AMDGPU_PTE_EXECUTABLE \
106 | AMDGPU_PTE_READABLE \
107 | AMDGPU_PTE_WRITEABLE \
108 | AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_CC))
109
110 /* gfx10 */
111 #define AMDGPU_PTE_MTYPE_NV10(a) ((uint64_t)(a) << 48)
112 #define AMDGPU_PTE_MTYPE_NV10_MASK AMDGPU_PTE_MTYPE_NV10(7ULL)
113
114 /* How to program VM fault handling */
115 #define AMDGPU_VM_FAULT_STOP_NEVER 0
116 #define AMDGPU_VM_FAULT_STOP_FIRST 1
117 #define AMDGPU_VM_FAULT_STOP_ALWAYS 2
118
119 /* Reserve 4MB VRAM for page tables */
120 #define AMDGPU_VM_RESERVED_VRAM (8ULL << 20)
121
122 /*
123 * max number of VMHUB
124 * layout: max 8 GFXHUB + 4 MMHUB0 + 1 MMHUB1
125 */
126 #define AMDGPU_MAX_VMHUBS 13
127 #define AMDGPU_GFXHUB(x) (x)
128 #define AMDGPU_MMHUB0(x) (8 + x)
129 #define AMDGPU_MMHUB1(x) (8 + 4 + x)
130
131 /* Reserve 2MB at top/bottom of address space for kernel use */
132 #define AMDGPU_VA_RESERVED_SIZE (2ULL << 20)
133
134 /* See vm_update_mode */
135 #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
136 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
137
138 /* VMPT level enumerate, and the hiberachy is:
139 * PDB2->PDB1->PDB0->PTB
140 */
141 enum amdgpu_vm_level {
142 AMDGPU_VM_PDB2,
143 AMDGPU_VM_PDB1,
144 AMDGPU_VM_PDB0,
145 AMDGPU_VM_PTB
146 };
147
148 /* base structure for tracking BO usage in a VM */
149 struct amdgpu_vm_bo_base {
150 /* constant after initialization */
151 struct amdgpu_vm *vm;
152 struct amdgpu_bo *bo;
153
154 /* protected by bo being reserved */
155 struct amdgpu_vm_bo_base *next;
156
157 /* protected by spinlock */
158 struct list_head vm_status;
159
160 /* protected by the BO being reserved */
161 bool moved;
162 };
163
164 /* provided by hw blocks that can write ptes, e.g., sdma */
165 struct amdgpu_vm_pte_funcs {
166 /* number of dw to reserve per operation */
167 unsigned copy_pte_num_dw;
168
169 /* copy pte entries from GART */
170 void (*copy_pte)(struct amdgpu_ib *ib,
171 uint64_t pe, uint64_t src,
172 unsigned count);
173
174 /* write pte one entry at a time with addr mapping */
175 void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
176 uint64_t value, unsigned count,
177 uint32_t incr);
178 /* for linear pte/pde updates without addr mapping */
179 void (*set_pte_pde)(struct amdgpu_ib *ib,
180 uint64_t pe,
181 uint64_t addr, unsigned count,
182 uint32_t incr, uint64_t flags);
183 };
184
185 struct amdgpu_task_info {
186 char process_name[TASK_COMM_LEN];
187 char task_name[TASK_COMM_LEN];
188 pid_t pid;
189 pid_t tgid;
190 };
191
192 /**
193 * struct amdgpu_vm_update_params
194 *
195 * Encapsulate some VM table update parameters to reduce
196 * the number of function parameters
197 *
198 */
199 struct amdgpu_vm_update_params {
200
201 /**
202 * @adev: amdgpu device we do this update for
203 */
204 struct amdgpu_device *adev;
205
206 /**
207 * @vm: optional amdgpu_vm we do this update for
208 */
209 struct amdgpu_vm *vm;
210
211 /**
212 * @immediate: if changes should be made immediately
213 */
214 bool immediate;
215
216 /**
217 * @unlocked: true if the root BO is not locked
218 */
219 bool unlocked;
220
221 /**
222 * @pages_addr:
223 *
224 * DMA addresses to use for mapping
225 */
226 dma_addr_t *pages_addr;
227
228 /**
229 * @job: job to used for hw submission
230 */
231 struct amdgpu_job *job;
232
233 /**
234 * @num_dw_left: number of dw left for the IB
235 */
236 unsigned int num_dw_left;
237
238 /**
239 * @table_freed: return true if page table is freed when updating
240 */
241 bool table_freed;
242 };
243
244 struct amdgpu_vm_update_funcs {
245 int (*map_table)(struct amdgpu_bo_vm *bo);
246 int (*prepare)(struct amdgpu_vm_update_params *p, struct dma_resv *resv,
247 enum amdgpu_sync_mode sync_mode);
248 int (*update)(struct amdgpu_vm_update_params *p,
249 struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
250 unsigned count, uint32_t incr, uint64_t flags);
251 int (*commit)(struct amdgpu_vm_update_params *p,
252 struct dma_fence **fence);
253 };
254
255 struct amdgpu_vm {
256 /* tree of virtual addresses mapped */
257 struct rb_root_cached va;
258
259 /* Lock to prevent eviction while we are updating page tables
260 * use vm_eviction_lock/unlock(vm)
261 */
262 struct mutex eviction_lock;
263 bool evicting;
264 unsigned int saved_flags;
265
266 /* Lock to protect vm_bo add/del/move on all lists of vm */
267 spinlock_t status_lock;
268
269 /* BOs who needs a validation */
270 struct list_head evicted;
271
272 /* PT BOs which relocated and their parent need an update */
273 struct list_head relocated;
274
275 /* per VM BOs moved, but not yet updated in the PT */
276 struct list_head moved;
277
278 /* All BOs of this VM not currently in the state machine */
279 struct list_head idle;
280
281 /* regular invalidated BOs, but not yet updated in the PT */
282 struct list_head invalidated;
283
284 /* BO mappings freed, but not yet updated in the PT */
285 struct list_head freed;
286
287 /* BOs which are invalidated, has been updated in the PTs */
288 struct list_head done;
289
290 /* PT BOs scheduled to free and fill with zero if vm_resv is not hold */
291 struct list_head pt_freed;
292 struct work_struct pt_free_work;
293
294 /* contains the page directory */
295 struct amdgpu_vm_bo_base root;
296 struct dma_fence *last_update;
297
298 /* Scheduler entities for page table updates */
299 struct drm_sched_entity immediate;
300 struct drm_sched_entity delayed;
301
302 /* Last finished delayed update */
303 atomic64_t tlb_seq;
304 struct dma_fence *last_tlb_flush;
305
306 /* How many times we had to re-generate the page tables */
307 uint64_t generation;
308
309 /* Last unlocked submission to the scheduler entities */
310 struct dma_fence *last_unlocked;
311
312 unsigned int pasid;
313 bool reserved_vmid[AMDGPU_MAX_VMHUBS];
314
315 /* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */
316 bool use_cpu_for_update;
317
318 /* Functions to use for VM table updates */
319 const struct amdgpu_vm_update_funcs *update_funcs;
320
321 /* Flag to indicate ATS support from PTE for GFX9 */
322 bool pte_support_ats;
323
324 /* Up to 128 pending retry page faults */
325 DECLARE_KFIFO(faults, u64, 128);
326
327 /* Points to the KFD process VM info */
328 struct amdkfd_process_info *process_info;
329
330 /* List node in amdkfd_process_info.vm_list_head */
331 struct list_head vm_list_node;
332
333 /* Valid while the PD is reserved or fenced */
334 uint64_t pd_phys_addr;
335
336 /* Some basic info about the task */
337 struct amdgpu_task_info task_info;
338
339 /* Store positions of group of BOs */
340 struct ttm_lru_bulk_move lru_bulk_move;
341 /* Flag to indicate if VM is used for compute */
342 bool is_compute_context;
343
344 /* Memory partition number, -1 means any partition */
345 int8_t mem_id;
346 };
347
348 struct amdgpu_vm_manager {
349 /* Handling of VMIDs */
350 struct amdgpu_vmid_mgr id_mgr[AMDGPU_MAX_VMHUBS];
351 unsigned int first_kfd_vmid;
352 bool concurrent_flush;
353
354 /* Handling of VM fences */
355 u64 fence_context;
356 unsigned seqno[AMDGPU_MAX_RINGS];
357
358 uint64_t max_pfn;
359 uint32_t num_level;
360 uint32_t block_size;
361 uint32_t fragment_size;
362 enum amdgpu_vm_level root_level;
363 /* vram base address for page table entry */
364 u64 vram_base_offset;
365 /* vm pte handling */
366 const struct amdgpu_vm_pte_funcs *vm_pte_funcs;
367 struct drm_gpu_scheduler *vm_pte_scheds[AMDGPU_MAX_RINGS];
368 unsigned vm_pte_num_scheds;
369 struct amdgpu_ring *page_fault;
370
371 /* partial resident texture handling */
372 spinlock_t prt_lock;
373 atomic_t num_prt_users;
374
375 /* controls how VM page tables are updated for Graphics and Compute.
376 * BIT0[= 0] Graphics updated by SDMA [= 1] by CPU
377 * BIT1[= 0] Compute updated by SDMA [= 1] by CPU
378 */
379 int vm_update_mode;
380
381 /* PASID to VM mapping, will be used in interrupt context to
382 * look up VM of a page fault
383 */
384 struct xarray pasids;
385 };
386
387 struct amdgpu_bo_va_mapping;
388
389 #define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
390 #define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
391 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
392
393 extern const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs;
394 extern const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs;
395
396 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
397 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
398
399 int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
400 u32 pasid);
401
402 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
403 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int32_t xcp_id);
404 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
405 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
406 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
407 int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
408 unsigned int num_fences);
409 bool amdgpu_vm_ready(struct amdgpu_vm *vm);
410 uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
411 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
412 int (*callback)(void *p, struct amdgpu_bo *bo),
413 void *param);
414 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
415 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
416 struct amdgpu_vm *vm, bool immediate);
417 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
418 struct amdgpu_vm *vm,
419 struct dma_fence **fence);
420 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
421 struct amdgpu_vm *vm);
422 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
423 struct amdgpu_vm *vm, struct amdgpu_bo *bo);
424 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
425 bool immediate, bool unlocked, bool flush_tlb,
426 struct dma_resv *resv, uint64_t start, uint64_t last,
427 uint64_t flags, uint64_t offset, uint64_t vram_base,
428 struct ttm_resource *res, dma_addr_t *pages_addr,
429 struct dma_fence **fence);
430 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
431 struct amdgpu_bo_va *bo_va,
432 bool clear);
433 bool amdgpu_vm_evictable(struct amdgpu_bo *bo);
434 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
435 struct amdgpu_bo *bo, bool evicted);
436 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr);
437 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
438 struct amdgpu_bo *bo);
439 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
440 struct amdgpu_vm *vm,
441 struct amdgpu_bo *bo);
442 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
443 struct amdgpu_bo_va *bo_va,
444 uint64_t addr, uint64_t offset,
445 uint64_t size, uint64_t flags);
446 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
447 struct amdgpu_bo_va *bo_va,
448 uint64_t addr, uint64_t offset,
449 uint64_t size, uint64_t flags);
450 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
451 struct amdgpu_bo_va *bo_va,
452 uint64_t addr);
453 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
454 struct amdgpu_vm *vm,
455 uint64_t saddr, uint64_t size);
456 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
457 uint64_t addr);
458 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket);
459 void amdgpu_vm_bo_del(struct amdgpu_device *adev,
460 struct amdgpu_bo_va *bo_va);
461 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
462 uint32_t fragment_size_default, unsigned max_level,
463 unsigned max_bits);
464 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
465 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
466 struct amdgpu_job *job);
467 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev);
468
469 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
470 struct amdgpu_task_info *task_info);
471 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
472 u32 vmid, u32 node_id, uint64_t addr,
473 bool write_fault);
474
475 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm);
476
477 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
478 struct amdgpu_vm *vm);
479 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
480 struct amdgpu_mem_stats *stats);
481
482 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
483 struct amdgpu_bo_vm *vmbo, bool immediate);
484 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
485 int level, bool immediate, struct amdgpu_bo_vm **vmbo,
486 int32_t xcp_id);
487 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
488 bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev,
489 struct amdgpu_vm *vm);
490
491 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
492 struct amdgpu_vm_bo_base *entry);
493 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
494 uint64_t start, uint64_t end,
495 uint64_t dst, uint64_t flags);
496 void amdgpu_vm_pt_free_work(struct work_struct *work);
497
498 #if defined(CONFIG_DEBUG_FS)
499 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
500 #endif
501
502 int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
503
504 /**
505 * amdgpu_vm_tlb_seq - return tlb flush sequence number
506 * @vm: the amdgpu_vm structure to query
507 *
508 * Returns the tlb flush sequence number which indicates that the VM TLBs needs
509 * to be invalidated whenever the sequence number change.
510 */
amdgpu_vm_tlb_seq(struct amdgpu_vm * vm)511 static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
512 {
513 unsigned long flags;
514 spinlock_t *lock;
515
516 /*
517 * Workaround to stop racing between the fence signaling and handling
518 * the cb. The lock is static after initially setting it up, just make
519 * sure that the dma_fence structure isn't freed up.
520 */
521 rcu_read_lock();
522 lock = vm->last_tlb_flush->lock;
523 rcu_read_unlock();
524
525 spin_lock_irqsave(lock, flags);
526 spin_unlock_irqrestore(lock, flags);
527
528 return atomic64_read(&vm->tlb_seq);
529 }
530
531 /*
532 * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
533 * happens while holding this lock anywhere to prevent deadlocks when
534 * an MMU notifier runs in reclaim-FS context.
535 */
amdgpu_vm_eviction_lock(struct amdgpu_vm * vm)536 static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
537 {
538 mutex_lock(&vm->eviction_lock);
539 vm->saved_flags = memalloc_noreclaim_save();
540 }
541
amdgpu_vm_eviction_trylock(struct amdgpu_vm * vm)542 static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
543 {
544 if (mutex_trylock(&vm->eviction_lock)) {
545 vm->saved_flags = memalloc_noreclaim_save();
546 return true;
547 }
548 return false;
549 }
550
amdgpu_vm_eviction_unlock(struct amdgpu_vm * vm)551 static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
552 {
553 memalloc_noreclaim_restore(vm->saved_flags);
554 mutex_unlock(&vm->eviction_lock);
555 }
556
557 #endif
558