1 /*
2 * Copyright 2017 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #define NVKM_VMM_LEVELS_MAX 5
23 #include "vmm.h"
24
25 #include <subdev/fb.h>
26
27 static void
nvkm_vmm_pt_del(struct nvkm_vmm_pt ** ppgt)28 nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
29 {
30 struct nvkm_vmm_pt *pgt = *ppgt;
31 if (pgt) {
32 kvfree(pgt->pde);
33 kfree(pgt);
34 *ppgt = NULL;
35 }
36 }
37
38
39 static struct nvkm_vmm_pt *
nvkm_vmm_pt_new(const struct nvkm_vmm_desc * desc,bool sparse,const struct nvkm_vmm_page * page)40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
41 const struct nvkm_vmm_page *page)
42 {
43 const u32 pten = 1 << desc->bits;
44 struct nvkm_vmm_pt *pgt;
45 u32 lpte = 0;
46
47 if (desc->type > PGT) {
48 if (desc->type == SPT) {
49 const struct nvkm_vmm_desc *pair = page[-1].desc;
50 lpte = pten >> (desc->bits - pair->bits);
51 } else {
52 lpte = pten;
53 }
54 }
55
56 if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
57 return NULL;
58 pgt->page = page ? page->shift : 0;
59 pgt->sparse = sparse;
60
61 if (desc->type == PGD) {
62 pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
63 if (!pgt->pde) {
64 kfree(pgt);
65 return NULL;
66 }
67 }
68
69 return pgt;
70 }
71
72 struct nvkm_vmm_iter {
73 const struct nvkm_vmm_page *page;
74 const struct nvkm_vmm_desc *desc;
75 struct nvkm_vmm *vmm;
76 u64 cnt;
77 u16 max, lvl;
78 u32 pte[NVKM_VMM_LEVELS_MAX];
79 struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
80 int flush;
81 };
82
83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
84 static const char *
nvkm_vmm_desc_type(const struct nvkm_vmm_desc * desc)85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
86 {
87 switch (desc->type) {
88 case PGD: return "PGD";
89 case PGT: return "PGT";
90 case SPT: return "SPT";
91 case LPT: return "LPT";
92 default:
93 return "UNKNOWN";
94 }
95 }
96
97 static void
nvkm_vmm_trace(struct nvkm_vmm_iter * it,char * buf)98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
99 {
100 int lvl;
101 for (lvl = it->max; lvl >= 0; lvl--) {
102 if (lvl >= it->lvl)
103 buf += sprintf(buf, "%05x:", it->pte[lvl]);
104 else
105 buf += sprintf(buf, "xxxxx:");
106 }
107 }
108
109 #define TRA(i,f,a...) do { \
110 char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
111 struct nvkm_vmm_iter *_it = (i); \
112 nvkm_vmm_trace(_it, _buf); \
113 VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
114 } while(0)
115 #else
116 #define TRA(i,f,a...)
117 #endif
118
119 static inline void
nvkm_vmm_flush_mark(struct nvkm_vmm_iter * it)120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
121 {
122 it->flush = min(it->flush, it->max - it->lvl);
123 }
124
125 static inline void
nvkm_vmm_flush(struct nvkm_vmm_iter * it)126 nvkm_vmm_flush(struct nvkm_vmm_iter *it)
127 {
128 if (it->flush != NVKM_VMM_LEVELS_MAX) {
129 if (it->vmm->func->flush) {
130 TRA(it, "flush: %d", it->flush);
131 it->vmm->func->flush(it->vmm, it->flush);
132 }
133 it->flush = NVKM_VMM_LEVELS_MAX;
134 }
135 }
136
137 static void
nvkm_vmm_unref_pdes(struct nvkm_vmm_iter * it)138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
139 {
140 const struct nvkm_vmm_desc *desc = it->desc;
141 const int type = desc[it->lvl].type == SPT;
142 struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
143 struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144 struct nvkm_mmu_pt *pt = pgt->pt[type];
145 struct nvkm_vmm *vmm = it->vmm;
146 u32 pdei = it->pte[it->lvl + 1];
147
148 /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
149 it->lvl++;
150 if (--pgd->refs[0]) {
151 const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
152 /* PD has other valid PDEs, so we need a proper update. */
153 TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
154 pgt->pt[type] = NULL;
155 if (!pgt->refs[!type]) {
156 /* PDE no longer required. */
157 if (pgd->pt[0]) {
158 if (pgt->sparse) {
159 func->sparse(vmm, pgd->pt[0], pdei, 1);
160 pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
161 } else {
162 func->unmap(vmm, pgd->pt[0], pdei, 1);
163 pgd->pde[pdei] = NULL;
164 }
165 } else {
166 /* Special handling for Tesla-class GPUs,
167 * where there's no central PD, but each
168 * instance has its own embedded PD.
169 */
170 func->pde(vmm, pgd, pdei);
171 pgd->pde[pdei] = NULL;
172 }
173 } else {
174 /* PDE was pointing at dual-PTs and we're removing
175 * one of them, leaving the other in place.
176 */
177 func->pde(vmm, pgd, pdei);
178 }
179
180 /* GPU may have cached the PTs, flush before freeing. */
181 nvkm_vmm_flush_mark(it);
182 nvkm_vmm_flush(it);
183 } else {
184 /* PD has no valid PDEs left, so we can just destroy it. */
185 nvkm_vmm_unref_pdes(it);
186 }
187
188 /* Destroy PD/PT. */
189 TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
190 nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
191 if (!pgt->refs[!type])
192 nvkm_vmm_pt_del(&pgt);
193 it->lvl--;
194 }
195
196 static void
nvkm_vmm_unref_sptes(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgt,const struct nvkm_vmm_desc * desc,u32 ptei,u32 ptes)197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
198 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
199 {
200 const struct nvkm_vmm_desc *pair = it->page[-1].desc;
201 const u32 sptb = desc->bits - pair->bits;
202 const u32 sptn = 1 << sptb;
203 struct nvkm_vmm *vmm = it->vmm;
204 u32 spti = ptei & (sptn - 1), lpti, pteb;
205
206 /* Determine how many SPTEs are being touched under each LPTE,
207 * and drop reference counts.
208 */
209 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
210 const u32 pten = min(sptn - spti, ptes);
211 pgt->pte[lpti] -= pten;
212 ptes -= pten;
213 }
214
215 /* We're done here if there's no corresponding LPT. */
216 if (!pgt->refs[0])
217 return;
218
219 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
220 /* Skip over any LPTEs that still have valid SPTEs. */
221 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
222 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
223 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
224 break;
225 }
226 continue;
227 }
228
229 /* As there's no more non-UNMAPPED SPTEs left in the range
230 * covered by a number of LPTEs, the LPTEs once again take
231 * control over their address range.
232 *
233 * Determine how many LPTEs need to transition state.
234 */
235 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
236 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
237 if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
238 break;
239 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
240 }
241
242 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
243 TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
244 pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
245 } else
246 if (pair->func->invalid) {
247 /* If the MMU supports it, restore the LPTE to the
248 * INVALID state to tell the MMU there is no point
249 * trying to fetch the corresponding SPTEs.
250 */
251 TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
252 pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
253 }
254 }
255 }
256
257 static bool
nvkm_vmm_unref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
259 {
260 const struct nvkm_vmm_desc *desc = it->desc;
261 const int type = desc->type == SPT;
262 struct nvkm_vmm_pt *pgt = it->pt[0];
263 bool dma;
264
265 if (pfn) {
266 /* Need to clear PTE valid bits before we dma_unmap_page(). */
267 dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
268 if (dma) {
269 /* GPU may have cached the PT, flush before unmap. */
270 nvkm_vmm_flush_mark(it);
271 nvkm_vmm_flush(it);
272 desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
273 }
274 }
275
276 /* Drop PTE references. */
277 pgt->refs[type] -= ptes;
278
279 /* Dual-PTs need special handling, unless PDE becoming invalid. */
280 if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
281 nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
282
283 /* PT no longer needed? Destroy it. */
284 if (!pgt->refs[type]) {
285 it->lvl++;
286 TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
287 it->lvl--;
288 nvkm_vmm_unref_pdes(it);
289 return false; /* PTE writes for unmap() not necessary. */
290 }
291
292 return true;
293 }
294
295 static void
nvkm_vmm_ref_sptes(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgt,const struct nvkm_vmm_desc * desc,u32 ptei,u32 ptes)296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
297 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
298 {
299 const struct nvkm_vmm_desc *pair = it->page[-1].desc;
300 const u32 sptb = desc->bits - pair->bits;
301 const u32 sptn = 1 << sptb;
302 struct nvkm_vmm *vmm = it->vmm;
303 u32 spti = ptei & (sptn - 1), lpti, pteb;
304
305 /* Determine how many SPTEs are being touched under each LPTE,
306 * and increase reference counts.
307 */
308 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
309 const u32 pten = min(sptn - spti, ptes);
310 pgt->pte[lpti] += pten;
311 ptes -= pten;
312 }
313
314 /* We're done here if there's no corresponding LPT. */
315 if (!pgt->refs[0])
316 return;
317
318 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
319 /* Skip over any LPTEs that already have valid SPTEs. */
320 if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
321 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
322 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
323 break;
324 }
325 continue;
326 }
327
328 /* As there are now non-UNMAPPED SPTEs in the range covered
329 * by a number of LPTEs, we need to transfer control of the
330 * address range to the SPTEs.
331 *
332 * Determine how many LPTEs need to transition state.
333 */
334 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
335 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
336 if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
337 break;
338 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
339 }
340
341 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
342 const u32 spti = pteb * sptn;
343 const u32 sptc = ptes * sptn;
344 /* The entire LPTE is marked as sparse, we need
345 * to make sure that the SPTEs are too.
346 */
347 TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
348 desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
349 /* Sparse LPTEs prevent SPTEs from being accessed. */
350 TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
351 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
352 } else
353 if (pair->func->invalid) {
354 /* MMU supports blocking SPTEs by marking an LPTE
355 * as INVALID. We need to reverse that here.
356 */
357 TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
358 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
359 }
360 }
361 }
362
363 static bool
nvkm_vmm_ref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
365 {
366 const struct nvkm_vmm_desc *desc = it->desc;
367 const int type = desc->type == SPT;
368 struct nvkm_vmm_pt *pgt = it->pt[0];
369
370 /* Take PTE references. */
371 pgt->refs[type] += ptes;
372
373 /* Dual-PTs need special handling. */
374 if (desc->type == SPT)
375 nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
376
377 return true;
378 }
379
380 static void
nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc * desc,struct nvkm_vmm_pt * pgt,u32 ptei,u32 ptes)381 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
382 struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
383 {
384 if (desc->type == PGD) {
385 while (ptes--)
386 pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
387 } else
388 if (desc->type == LPT) {
389 memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
390 }
391 }
392
393 static bool
nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
395 {
396 struct nvkm_vmm_pt *pt = it->pt[0];
397 if (it->desc->type == PGD)
398 memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
399 else
400 if (it->desc->type == LPT)
401 memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
402 return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
403 }
404
405 static bool
nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
407 {
408 nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
409 return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
410 }
411
412 static bool
nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgd,u32 pdei)413 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
414 {
415 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
416 const int type = desc->type == SPT;
417 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
418 const bool zero = !pgt->sparse && !desc->func->invalid;
419 struct nvkm_vmm *vmm = it->vmm;
420 struct nvkm_mmu *mmu = vmm->mmu;
421 struct nvkm_mmu_pt *pt;
422 u32 pten = 1 << desc->bits;
423 u32 pteb, ptei, ptes;
424 u32 size = desc->size * pten;
425
426 pgd->refs[0]++;
427
428 pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
429 if (!pgt->pt[type]) {
430 it->lvl--;
431 nvkm_vmm_unref_pdes(it);
432 return false;
433 }
434
435 if (zero)
436 goto done;
437
438 pt = pgt->pt[type];
439
440 if (desc->type == LPT && pgt->refs[1]) {
441 /* SPT already exists covering the same range as this LPT,
442 * which means we need to be careful that any LPTEs which
443 * overlap valid SPTEs are unmapped as opposed to invalid
444 * or sparse, which would prevent the MMU from looking at
445 * the SPTEs on some GPUs.
446 */
447 for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
448 bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
449 for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
450 bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
451 if (spte != next)
452 break;
453 }
454
455 if (!spte) {
456 if (pgt->sparse)
457 desc->func->sparse(vmm, pt, pteb, ptes);
458 else
459 desc->func->invalid(vmm, pt, pteb, ptes);
460 memset(&pgt->pte[pteb], 0x00, ptes);
461 } else {
462 desc->func->unmap(vmm, pt, pteb, ptes);
463 while (ptes--)
464 pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
465 }
466 }
467 } else {
468 if (pgt->sparse) {
469 nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
470 desc->func->sparse(vmm, pt, 0, pten);
471 } else {
472 desc->func->invalid(vmm, pt, 0, pten);
473 }
474 }
475
476 done:
477 TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
478 it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
479 nvkm_vmm_flush_mark(it);
480 return true;
481 }
482
483 static bool
nvkm_vmm_ref_swpt(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgd,u32 pdei)484 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
485 {
486 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
487 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
488
489 pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
490 if (!pgt) {
491 if (!pgd->refs[0])
492 nvkm_vmm_unref_pdes(it);
493 return false;
494 }
495
496 pgd->pde[pdei] = pgt;
497 return true;
498 }
499
500 static inline u64
nvkm_vmm_iter(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,const char * name,bool ref,bool pfn,bool (* REF_PTES)(struct nvkm_vmm_iter *,bool pfn,u32,u32),nvkm_vmm_pte_func MAP_PTES,struct nvkm_vmm_map * map,nvkm_vmm_pxe_func CLR_PTES)501 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
502 u64 addr, u64 size, const char *name, bool ref, bool pfn,
503 bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
504 nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
505 nvkm_vmm_pxe_func CLR_PTES)
506 {
507 const struct nvkm_vmm_desc *desc = page->desc;
508 struct nvkm_vmm_iter it;
509 u64 bits = addr >> page->shift;
510
511 it.page = page;
512 it.desc = desc;
513 it.vmm = vmm;
514 it.cnt = size >> page->shift;
515 it.flush = NVKM_VMM_LEVELS_MAX;
516
517 /* Deconstruct address into PTE indices for each mapping level. */
518 for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
519 it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
520 bits >>= desc[it.lvl].bits;
521 }
522 it.max = --it.lvl;
523 it.pt[it.max] = vmm->pd;
524
525 it.lvl = 0;
526 TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
527 addr, size, page->shift, it.cnt);
528 it.lvl = it.max;
529
530 /* Depth-first traversal of page tables. */
531 while (it.cnt) {
532 struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
533 const int type = desc->type == SPT;
534 const u32 pten = 1 << desc->bits;
535 const u32 ptei = it.pte[0];
536 const u32 ptes = min_t(u64, it.cnt, pten - ptei);
537
538 /* Walk down the tree, finding page tables for each level. */
539 for (; it.lvl; it.lvl--) {
540 const u32 pdei = it.pte[it.lvl];
541 struct nvkm_vmm_pt *pgd = pgt;
542
543 /* Software PT. */
544 if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
545 if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
546 goto fail;
547 }
548 it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
549
550 /* Hardware PT.
551 *
552 * This is a separate step from above due to GF100 and
553 * newer having dual page tables at some levels, which
554 * are refcounted independently.
555 */
556 if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
557 if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
558 goto fail;
559 }
560 }
561
562 /* Handle PTE updates. */
563 if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
564 struct nvkm_mmu_pt *pt = pgt->pt[type];
565 if (MAP_PTES || CLR_PTES) {
566 if (MAP_PTES)
567 MAP_PTES(vmm, pt, ptei, ptes, map);
568 else
569 CLR_PTES(vmm, pt, ptei, ptes);
570 nvkm_vmm_flush_mark(&it);
571 }
572 }
573
574 /* Walk back up the tree to the next position. */
575 it.pte[it.lvl] += ptes;
576 it.cnt -= ptes;
577 if (it.cnt) {
578 while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
579 it.pte[it.lvl++] = 0;
580 it.pte[it.lvl]++;
581 }
582 }
583 }
584
585 nvkm_vmm_flush(&it);
586 return ~0ULL;
587
588 fail:
589 /* Reconstruct the failure address so the caller is able to
590 * reverse any partially completed operations.
591 */
592 addr = it.pte[it.max--];
593 do {
594 addr = addr << desc[it.max].bits;
595 addr |= it.pte[it.max];
596 } while (it.max--);
597
598 return addr << page->shift;
599 }
600
601 static void
nvkm_vmm_ptes_sparse_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)602 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
603 u64 addr, u64 size)
604 {
605 nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
606 nvkm_vmm_sparse_unref_ptes, NULL, NULL,
607 page->desc->func->invalid ?
608 page->desc->func->invalid : page->desc->func->unmap);
609 }
610
611 static int
nvkm_vmm_ptes_sparse_get(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)612 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
613 u64 addr, u64 size)
614 {
615 if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
616 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
617 true, false, nvkm_vmm_sparse_ref_ptes,
618 NULL, NULL, page->desc->func->sparse);
619 if (fail != ~0ULL) {
620 if ((size = fail - addr))
621 nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
622 return -ENOMEM;
623 }
624 return 0;
625 }
626 return -EINVAL;
627 }
628
629 static int
nvkm_vmm_ptes_sparse(struct nvkm_vmm * vmm,u64 addr,u64 size,bool ref)630 nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
631 {
632 const struct nvkm_vmm_page *page = vmm->func->page;
633 int m = 0, i;
634 u64 start = addr;
635 u64 block;
636
637 while (size) {
638 /* Limit maximum page size based on remaining size. */
639 while (size < (1ULL << page[m].shift))
640 m++;
641 i = m;
642
643 /* Find largest page size suitable for alignment. */
644 while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
645 i++;
646
647 /* Determine number of PTEs at this page size. */
648 if (i != m) {
649 /* Limited to alignment boundary of next page size. */
650 u64 next = 1ULL << page[i - 1].shift;
651 u64 part = ALIGN(addr, next) - addr;
652 if (size - part >= next)
653 block = (part >> page[i].shift) << page[i].shift;
654 else
655 block = (size >> page[i].shift) << page[i].shift;
656 } else {
657 block = (size >> page[i].shift) << page[i].shift;
658 }
659
660 /* Perform operation. */
661 if (ref) {
662 int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
663 if (ret) {
664 if ((size = addr - start))
665 nvkm_vmm_ptes_sparse(vmm, start, size, false);
666 return ret;
667 }
668 } else {
669 nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
670 }
671
672 size -= block;
673 addr += block;
674 }
675
676 return 0;
677 }
678
679 static void
nvkm_vmm_ptes_unmap(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,bool sparse,bool pfn)680 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
681 u64 addr, u64 size, bool sparse, bool pfn)
682 {
683 const struct nvkm_vmm_desc_func *func = page->desc->func;
684
685 mutex_lock(&vmm->mutex.map);
686 nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
687 NULL, NULL, NULL,
688 sparse ? func->sparse : func->invalid ? func->invalid :
689 func->unmap);
690 mutex_unlock(&vmm->mutex.map);
691 }
692
693 static void
nvkm_vmm_ptes_map(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,struct nvkm_vmm_map * map,nvkm_vmm_pte_func func)694 nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
695 u64 addr, u64 size, struct nvkm_vmm_map *map,
696 nvkm_vmm_pte_func func)
697 {
698 mutex_lock(&vmm->mutex.map);
699 nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
700 NULL, func, map, NULL);
701 mutex_unlock(&vmm->mutex.map);
702 }
703
704 static void
nvkm_vmm_ptes_put_locked(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)705 nvkm_vmm_ptes_put_locked(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
706 u64 addr, u64 size)
707 {
708 nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
709 nvkm_vmm_unref_ptes, NULL, NULL, NULL);
710 }
711
712 static void
nvkm_vmm_ptes_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)713 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
714 u64 addr, u64 size)
715 {
716 mutex_lock(&vmm->mutex.ref);
717 nvkm_vmm_ptes_put_locked(vmm, page, addr, size);
718 mutex_unlock(&vmm->mutex.ref);
719 }
720
721 static int
nvkm_vmm_ptes_get(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)722 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
723 u64 addr, u64 size)
724 {
725 u64 fail;
726
727 mutex_lock(&vmm->mutex.ref);
728 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
729 nvkm_vmm_ref_ptes, NULL, NULL, NULL);
730 if (fail != ~0ULL) {
731 if (fail != addr)
732 nvkm_vmm_ptes_put_locked(vmm, page, addr, fail - addr);
733 mutex_unlock(&vmm->mutex.ref);
734 return -ENOMEM;
735 }
736 mutex_unlock(&vmm->mutex.ref);
737 return 0;
738 }
739
740 static void
__nvkm_vmm_ptes_unmap_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,bool sparse,bool pfn)741 __nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
742 u64 addr, u64 size, bool sparse, bool pfn)
743 {
744 const struct nvkm_vmm_desc_func *func = page->desc->func;
745
746 nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
747 false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
748 sparse ? func->sparse : func->invalid ? func->invalid :
749 func->unmap);
750 }
751
752 static void
nvkm_vmm_ptes_unmap_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,bool sparse,bool pfn)753 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
754 u64 addr, u64 size, bool sparse, bool pfn)
755 {
756 if (vmm->managed.raw) {
757 nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, pfn);
758 nvkm_vmm_ptes_put(vmm, page, addr, size);
759 } else {
760 __nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, sparse, pfn);
761 }
762 }
763
764 static int
__nvkm_vmm_ptes_get_map(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,struct nvkm_vmm_map * map,nvkm_vmm_pte_func func)765 __nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
766 u64 addr, u64 size, struct nvkm_vmm_map *map,
767 nvkm_vmm_pte_func func)
768 {
769 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
770 false, nvkm_vmm_ref_ptes, func, map, NULL);
771 if (fail != ~0ULL) {
772 if ((size = fail - addr))
773 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
774 return -ENOMEM;
775 }
776 return 0;
777 }
778
779 static int
nvkm_vmm_ptes_get_map(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,struct nvkm_vmm_map * map,nvkm_vmm_pte_func func)780 nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
781 u64 addr, u64 size, struct nvkm_vmm_map *map,
782 nvkm_vmm_pte_func func)
783 {
784 int ret;
785
786 if (vmm->managed.raw) {
787 ret = nvkm_vmm_ptes_get(vmm, page, addr, size);
788 if (ret)
789 return ret;
790
791 nvkm_vmm_ptes_map(vmm, page, addr, size, map, func);
792
793 return 0;
794 } else {
795 return __nvkm_vmm_ptes_get_map(vmm, page, addr, size, map, func);
796 }
797 }
798
799 struct nvkm_vma *
nvkm_vma_new(u64 addr,u64 size)800 nvkm_vma_new(u64 addr, u64 size)
801 {
802 struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
803 if (vma) {
804 vma->addr = addr;
805 vma->size = size;
806 vma->page = NVKM_VMA_PAGE_NONE;
807 vma->refd = NVKM_VMA_PAGE_NONE;
808 }
809 return vma;
810 }
811
812 struct nvkm_vma *
nvkm_vma_tail(struct nvkm_vma * vma,u64 tail)813 nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
814 {
815 struct nvkm_vma *new;
816
817 BUG_ON(vma->size == tail);
818
819 if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
820 return NULL;
821 vma->size -= tail;
822
823 new->mapref = vma->mapref;
824 new->sparse = vma->sparse;
825 new->page = vma->page;
826 new->refd = vma->refd;
827 new->used = vma->used;
828 new->part = vma->part;
829 new->busy = vma->busy;
830 new->mapped = vma->mapped;
831 list_add(&new->head, &vma->head);
832 return new;
833 }
834
835 static inline void
nvkm_vmm_free_remove(struct nvkm_vmm * vmm,struct nvkm_vma * vma)836 nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
837 {
838 rb_erase(&vma->tree, &vmm->free);
839 }
840
841 static inline void
nvkm_vmm_free_delete(struct nvkm_vmm * vmm,struct nvkm_vma * vma)842 nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
843 {
844 nvkm_vmm_free_remove(vmm, vma);
845 list_del(&vma->head);
846 kfree(vma);
847 }
848
849 static void
nvkm_vmm_free_insert(struct nvkm_vmm * vmm,struct nvkm_vma * vma)850 nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
851 {
852 struct rb_node **ptr = &vmm->free.rb_node;
853 struct rb_node *parent = NULL;
854
855 while (*ptr) {
856 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
857 parent = *ptr;
858 if (vma->size < this->size)
859 ptr = &parent->rb_left;
860 else
861 if (vma->size > this->size)
862 ptr = &parent->rb_right;
863 else
864 if (vma->addr < this->addr)
865 ptr = &parent->rb_left;
866 else
867 if (vma->addr > this->addr)
868 ptr = &parent->rb_right;
869 else
870 BUG();
871 }
872
873 rb_link_node(&vma->tree, parent, ptr);
874 rb_insert_color(&vma->tree, &vmm->free);
875 }
876
877 static inline void
nvkm_vmm_node_remove(struct nvkm_vmm * vmm,struct nvkm_vma * vma)878 nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
879 {
880 rb_erase(&vma->tree, &vmm->root);
881 }
882
883 static inline void
nvkm_vmm_node_delete(struct nvkm_vmm * vmm,struct nvkm_vma * vma)884 nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
885 {
886 nvkm_vmm_node_remove(vmm, vma);
887 list_del(&vma->head);
888 kfree(vma);
889 }
890
891 static void
nvkm_vmm_node_insert(struct nvkm_vmm * vmm,struct nvkm_vma * vma)892 nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
893 {
894 struct rb_node **ptr = &vmm->root.rb_node;
895 struct rb_node *parent = NULL;
896
897 while (*ptr) {
898 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
899 parent = *ptr;
900 if (vma->addr < this->addr)
901 ptr = &parent->rb_left;
902 else
903 if (vma->addr > this->addr)
904 ptr = &parent->rb_right;
905 else
906 BUG();
907 }
908
909 rb_link_node(&vma->tree, parent, ptr);
910 rb_insert_color(&vma->tree, &vmm->root);
911 }
912
913 struct nvkm_vma *
nvkm_vmm_node_search(struct nvkm_vmm * vmm,u64 addr)914 nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
915 {
916 struct rb_node *node = vmm->root.rb_node;
917 while (node) {
918 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
919 if (addr < vma->addr)
920 node = node->rb_left;
921 else
922 if (addr >= vma->addr + vma->size)
923 node = node->rb_right;
924 else
925 return vma;
926 }
927 return NULL;
928 }
929
930 #define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL : \
931 list_entry((root)->head.dir, struct nvkm_vma, head))
932
933 static struct nvkm_vma *
nvkm_vmm_node_merge(struct nvkm_vmm * vmm,struct nvkm_vma * prev,struct nvkm_vma * vma,struct nvkm_vma * next,u64 size)934 nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
935 struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
936 {
937 if (next) {
938 if (vma->size == size) {
939 vma->size += next->size;
940 nvkm_vmm_node_delete(vmm, next);
941 if (prev) {
942 prev->size += vma->size;
943 nvkm_vmm_node_delete(vmm, vma);
944 return prev;
945 }
946 return vma;
947 }
948 BUG_ON(prev);
949
950 nvkm_vmm_node_remove(vmm, next);
951 vma->size -= size;
952 next->addr -= size;
953 next->size += size;
954 nvkm_vmm_node_insert(vmm, next);
955 return next;
956 }
957
958 if (prev) {
959 if (vma->size != size) {
960 nvkm_vmm_node_remove(vmm, vma);
961 prev->size += size;
962 vma->addr += size;
963 vma->size -= size;
964 nvkm_vmm_node_insert(vmm, vma);
965 } else {
966 prev->size += vma->size;
967 nvkm_vmm_node_delete(vmm, vma);
968 }
969 return prev;
970 }
971
972 return vma;
973 }
974
975 struct nvkm_vma *
nvkm_vmm_node_split(struct nvkm_vmm * vmm,struct nvkm_vma * vma,u64 addr,u64 size)976 nvkm_vmm_node_split(struct nvkm_vmm *vmm,
977 struct nvkm_vma *vma, u64 addr, u64 size)
978 {
979 struct nvkm_vma *prev = NULL;
980
981 if (vma->addr != addr) {
982 prev = vma;
983 if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
984 return NULL;
985 vma->part = true;
986 nvkm_vmm_node_insert(vmm, vma);
987 }
988
989 if (vma->size != size) {
990 struct nvkm_vma *tmp;
991 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
992 nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
993 return NULL;
994 }
995 tmp->part = true;
996 nvkm_vmm_node_insert(vmm, tmp);
997 }
998
999 return vma;
1000 }
1001
1002 static void
nvkm_vma_dump(struct nvkm_vma * vma)1003 nvkm_vma_dump(struct nvkm_vma *vma)
1004 {
1005 printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n",
1006 vma->addr, (u64)vma->size,
1007 vma->used ? '-' : 'F',
1008 vma->mapref ? 'R' : '-',
1009 vma->sparse ? 'S' : '-',
1010 vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
1011 vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
1012 vma->part ? 'P' : '-',
1013 vma->busy ? 'B' : '-',
1014 vma->mapped ? 'M' : '-',
1015 vma->memory);
1016 }
1017
1018 static void
nvkm_vmm_dump(struct nvkm_vmm * vmm)1019 nvkm_vmm_dump(struct nvkm_vmm *vmm)
1020 {
1021 struct nvkm_vma *vma;
1022 list_for_each_entry(vma, &vmm->list, head) {
1023 nvkm_vma_dump(vma);
1024 }
1025 }
1026
1027 static void
nvkm_vmm_dtor(struct nvkm_vmm * vmm)1028 nvkm_vmm_dtor(struct nvkm_vmm *vmm)
1029 {
1030 struct nvkm_vma *vma;
1031 struct rb_node *node;
1032
1033 if (0)
1034 nvkm_vmm_dump(vmm);
1035
1036 while ((node = rb_first(&vmm->root))) {
1037 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
1038 nvkm_vmm_put(vmm, &vma);
1039 }
1040
1041 if (vmm->bootstrapped) {
1042 const struct nvkm_vmm_page *page = vmm->func->page;
1043 const u64 limit = vmm->limit - vmm->start;
1044
1045 while (page[1].shift)
1046 page++;
1047
1048 nvkm_mmu_ptc_dump(vmm->mmu);
1049 nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
1050 }
1051
1052 vma = list_first_entry(&vmm->list, typeof(*vma), head);
1053 list_del(&vma->head);
1054 kfree(vma);
1055 WARN_ON(!list_empty(&vmm->list));
1056
1057 if (vmm->nullp) {
1058 dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
1059 vmm->nullp, vmm->null);
1060 }
1061
1062 if (vmm->pd) {
1063 nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1064 nvkm_vmm_pt_del(&vmm->pd);
1065 }
1066 }
1067
1068 static int
nvkm_vmm_ctor_managed(struct nvkm_vmm * vmm,u64 addr,u64 size)1069 nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
1070 {
1071 struct nvkm_vma *vma;
1072 if (!(vma = nvkm_vma_new(addr, size)))
1073 return -ENOMEM;
1074 vma->mapref = true;
1075 vma->sparse = false;
1076 vma->used = true;
1077 nvkm_vmm_node_insert(vmm, vma);
1078 list_add_tail(&vma->head, &vmm->list);
1079 return 0;
1080 }
1081
1082 static int
nvkm_vmm_ctor(const struct nvkm_vmm_func * func,struct nvkm_mmu * mmu,u32 pd_header,bool managed,u64 addr,u64 size,struct lock_class_key * key,const char * name,struct nvkm_vmm * vmm)1083 nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1084 u32 pd_header, bool managed, u64 addr, u64 size,
1085 struct lock_class_key *key, const char *name,
1086 struct nvkm_vmm *vmm)
1087 {
1088 static struct lock_class_key _key;
1089 const struct nvkm_vmm_page *page = func->page;
1090 const struct nvkm_vmm_desc *desc;
1091 struct nvkm_vma *vma;
1092 int levels, bits = 0, ret;
1093
1094 vmm->func = func;
1095 vmm->mmu = mmu;
1096 vmm->name = name;
1097 vmm->debug = mmu->subdev.debug;
1098 kref_init(&vmm->kref);
1099
1100 __mutex_init(&vmm->mutex.vmm, "&vmm->mutex.vmm", key ? key : &_key);
1101 mutex_init(&vmm->mutex.ref);
1102 mutex_init(&vmm->mutex.map);
1103
1104 /* Locate the smallest page size supported by the backend, it will
1105 * have the deepest nesting of page tables.
1106 */
1107 while (page[1].shift)
1108 page++;
1109
1110 /* Locate the structure that describes the layout of the top-level
1111 * page table, and determine the number of valid bits in a virtual
1112 * address.
1113 */
1114 for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1115 bits += desc->bits;
1116 bits += page->shift;
1117 desc--;
1118
1119 if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1120 return -EINVAL;
1121
1122 /* Allocate top-level page table. */
1123 vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1124 if (!vmm->pd)
1125 return -ENOMEM;
1126 vmm->pd->refs[0] = 1;
1127 INIT_LIST_HEAD(&vmm->join);
1128
1129 /* ... and the GPU storage for it, except on Tesla-class GPUs that
1130 * have the PD embedded in the instance structure.
1131 */
1132 if (desc->size) {
1133 const u32 size = pd_header + desc->size * (1 << desc->bits);
1134 vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1135 if (!vmm->pd->pt[0])
1136 return -ENOMEM;
1137 }
1138
1139 /* Initialise address-space MM. */
1140 INIT_LIST_HEAD(&vmm->list);
1141 vmm->free = RB_ROOT;
1142 vmm->root = RB_ROOT;
1143
1144 if (managed) {
1145 /* Address-space will be managed by the client for the most
1146 * part, except for a specified area where NVKM allocations
1147 * are allowed to be placed.
1148 */
1149 vmm->start = 0;
1150 vmm->limit = 1ULL << bits;
1151 if (addr + size < addr || addr + size > vmm->limit)
1152 return -EINVAL;
1153
1154 /* Client-managed area before the NVKM-managed area. */
1155 if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
1156 return ret;
1157
1158 vmm->managed.p.addr = 0;
1159 vmm->managed.p.size = addr;
1160
1161 /* NVKM-managed area. */
1162 if (size) {
1163 if (!(vma = nvkm_vma_new(addr, size)))
1164 return -ENOMEM;
1165 nvkm_vmm_free_insert(vmm, vma);
1166 list_add_tail(&vma->head, &vmm->list);
1167 }
1168
1169 /* Client-managed area after the NVKM-managed area. */
1170 addr = addr + size;
1171 size = vmm->limit - addr;
1172 if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
1173 return ret;
1174
1175 vmm->managed.n.addr = addr;
1176 vmm->managed.n.size = size;
1177 } else {
1178 /* Address-space fully managed by NVKM, requiring calls to
1179 * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space.
1180 */
1181 vmm->start = addr;
1182 vmm->limit = size ? (addr + size) : (1ULL << bits);
1183 if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
1184 return -EINVAL;
1185
1186 if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1187 return -ENOMEM;
1188
1189 nvkm_vmm_free_insert(vmm, vma);
1190 list_add(&vma->head, &vmm->list);
1191 }
1192
1193 return 0;
1194 }
1195
1196 int
nvkm_vmm_new_(const struct nvkm_vmm_func * func,struct nvkm_mmu * mmu,u32 hdr,bool managed,u64 addr,u64 size,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)1197 nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1198 u32 hdr, bool managed, u64 addr, u64 size,
1199 struct lock_class_key *key, const char *name,
1200 struct nvkm_vmm **pvmm)
1201 {
1202 if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1203 return -ENOMEM;
1204 return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1205 }
1206
1207 static struct nvkm_vma *
nvkm_vmm_pfn_split_merge(struct nvkm_vmm * vmm,struct nvkm_vma * vma,u64 addr,u64 size,u8 page,bool map)1208 nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1209 u64 addr, u64 size, u8 page, bool map)
1210 {
1211 struct nvkm_vma *prev = NULL;
1212 struct nvkm_vma *next = NULL;
1213
1214 if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1215 if (prev->memory || prev->mapped != map)
1216 prev = NULL;
1217 }
1218
1219 if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1220 if (!next->part ||
1221 next->memory || next->mapped != map)
1222 next = NULL;
1223 }
1224
1225 if (prev || next)
1226 return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1227 return nvkm_vmm_node_split(vmm, vma, addr, size);
1228 }
1229
1230 int
nvkm_vmm_pfn_unmap(struct nvkm_vmm * vmm,u64 addr,u64 size)1231 nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1232 {
1233 struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1234 struct nvkm_vma *next;
1235 u64 limit = addr + size;
1236 u64 start = addr;
1237
1238 if (!vma)
1239 return -EINVAL;
1240
1241 do {
1242 if (!vma->mapped || vma->memory)
1243 continue;
1244
1245 size = min(limit - start, vma->size - (start - vma->addr));
1246
1247 nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1248 start, size, false, true);
1249
1250 next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1251 if (!WARN_ON(!next)) {
1252 vma = next;
1253 vma->refd = NVKM_VMA_PAGE_NONE;
1254 vma->mapped = false;
1255 }
1256 } while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1257
1258 return 0;
1259 }
1260
1261 /*TODO:
1262 * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1263 * with inside HMM, which would be a lot nicer for us to deal with.
1264 * - Support for systems without a 4KiB page size.
1265 */
1266 int
nvkm_vmm_pfn_map(struct nvkm_vmm * vmm,u8 shift,u64 addr,u64 size,u64 * pfn)1267 nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1268 {
1269 const struct nvkm_vmm_page *page = vmm->func->page;
1270 struct nvkm_vma *vma, *tmp;
1271 u64 limit = addr + size;
1272 u64 start = addr;
1273 int pm = size >> shift;
1274 int pi = 0;
1275
1276 /* Only support mapping where the page size of the incoming page
1277 * array matches a page size available for direct mapping.
1278 */
1279 while (page->shift && (page->shift != shift ||
1280 page->desc->func->pfn == NULL))
1281 page++;
1282
1283 if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1284 !IS_ALIGNED(size, 1ULL << shift) ||
1285 addr + size < addr || addr + size > vmm->limit) {
1286 VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1287 shift, page->shift, addr, size);
1288 return -EINVAL;
1289 }
1290
1291 if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1292 return -ENOENT;
1293
1294 do {
1295 bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1296 bool mapped = vma->mapped;
1297 u64 size = limit - start;
1298 u64 addr = start;
1299 int pn, ret = 0;
1300
1301 /* Narrow the operation window to cover a single action (page
1302 * should be mapped or not) within a single VMA.
1303 */
1304 for (pn = 0; pi + pn < pm; pn++) {
1305 if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1306 break;
1307 }
1308 size = min_t(u64, size, pn << page->shift);
1309 size = min_t(u64, size, vma->size + vma->addr - addr);
1310
1311 /* Reject any operation to unmanaged regions, and areas that
1312 * have nvkm_memory objects mapped in them already.
1313 */
1314 if (!vma->mapref || vma->memory) {
1315 ret = -EINVAL;
1316 goto next;
1317 }
1318
1319 /* In order to both properly refcount GPU page tables, and
1320 * prevent "normal" mappings and these direct mappings from
1321 * interfering with each other, we need to track contiguous
1322 * ranges that have been mapped with this interface.
1323 *
1324 * Here we attempt to either split an existing VMA so we're
1325 * able to flag the region as either unmapped/mapped, or to
1326 * merge with adjacent VMAs that are already compatible.
1327 *
1328 * If the region is already compatible, nothing is required.
1329 */
1330 if (map != mapped) {
1331 tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1332 page -
1333 vmm->func->page, map);
1334 if (WARN_ON(!tmp)) {
1335 ret = -ENOMEM;
1336 goto next;
1337 }
1338
1339 if ((tmp->mapped = map))
1340 tmp->refd = page - vmm->func->page;
1341 else
1342 tmp->refd = NVKM_VMA_PAGE_NONE;
1343 vma = tmp;
1344 }
1345
1346 /* Update HW page tables. */
1347 if (map) {
1348 struct nvkm_vmm_map args;
1349 args.page = page;
1350 args.pfn = &pfn[pi];
1351
1352 if (!mapped) {
1353 ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1354 size, &args, page->
1355 desc->func->pfn);
1356 } else {
1357 nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1358 page->desc->func->pfn);
1359 }
1360 } else {
1361 if (mapped) {
1362 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1363 false, true);
1364 }
1365 }
1366
1367 next:
1368 /* Iterate to next operation. */
1369 if (vma->addr + vma->size == addr + size)
1370 vma = node(vma, next);
1371 start += size;
1372
1373 if (ret) {
1374 /* Failure is signalled by clearing the valid bit on
1375 * any PFN that couldn't be modified as requested.
1376 */
1377 while (size) {
1378 pfn[pi++] = NVKM_VMM_PFN_NONE;
1379 size -= 1 << page->shift;
1380 }
1381 } else {
1382 pi += size >> page->shift;
1383 }
1384 } while (vma && start < limit);
1385
1386 return 0;
1387 }
1388
1389 void
nvkm_vmm_unmap_region(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1390 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1391 {
1392 struct nvkm_vma *prev = NULL;
1393 struct nvkm_vma *next;
1394
1395 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1396 nvkm_memory_unref(&vma->memory);
1397 vma->mapped = false;
1398
1399 if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1400 prev = NULL;
1401 if ((next = node(vma, next)) && (!next->part || next->mapped))
1402 next = NULL;
1403 nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1404 }
1405
1406 void
nvkm_vmm_unmap_locked(struct nvkm_vmm * vmm,struct nvkm_vma * vma,bool pfn)1407 nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1408 {
1409 const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1410
1411 if (vma->mapref) {
1412 nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1413 vma->refd = NVKM_VMA_PAGE_NONE;
1414 } else {
1415 nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1416 }
1417
1418 nvkm_vmm_unmap_region(vmm, vma);
1419 }
1420
1421 void
nvkm_vmm_unmap(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1422 nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1423 {
1424 if (vma->memory) {
1425 mutex_lock(&vmm->mutex.vmm);
1426 nvkm_vmm_unmap_locked(vmm, vma, false);
1427 mutex_unlock(&vmm->mutex.vmm);
1428 }
1429 }
1430
1431 static int
nvkm_vmm_map_valid(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1432 nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1433 void *argv, u32 argc, struct nvkm_vmm_map *map)
1434 {
1435 switch (nvkm_memory_target(map->memory)) {
1436 case NVKM_MEM_TARGET_VRAM:
1437 if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1438 VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1439 return -EINVAL;
1440 }
1441 break;
1442 case NVKM_MEM_TARGET_HOST:
1443 case NVKM_MEM_TARGET_NCOH:
1444 if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1445 VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1446 return -EINVAL;
1447 }
1448 break;
1449 default:
1450 WARN_ON(1);
1451 return -ENOSYS;
1452 }
1453
1454 if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) ||
1455 !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1456 !IS_ALIGNED( map->offset, 1ULL << map->page->shift) ||
1457 nvkm_memory_page(map->memory) < map->page->shift) {
1458 VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1459 vma->addr, (u64)vma->size, map->offset, map->page->shift,
1460 nvkm_memory_page(map->memory));
1461 return -EINVAL;
1462 }
1463
1464 return vmm->func->valid(vmm, argv, argc, map);
1465 }
1466
1467 static int
nvkm_vmm_map_choose(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1468 nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1469 void *argv, u32 argc, struct nvkm_vmm_map *map)
1470 {
1471 for (map->page = vmm->func->page; map->page->shift; map->page++) {
1472 VMM_DEBUG(vmm, "trying %d", map->page->shift);
1473 if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1474 return 0;
1475 }
1476 return -EINVAL;
1477 }
1478
1479 static int
nvkm_vmm_map_locked(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1480 nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1481 void *argv, u32 argc, struct nvkm_vmm_map *map)
1482 {
1483 nvkm_vmm_pte_func func;
1484 int ret;
1485
1486 map->no_comp = vma->no_comp;
1487
1488 /* Make sure we won't overrun the end of the memory object. */
1489 if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1490 VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1491 nvkm_memory_size(map->memory),
1492 map->offset, (u64)vma->size);
1493 return -EINVAL;
1494 }
1495
1496 /* Check remaining arguments for validity. */
1497 if (vma->page == NVKM_VMA_PAGE_NONE &&
1498 vma->refd == NVKM_VMA_PAGE_NONE) {
1499 /* Find the largest page size we can perform the mapping at. */
1500 const u32 debug = vmm->debug;
1501 vmm->debug = 0;
1502 ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1503 vmm->debug = debug;
1504 if (ret) {
1505 VMM_DEBUG(vmm, "invalid at any page size");
1506 nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1507 return -EINVAL;
1508 }
1509 } else {
1510 /* Page size of the VMA is already pre-determined. */
1511 if (vma->refd != NVKM_VMA_PAGE_NONE)
1512 map->page = &vmm->func->page[vma->refd];
1513 else
1514 map->page = &vmm->func->page[vma->page];
1515
1516 ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1517 if (ret) {
1518 VMM_DEBUG(vmm, "invalid %d\n", ret);
1519 return ret;
1520 }
1521 }
1522
1523 /* Deal with the 'offset' argument, and fetch the backend function. */
1524 map->off = map->offset;
1525 if (map->mem) {
1526 for (; map->off; map->mem = map->mem->next) {
1527 u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1528 if (size > map->off)
1529 break;
1530 map->off -= size;
1531 }
1532 func = map->page->desc->func->mem;
1533 } else
1534 if (map->sgl) {
1535 for (; map->off; map->sgl = sg_next(map->sgl)) {
1536 u64 size = sg_dma_len(map->sgl);
1537 if (size > map->off)
1538 break;
1539 map->off -= size;
1540 }
1541 func = map->page->desc->func->sgl;
1542 } else {
1543 map->dma += map->offset >> PAGE_SHIFT;
1544 map->off = map->offset & PAGE_MASK;
1545 func = map->page->desc->func->dma;
1546 }
1547
1548 /* Perform the map. */
1549 if (vma->refd == NVKM_VMA_PAGE_NONE) {
1550 ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1551 if (ret)
1552 return ret;
1553
1554 vma->refd = map->page - vmm->func->page;
1555 } else {
1556 nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1557 }
1558
1559 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1560 nvkm_memory_unref(&vma->memory);
1561 vma->memory = nvkm_memory_ref(map->memory);
1562 vma->mapped = true;
1563 vma->tags = map->tags;
1564 return 0;
1565 }
1566
1567 int
nvkm_vmm_map(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1568 nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1569 struct nvkm_vmm_map *map)
1570 {
1571 int ret;
1572
1573 if (nvkm_vmm_in_managed_range(vmm, vma->addr, vma->size) &&
1574 vmm->managed.raw)
1575 return nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1576
1577 mutex_lock(&vmm->mutex.vmm);
1578 ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1579 vma->busy = false;
1580 mutex_unlock(&vmm->mutex.vmm);
1581 return ret;
1582 }
1583
1584 static void
nvkm_vmm_put_region(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1585 nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1586 {
1587 struct nvkm_vma *prev, *next;
1588
1589 if ((prev = node(vma, prev)) && !prev->used) {
1590 vma->addr = prev->addr;
1591 vma->size += prev->size;
1592 nvkm_vmm_free_delete(vmm, prev);
1593 }
1594
1595 if ((next = node(vma, next)) && !next->used) {
1596 vma->size += next->size;
1597 nvkm_vmm_free_delete(vmm, next);
1598 }
1599
1600 nvkm_vmm_free_insert(vmm, vma);
1601 }
1602
1603 void
nvkm_vmm_put_locked(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1604 nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1605 {
1606 const struct nvkm_vmm_page *page = vmm->func->page;
1607 struct nvkm_vma *next = vma;
1608
1609 BUG_ON(vma->part);
1610
1611 if (vma->mapref || !vma->sparse) {
1612 do {
1613 const bool mem = next->memory != NULL;
1614 const bool map = next->mapped;
1615 const u8 refd = next->refd;
1616 const u64 addr = next->addr;
1617 u64 size = next->size;
1618
1619 /* Merge regions that are in the same state. */
1620 while ((next = node(next, next)) && next->part &&
1621 (next->mapped == map) &&
1622 (next->memory != NULL) == mem &&
1623 (next->refd == refd))
1624 size += next->size;
1625
1626 if (map) {
1627 /* Region(s) are mapped, merge the unmap
1628 * and dereference into a single walk of
1629 * the page tree.
1630 */
1631 nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1632 size, vma->sparse,
1633 !mem);
1634 } else
1635 if (refd != NVKM_VMA_PAGE_NONE) {
1636 /* Drop allocation-time PTE references. */
1637 nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1638 }
1639 } while (next && next->part);
1640 }
1641
1642 /* Merge any mapped regions that were split from the initial
1643 * address-space allocation back into the allocated VMA, and
1644 * release memory/compression resources.
1645 */
1646 next = vma;
1647 do {
1648 if (next->mapped)
1649 nvkm_vmm_unmap_region(vmm, next);
1650 } while ((next = node(vma, next)) && next->part);
1651
1652 if (vma->sparse && !vma->mapref) {
1653 /* Sparse region that was allocated with a fixed page size,
1654 * meaning all relevant PTEs were referenced once when the
1655 * region was allocated, and remained that way, regardless
1656 * of whether memory was mapped into it afterwards.
1657 *
1658 * The process of unmapping, unsparsing, and dereferencing
1659 * PTEs can be done in a single page tree walk.
1660 */
1661 nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1662 } else
1663 if (vma->sparse) {
1664 /* Sparse region that wasn't allocated with a fixed page size,
1665 * PTE references were taken both at allocation time (to make
1666 * the GPU see the region as sparse), and when mapping memory
1667 * into the region.
1668 *
1669 * The latter was handled above, and the remaining references
1670 * are dealt with here.
1671 */
1672 nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1673 }
1674
1675 /* Remove VMA from the list of allocated nodes. */
1676 nvkm_vmm_node_remove(vmm, vma);
1677
1678 /* Merge VMA back into the free list. */
1679 vma->page = NVKM_VMA_PAGE_NONE;
1680 vma->refd = NVKM_VMA_PAGE_NONE;
1681 vma->used = false;
1682 nvkm_vmm_put_region(vmm, vma);
1683 }
1684
1685 void
nvkm_vmm_put(struct nvkm_vmm * vmm,struct nvkm_vma ** pvma)1686 nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1687 {
1688 struct nvkm_vma *vma = *pvma;
1689 if (vma) {
1690 mutex_lock(&vmm->mutex.vmm);
1691 nvkm_vmm_put_locked(vmm, vma);
1692 mutex_unlock(&vmm->mutex.vmm);
1693 *pvma = NULL;
1694 }
1695 }
1696
1697 int
nvkm_vmm_get_locked(struct nvkm_vmm * vmm,bool getref,bool mapref,bool sparse,u8 shift,u8 align,u64 size,struct nvkm_vma ** pvma)1698 nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1699 u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1700 {
1701 const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1702 struct rb_node *node = NULL, *temp;
1703 struct nvkm_vma *vma = NULL, *tmp;
1704 u64 addr, tail;
1705 int ret;
1706
1707 VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1708 "shift: %d align: %d size: %016llx",
1709 getref, mapref, sparse, shift, align, size);
1710
1711 /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1712 if (unlikely(!size || (!getref && !mapref && sparse))) {
1713 VMM_DEBUG(vmm, "args %016llx %d %d %d",
1714 size, getref, mapref, sparse);
1715 return -EINVAL;
1716 }
1717
1718 /* Tesla-class GPUs can only select page size per-PDE, which means
1719 * we're required to know the mapping granularity up-front to find
1720 * a suitable region of address-space.
1721 *
1722 * The same goes if we're requesting up-front allocation of PTES.
1723 */
1724 if (unlikely((getref || vmm->func->page_block) && !shift)) {
1725 VMM_DEBUG(vmm, "page size required: %d %016llx",
1726 getref, vmm->func->page_block);
1727 return -EINVAL;
1728 }
1729
1730 /* If a specific page size was requested, determine its index and
1731 * make sure the requested size is a multiple of the page size.
1732 */
1733 if (shift) {
1734 for (page = vmm->func->page; page->shift; page++) {
1735 if (shift == page->shift)
1736 break;
1737 }
1738
1739 if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1740 VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1741 return -EINVAL;
1742 }
1743 align = max_t(u8, align, shift);
1744 } else {
1745 align = max_t(u8, align, 12);
1746 }
1747
1748 /* Locate smallest block that can possibly satisfy the allocation. */
1749 temp = vmm->free.rb_node;
1750 while (temp) {
1751 struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1752 if (this->size < size) {
1753 temp = temp->rb_right;
1754 } else {
1755 node = temp;
1756 temp = temp->rb_left;
1757 }
1758 }
1759
1760 if (unlikely(!node))
1761 return -ENOSPC;
1762
1763 /* Take into account alignment restrictions, trying larger blocks
1764 * in turn until we find a suitable free block.
1765 */
1766 do {
1767 struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1768 struct nvkm_vma *prev = node(this, prev);
1769 struct nvkm_vma *next = node(this, next);
1770 const int p = page - vmm->func->page;
1771
1772 addr = this->addr;
1773 if (vmm->func->page_block && prev && prev->page != p)
1774 addr = ALIGN(addr, vmm->func->page_block);
1775 addr = ALIGN(addr, 1ULL << align);
1776
1777 tail = this->addr + this->size;
1778 if (vmm->func->page_block && next && next->page != p)
1779 tail = ALIGN_DOWN(tail, vmm->func->page_block);
1780
1781 if (addr <= tail && tail - addr >= size) {
1782 nvkm_vmm_free_remove(vmm, this);
1783 vma = this;
1784 break;
1785 }
1786 } while ((node = rb_next(node)));
1787
1788 if (unlikely(!vma))
1789 return -ENOSPC;
1790
1791 /* If the VMA we found isn't already exactly the requested size,
1792 * it needs to be split, and the remaining free blocks returned.
1793 */
1794 if (addr != vma->addr) {
1795 if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1796 nvkm_vmm_put_region(vmm, vma);
1797 return -ENOMEM;
1798 }
1799 nvkm_vmm_free_insert(vmm, vma);
1800 vma = tmp;
1801 }
1802
1803 if (size != vma->size) {
1804 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1805 nvkm_vmm_put_region(vmm, vma);
1806 return -ENOMEM;
1807 }
1808 nvkm_vmm_free_insert(vmm, tmp);
1809 }
1810
1811 /* Pre-allocate page tables and/or setup sparse mappings. */
1812 if (sparse && getref)
1813 ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1814 else if (sparse)
1815 ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1816 else if (getref)
1817 ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1818 else
1819 ret = 0;
1820 if (ret) {
1821 nvkm_vmm_put_region(vmm, vma);
1822 return ret;
1823 }
1824
1825 vma->mapref = mapref && !getref;
1826 vma->sparse = sparse;
1827 vma->page = page - vmm->func->page;
1828 vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1829 vma->used = true;
1830 nvkm_vmm_node_insert(vmm, vma);
1831 *pvma = vma;
1832 return 0;
1833 }
1834
1835 int
nvkm_vmm_get(struct nvkm_vmm * vmm,u8 page,u64 size,struct nvkm_vma ** pvma)1836 nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1837 {
1838 int ret;
1839 mutex_lock(&vmm->mutex.vmm);
1840 ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1841 mutex_unlock(&vmm->mutex.vmm);
1842 return ret;
1843 }
1844
1845 void
nvkm_vmm_raw_unmap(struct nvkm_vmm * vmm,u64 addr,u64 size,bool sparse,u8 refd)1846 nvkm_vmm_raw_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size,
1847 bool sparse, u8 refd)
1848 {
1849 const struct nvkm_vmm_page *page = &vmm->func->page[refd];
1850
1851 nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, false);
1852 }
1853
1854 void
nvkm_vmm_raw_put(struct nvkm_vmm * vmm,u64 addr,u64 size,u8 refd)1855 nvkm_vmm_raw_put(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1856 {
1857 const struct nvkm_vmm_page *page = vmm->func->page;
1858
1859 nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1860 }
1861
1862 int
nvkm_vmm_raw_get(struct nvkm_vmm * vmm,u64 addr,u64 size,u8 refd)1863 nvkm_vmm_raw_get(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1864 {
1865 const struct nvkm_vmm_page *page = vmm->func->page;
1866
1867 if (unlikely(!size))
1868 return -EINVAL;
1869
1870 return nvkm_vmm_ptes_get(vmm, &page[refd], addr, size);
1871 }
1872
1873 int
nvkm_vmm_raw_sparse(struct nvkm_vmm * vmm,u64 addr,u64 size,bool ref)1874 nvkm_vmm_raw_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
1875 {
1876 int ret;
1877
1878 mutex_lock(&vmm->mutex.ref);
1879 ret = nvkm_vmm_ptes_sparse(vmm, addr, size, ref);
1880 mutex_unlock(&vmm->mutex.ref);
1881
1882 return ret;
1883 }
1884
1885 void
nvkm_vmm_part(struct nvkm_vmm * vmm,struct nvkm_memory * inst)1886 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1887 {
1888 if (inst && vmm && vmm->func->part) {
1889 mutex_lock(&vmm->mutex.vmm);
1890 vmm->func->part(vmm, inst);
1891 mutex_unlock(&vmm->mutex.vmm);
1892 }
1893 }
1894
1895 int
nvkm_vmm_join(struct nvkm_vmm * vmm,struct nvkm_memory * inst)1896 nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1897 {
1898 int ret = 0;
1899 if (vmm->func->join) {
1900 mutex_lock(&vmm->mutex.vmm);
1901 ret = vmm->func->join(vmm, inst);
1902 mutex_unlock(&vmm->mutex.vmm);
1903 }
1904 return ret;
1905 }
1906
1907 static bool
nvkm_vmm_boot_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)1908 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1909 {
1910 const struct nvkm_vmm_desc *desc = it->desc;
1911 const int type = desc->type == SPT;
1912 nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1913 return false;
1914 }
1915
1916 int
nvkm_vmm_boot(struct nvkm_vmm * vmm)1917 nvkm_vmm_boot(struct nvkm_vmm *vmm)
1918 {
1919 const struct nvkm_vmm_page *page = vmm->func->page;
1920 const u64 limit = vmm->limit - vmm->start;
1921 int ret;
1922
1923 while (page[1].shift)
1924 page++;
1925
1926 ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1927 if (ret)
1928 return ret;
1929
1930 nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1931 nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1932 vmm->bootstrapped = true;
1933 return 0;
1934 }
1935
1936 static void
nvkm_vmm_del(struct kref * kref)1937 nvkm_vmm_del(struct kref *kref)
1938 {
1939 struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1940 nvkm_vmm_dtor(vmm);
1941 kfree(vmm);
1942 }
1943
1944 void
nvkm_vmm_unref(struct nvkm_vmm ** pvmm)1945 nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1946 {
1947 struct nvkm_vmm *vmm = *pvmm;
1948 if (vmm) {
1949 kref_put(&vmm->kref, nvkm_vmm_del);
1950 *pvmm = NULL;
1951 }
1952 }
1953
1954 struct nvkm_vmm *
nvkm_vmm_ref(struct nvkm_vmm * vmm)1955 nvkm_vmm_ref(struct nvkm_vmm *vmm)
1956 {
1957 if (vmm)
1958 kref_get(&vmm->kref);
1959 return vmm;
1960 }
1961
1962 int
nvkm_vmm_new(struct nvkm_device * device,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)1963 nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1964 u32 argc, struct lock_class_key *key, const char *name,
1965 struct nvkm_vmm **pvmm)
1966 {
1967 struct nvkm_mmu *mmu = device->mmu;
1968 struct nvkm_vmm *vmm = NULL;
1969 int ret;
1970 ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
1971 key, name, &vmm);
1972 if (ret)
1973 nvkm_vmm_unref(&vmm);
1974 *pvmm = vmm;
1975 return ret;
1976 }
1977