1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <drm/ttm/ttm_range_manager.h>
27
28 #include "amdgpu.h"
29 #include "amdgpu_vm.h"
30 #include "amdgpu_res_cursor.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "atom.h"
33
34 struct amdgpu_vram_reservation {
35 struct list_head node;
36 struct drm_mm_node mm_node;
37 };
38
39 static inline struct amdgpu_vram_mgr *
to_vram_mgr(struct ttm_resource_manager * man)40 to_vram_mgr(struct ttm_resource_manager *man)
41 {
42 return container_of(man, struct amdgpu_vram_mgr, manager);
43 }
44
45 static inline struct amdgpu_device *
to_amdgpu_device(struct amdgpu_vram_mgr * mgr)46 to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
47 {
48 return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
49 }
50
51 /**
52 * DOC: mem_info_vram_total
53 *
54 * The amdgpu driver provides a sysfs API for reporting current total VRAM
55 * available on the device
56 * The file mem_info_vram_total is used for this and returns the total
57 * amount of VRAM in bytes
58 */
amdgpu_mem_info_vram_total_show(struct device * dev,struct device_attribute * attr,char * buf)59 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
60 struct device_attribute *attr, char *buf)
61 {
62 struct drm_device *ddev = dev_get_drvdata(dev);
63 struct amdgpu_device *adev = drm_to_adev(ddev);
64
65 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size);
66 }
67
68 /**
69 * DOC: mem_info_vis_vram_total
70 *
71 * The amdgpu driver provides a sysfs API for reporting current total
72 * visible VRAM available on the device
73 * The file mem_info_vis_vram_total is used for this and returns the total
74 * amount of visible VRAM in bytes
75 */
amdgpu_mem_info_vis_vram_total_show(struct device * dev,struct device_attribute * attr,char * buf)76 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
77 struct device_attribute *attr, char *buf)
78 {
79 struct drm_device *ddev = dev_get_drvdata(dev);
80 struct amdgpu_device *adev = drm_to_adev(ddev);
81
82 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size);
83 }
84
85 /**
86 * DOC: mem_info_vram_used
87 *
88 * The amdgpu driver provides a sysfs API for reporting current total VRAM
89 * available on the device
90 * The file mem_info_vram_used is used for this and returns the total
91 * amount of currently used VRAM in bytes
92 */
amdgpu_mem_info_vram_used_show(struct device * dev,struct device_attribute * attr,char * buf)93 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
94 struct device_attribute *attr,
95 char *buf)
96 {
97 struct drm_device *ddev = dev_get_drvdata(dev);
98 struct amdgpu_device *adev = drm_to_adev(ddev);
99 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager;
100
101 return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man));
102 }
103
104 /**
105 * DOC: mem_info_vis_vram_used
106 *
107 * The amdgpu driver provides a sysfs API for reporting current total of
108 * used visible VRAM
109 * The file mem_info_vis_vram_used is used for this and returns the total
110 * amount of currently used visible VRAM in bytes
111 */
amdgpu_mem_info_vis_vram_used_show(struct device * dev,struct device_attribute * attr,char * buf)112 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
113 struct device_attribute *attr,
114 char *buf)
115 {
116 struct drm_device *ddev = dev_get_drvdata(dev);
117 struct amdgpu_device *adev = drm_to_adev(ddev);
118
119 return sysfs_emit(buf, "%llu\n",
120 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr));
121 }
122
123 /**
124 * DOC: mem_info_vram_vendor
125 *
126 * The amdgpu driver provides a sysfs API for reporting the vendor of the
127 * installed VRAM
128 * The file mem_info_vram_vendor is used for this and returns the name of the
129 * vendor.
130 */
amdgpu_mem_info_vram_vendor(struct device * dev,struct device_attribute * attr,char * buf)131 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
132 struct device_attribute *attr,
133 char *buf)
134 {
135 struct drm_device *ddev = dev_get_drvdata(dev);
136 struct amdgpu_device *adev = drm_to_adev(ddev);
137
138 switch (adev->gmc.vram_vendor) {
139 case SAMSUNG:
140 return sysfs_emit(buf, "samsung\n");
141 case INFINEON:
142 return sysfs_emit(buf, "infineon\n");
143 case ELPIDA:
144 return sysfs_emit(buf, "elpida\n");
145 case ETRON:
146 return sysfs_emit(buf, "etron\n");
147 case NANYA:
148 return sysfs_emit(buf, "nanya\n");
149 case HYNIX:
150 return sysfs_emit(buf, "hynix\n");
151 case MOSEL:
152 return sysfs_emit(buf, "mosel\n");
153 case WINBOND:
154 return sysfs_emit(buf, "winbond\n");
155 case ESMT:
156 return sysfs_emit(buf, "esmt\n");
157 case MICRON:
158 return sysfs_emit(buf, "micron\n");
159 default:
160 return sysfs_emit(buf, "unknown\n");
161 }
162 }
163
164 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
165 amdgpu_mem_info_vram_total_show, NULL);
166 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
167 amdgpu_mem_info_vis_vram_total_show,NULL);
168 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
169 amdgpu_mem_info_vram_used_show, NULL);
170 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
171 amdgpu_mem_info_vis_vram_used_show, NULL);
172 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
173 amdgpu_mem_info_vram_vendor, NULL);
174
175 static struct attribute *amdgpu_vram_mgr_attributes[] = {
176 &dev_attr_mem_info_vram_total.attr,
177 &dev_attr_mem_info_vis_vram_total.attr,
178 &dev_attr_mem_info_vram_used.attr,
179 &dev_attr_mem_info_vis_vram_used.attr,
180 &dev_attr_mem_info_vram_vendor.attr,
181 NULL
182 };
183
184 const struct attribute_group amdgpu_vram_mgr_attr_group = {
185 .attrs = amdgpu_vram_mgr_attributes
186 };
187
188 /**
189 * amdgpu_vram_mgr_vis_size - Calculate visible node size
190 *
191 * @adev: amdgpu_device pointer
192 * @node: MM node structure
193 *
194 * Calculate how many bytes of the MM node are inside visible VRAM
195 */
amdgpu_vram_mgr_vis_size(struct amdgpu_device * adev,struct drm_mm_node * node)196 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
197 struct drm_mm_node *node)
198 {
199 uint64_t start = node->start << PAGE_SHIFT;
200 uint64_t end = (node->size + node->start) << PAGE_SHIFT;
201
202 if (start >= adev->gmc.visible_vram_size)
203 return 0;
204
205 return (end > adev->gmc.visible_vram_size ?
206 adev->gmc.visible_vram_size : end) - start;
207 }
208
209 /**
210 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
211 *
212 * @bo: &amdgpu_bo buffer object (must be in VRAM)
213 *
214 * Returns:
215 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
216 */
amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo * bo)217 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
218 {
219 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
220 struct ttm_resource *res = bo->tbo.resource;
221 unsigned pages = res->num_pages;
222 struct drm_mm_node *mm;
223 u64 usage;
224
225 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
226 return amdgpu_bo_size(bo);
227
228 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
229 return 0;
230
231 mm = &container_of(res, struct ttm_range_mgr_node, base)->mm_nodes[0];
232 for (usage = 0; pages; pages -= mm->size, mm++)
233 usage += amdgpu_vram_mgr_vis_size(adev, mm);
234
235 return usage;
236 }
237
238 /* Commit the reservation of VRAM pages */
amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager * man)239 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
240 {
241 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
242 struct amdgpu_device *adev = to_amdgpu_device(mgr);
243 struct drm_mm *mm = &mgr->mm;
244 struct amdgpu_vram_reservation *rsv, *temp;
245 uint64_t vis_usage;
246
247 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) {
248 if (drm_mm_reserve_node(mm, &rsv->mm_node))
249 continue;
250
251 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
252 rsv->mm_node.start, rsv->mm_node.size);
253
254 vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node);
255 atomic64_add(vis_usage, &mgr->vis_usage);
256 spin_lock(&man->bdev->lru_lock);
257 man->usage += rsv->mm_node.size << PAGE_SHIFT;
258 spin_unlock(&man->bdev->lru_lock);
259 list_move(&rsv->node, &mgr->reserved_pages);
260 }
261 }
262
263 /**
264 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
265 *
266 * @mgr: amdgpu_vram_mgr pointer
267 * @start: start address of the range in VRAM
268 * @size: size of the range
269 *
270 * Reserve memory from start address with the specified size in VRAM
271 */
amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr * mgr,uint64_t start,uint64_t size)272 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
273 uint64_t start, uint64_t size)
274 {
275 struct amdgpu_vram_reservation *rsv;
276
277 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
278 if (!rsv)
279 return -ENOMEM;
280
281 INIT_LIST_HEAD(&rsv->node);
282 rsv->mm_node.start = start >> PAGE_SHIFT;
283 rsv->mm_node.size = size >> PAGE_SHIFT;
284
285 spin_lock(&mgr->lock);
286 list_add_tail(&rsv->node, &mgr->reservations_pending);
287 amdgpu_vram_mgr_do_reserve(&mgr->manager);
288 spin_unlock(&mgr->lock);
289
290 return 0;
291 }
292
293 /**
294 * amdgpu_vram_mgr_query_page_status - query the reservation status
295 *
296 * @mgr: amdgpu_vram_mgr pointer
297 * @start: start address of a page in VRAM
298 *
299 * Returns:
300 * -EBUSY: the page is still hold and in pending list
301 * 0: the page has been reserved
302 * -ENOENT: the input page is not a reservation
303 */
amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr * mgr,uint64_t start)304 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
305 uint64_t start)
306 {
307 struct amdgpu_vram_reservation *rsv;
308 int ret;
309
310 spin_lock(&mgr->lock);
311
312 list_for_each_entry(rsv, &mgr->reservations_pending, node) {
313 if ((rsv->mm_node.start <= start) &&
314 (start < (rsv->mm_node.start + rsv->mm_node.size))) {
315 ret = -EBUSY;
316 goto out;
317 }
318 }
319
320 list_for_each_entry(rsv, &mgr->reserved_pages, node) {
321 if ((rsv->mm_node.start <= start) &&
322 (start < (rsv->mm_node.start + rsv->mm_node.size))) {
323 ret = 0;
324 goto out;
325 }
326 }
327
328 ret = -ENOENT;
329 out:
330 spin_unlock(&mgr->lock);
331 return ret;
332 }
333
334 /**
335 * amdgpu_vram_mgr_virt_start - update virtual start address
336 *
337 * @mem: ttm_resource to update
338 * @node: just allocated node
339 *
340 * Calculate a virtual BO start address to easily check if everything is CPU
341 * accessible.
342 */
amdgpu_vram_mgr_virt_start(struct ttm_resource * mem,struct drm_mm_node * node)343 static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem,
344 struct drm_mm_node *node)
345 {
346 unsigned long start;
347
348 start = node->start + node->size;
349 if (start > mem->num_pages)
350 start -= mem->num_pages;
351 else
352 start = 0;
353 mem->start = max(mem->start, start);
354 }
355
356 /**
357 * amdgpu_vram_mgr_new - allocate new ranges
358 *
359 * @man: TTM memory type manager
360 * @tbo: TTM BO we need this range for
361 * @place: placement flags and restrictions
362 * @res: the resulting mem object
363 *
364 * Allocate VRAM for the given BO.
365 */
amdgpu_vram_mgr_new(struct ttm_resource_manager * man,struct ttm_buffer_object * tbo,const struct ttm_place * place,struct ttm_resource ** res)366 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
367 struct ttm_buffer_object *tbo,
368 const struct ttm_place *place,
369 struct ttm_resource **res)
370 {
371 unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages;
372 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
373 struct amdgpu_device *adev = to_amdgpu_device(mgr);
374 uint64_t vis_usage = 0, mem_bytes, max_bytes;
375 struct ttm_range_mgr_node *node;
376 struct drm_mm *mm = &mgr->mm;
377 enum drm_mm_insert_mode mode;
378 unsigned i;
379 int r;
380
381 lpfn = place->lpfn;
382 if (!lpfn)
383 lpfn = man->size >> PAGE_SHIFT;
384
385 max_bytes = adev->gmc.mc_vram_size;
386 if (tbo->type != ttm_bo_type_kernel)
387 max_bytes -= AMDGPU_VM_RESERVED_VRAM;
388
389 mem_bytes = tbo->base.size;
390 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
391 pages_per_node = ~0ul;
392 num_nodes = 1;
393 } else {
394 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
395 pages_per_node = HPAGE_PMD_NR;
396 #else
397 /* default to 2MB */
398 pages_per_node = 2UL << (20UL - PAGE_SHIFT);
399 #endif
400 pages_per_node = max_t(uint32_t, pages_per_node,
401 tbo->page_alignment);
402 num_nodes = DIV_ROUND_UP_ULL(PFN_UP(mem_bytes), pages_per_node);
403 }
404
405 node = kvmalloc(struct_size(node, mm_nodes, num_nodes),
406 GFP_KERNEL | __GFP_ZERO);
407 if (!node)
408 return -ENOMEM;
409
410 ttm_resource_init(tbo, place, &node->base);
411
412 /* bail out quickly if there's likely not enough VRAM for this BO */
413 if (ttm_resource_manager_usage(man) > max_bytes) {
414 r = -ENOSPC;
415 goto error_fini;
416 }
417
418 mode = DRM_MM_INSERT_BEST;
419 if (place->flags & TTM_PL_FLAG_TOPDOWN)
420 mode = DRM_MM_INSERT_HIGH;
421
422 pages_left = node->base.num_pages;
423
424 /* Limit maximum size to 2GB due to SG table limitations */
425 pages = min(pages_left, 2UL << (30 - PAGE_SHIFT));
426
427 i = 0;
428 spin_lock(&mgr->lock);
429 while (pages_left) {
430 uint32_t alignment = tbo->page_alignment;
431
432 if (pages >= pages_per_node)
433 alignment = pages_per_node;
434
435 r = drm_mm_insert_node_in_range(mm, &node->mm_nodes[i], pages,
436 alignment, 0, place->fpfn,
437 lpfn, mode);
438 if (unlikely(r)) {
439 if (pages > pages_per_node) {
440 if (is_power_of_2(pages))
441 pages = pages / 2;
442 else
443 pages = rounddown_pow_of_two(pages);
444 continue;
445 }
446 goto error_free;
447 }
448
449 vis_usage += amdgpu_vram_mgr_vis_size(adev, &node->mm_nodes[i]);
450 amdgpu_vram_mgr_virt_start(&node->base, &node->mm_nodes[i]);
451 pages_left -= pages;
452 ++i;
453
454 if (pages > pages_left)
455 pages = pages_left;
456 }
457 spin_unlock(&mgr->lock);
458
459 if (i == 1)
460 node->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
461
462 if (adev->gmc.xgmi.connected_to_cpu)
463 node->base.bus.caching = ttm_cached;
464 else
465 node->base.bus.caching = ttm_write_combined;
466
467 atomic64_add(vis_usage, &mgr->vis_usage);
468 *res = &node->base;
469 return 0;
470
471 error_free:
472 while (i--)
473 drm_mm_remove_node(&node->mm_nodes[i]);
474 spin_unlock(&mgr->lock);
475 error_fini:
476 ttm_resource_fini(man, &node->base);
477 kvfree(node);
478
479 return r;
480 }
481
482 /**
483 * amdgpu_vram_mgr_del - free ranges
484 *
485 * @man: TTM memory type manager
486 * @res: TTM memory object
487 *
488 * Free the allocated VRAM again.
489 */
amdgpu_vram_mgr_del(struct ttm_resource_manager * man,struct ttm_resource * res)490 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
491 struct ttm_resource *res)
492 {
493 struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res);
494 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
495 struct amdgpu_device *adev = to_amdgpu_device(mgr);
496 uint64_t vis_usage = 0;
497 unsigned i, pages;
498
499 spin_lock(&mgr->lock);
500 for (i = 0, pages = res->num_pages; pages;
501 pages -= node->mm_nodes[i].size, ++i) {
502 struct drm_mm_node *mm = &node->mm_nodes[i];
503
504 drm_mm_remove_node(mm);
505 vis_usage += amdgpu_vram_mgr_vis_size(adev, mm);
506 }
507 amdgpu_vram_mgr_do_reserve(man);
508 spin_unlock(&mgr->lock);
509
510 atomic64_sub(vis_usage, &mgr->vis_usage);
511
512 ttm_resource_fini(man, res);
513 kvfree(node);
514 }
515
516 /**
517 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
518 *
519 * @adev: amdgpu device pointer
520 * @res: TTM memory object
521 * @offset: byte offset from the base of VRAM BO
522 * @length: number of bytes to export in sg_table
523 * @dev: the other device
524 * @dir: dma direction
525 * @sgt: resulting sg table
526 *
527 * Allocate and fill a sg table from a VRAM allocation.
528 */
amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device * adev,struct ttm_resource * res,u64 offset,u64 length,struct device * dev,enum dma_data_direction dir,struct sg_table ** sgt)529 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
530 struct ttm_resource *res,
531 u64 offset, u64 length,
532 struct device *dev,
533 enum dma_data_direction dir,
534 struct sg_table **sgt)
535 {
536 struct amdgpu_res_cursor cursor;
537 struct scatterlist *sg;
538 int num_entries = 0;
539 int i, r;
540
541 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
542 if (!*sgt)
543 return -ENOMEM;
544
545 /* Determine the number of DRM_MM nodes to export */
546 amdgpu_res_first(res, offset, length, &cursor);
547 while (cursor.remaining) {
548 num_entries++;
549 amdgpu_res_next(&cursor, cursor.size);
550 }
551
552 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
553 if (r)
554 goto error_free;
555
556 /* Initialize scatterlist nodes of sg_table */
557 for_each_sgtable_sg((*sgt), sg, i)
558 sg->length = 0;
559
560 /*
561 * Walk down DRM_MM nodes to populate scatterlist nodes
562 * @note: Use iterator api to get first the DRM_MM node
563 * and the number of bytes from it. Access the following
564 * DRM_MM node(s) if more buffer needs to exported
565 */
566 amdgpu_res_first(res, offset, length, &cursor);
567 for_each_sgtable_sg((*sgt), sg, i) {
568 phys_addr_t phys = cursor.start + adev->gmc.aper_base;
569 size_t size = cursor.size;
570 dma_addr_t addr;
571
572 addr = dma_map_resource(dev, phys, size, dir,
573 DMA_ATTR_SKIP_CPU_SYNC);
574 r = dma_mapping_error(dev, addr);
575 if (r)
576 goto error_unmap;
577
578 sg_set_page(sg, NULL, size, 0);
579 sg_dma_address(sg) = addr;
580 sg_dma_len(sg) = size;
581
582 amdgpu_res_next(&cursor, cursor.size);
583 }
584
585 return 0;
586
587 error_unmap:
588 for_each_sgtable_sg((*sgt), sg, i) {
589 if (!sg->length)
590 continue;
591
592 dma_unmap_resource(dev, sg->dma_address,
593 sg->length, dir,
594 DMA_ATTR_SKIP_CPU_SYNC);
595 }
596 sg_free_table(*sgt);
597
598 error_free:
599 kfree(*sgt);
600 return r;
601 }
602
603 /**
604 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
605 *
606 * @dev: device pointer
607 * @dir: data direction of resource to unmap
608 * @sgt: sg table to free
609 *
610 * Free a previously allocate sg table.
611 */
amdgpu_vram_mgr_free_sgt(struct device * dev,enum dma_data_direction dir,struct sg_table * sgt)612 void amdgpu_vram_mgr_free_sgt(struct device *dev,
613 enum dma_data_direction dir,
614 struct sg_table *sgt)
615 {
616 struct scatterlist *sg;
617 int i;
618
619 for_each_sgtable_sg(sgt, sg, i)
620 dma_unmap_resource(dev, sg->dma_address,
621 sg->length, dir,
622 DMA_ATTR_SKIP_CPU_SYNC);
623 sg_free_table(sgt);
624 kfree(sgt);
625 }
626
627 /**
628 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
629 *
630 * @mgr: amdgpu_vram_mgr pointer
631 *
632 * Returns how many bytes are used in the visible part of VRAM
633 */
amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr * mgr)634 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
635 {
636 return atomic64_read(&mgr->vis_usage);
637 }
638
639 /**
640 * amdgpu_vram_mgr_debug - dump VRAM table
641 *
642 * @man: TTM memory type manager
643 * @printer: DRM printer to use
644 *
645 * Dump the table content using printk.
646 */
amdgpu_vram_mgr_debug(struct ttm_resource_manager * man,struct drm_printer * printer)647 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
648 struct drm_printer *printer)
649 {
650 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
651
652 drm_printf(printer, " vis usage:%llu\n",
653 amdgpu_vram_mgr_vis_usage(mgr));
654
655 spin_lock(&mgr->lock);
656 drm_mm_print(&mgr->mm, printer);
657 spin_unlock(&mgr->lock);
658 }
659
660 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
661 .alloc = amdgpu_vram_mgr_new,
662 .free = amdgpu_vram_mgr_del,
663 .debug = amdgpu_vram_mgr_debug
664 };
665
666 /**
667 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
668 *
669 * @adev: amdgpu_device pointer
670 *
671 * Allocate and initialize the VRAM manager.
672 */
amdgpu_vram_mgr_init(struct amdgpu_device * adev)673 int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
674 {
675 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
676 struct ttm_resource_manager *man = &mgr->manager;
677
678 ttm_resource_manager_init(man, &adev->mman.bdev,
679 adev->gmc.real_vram_size);
680
681 man->func = &amdgpu_vram_mgr_func;
682
683 drm_mm_init(&mgr->mm, 0, man->size >> PAGE_SHIFT);
684 spin_lock_init(&mgr->lock);
685 INIT_LIST_HEAD(&mgr->reservations_pending);
686 INIT_LIST_HEAD(&mgr->reserved_pages);
687
688 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
689 ttm_resource_manager_set_used(man, true);
690 return 0;
691 }
692
693 /**
694 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
695 *
696 * @adev: amdgpu_device pointer
697 *
698 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
699 * allocated inside it.
700 */
amdgpu_vram_mgr_fini(struct amdgpu_device * adev)701 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
702 {
703 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
704 struct ttm_resource_manager *man = &mgr->manager;
705 int ret;
706 struct amdgpu_vram_reservation *rsv, *temp;
707
708 ttm_resource_manager_set_used(man, false);
709
710 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
711 if (ret)
712 return;
713
714 spin_lock(&mgr->lock);
715 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
716 kfree(rsv);
717
718 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
719 drm_mm_remove_node(&rsv->mm_node);
720 kfree(rsv);
721 }
722 drm_mm_takedown(&mgr->mm);
723 spin_unlock(&mgr->lock);
724
725 ttm_resource_manager_cleanup(man);
726 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
727 }
728