1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #include <linux/iosys-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28
29 #include <drm/ttm/ttm_resource.h>
30 #include <drm/ttm/ttm_bo_driver.h>
31
32 /**
33 * ttm_lru_bulk_move_init - initialize a bulk move structure
34 * @bulk: the structure to init
35 *
36 * For now just memset the structure to zero.
37 */
ttm_lru_bulk_move_init(struct ttm_lru_bulk_move * bulk)38 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
39 {
40 memset(bulk, 0, sizeof(*bulk));
41 }
42 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
43
44 /**
45 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
46 *
47 * @bulk: bulk move structure
48 *
49 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
50 * resource order never changes. Should be called with &ttm_device.lru_lock held.
51 */
ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move * bulk)52 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
53 {
54 unsigned i, j;
55
56 for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
57 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
58 struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
59 struct ttm_resource_manager *man;
60
61 if (!pos->first)
62 continue;
63
64 lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
65 dma_resv_assert_held(pos->first->bo->base.resv);
66 dma_resv_assert_held(pos->last->bo->base.resv);
67
68 man = ttm_manager_type(pos->first->bo->bdev, i);
69 list_bulk_move_tail(&man->lru[j], &pos->first->lru,
70 &pos->last->lru);
71 }
72 }
73 }
74 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
75
76 /* Return the bulk move pos object for this resource */
77 static struct ttm_lru_bulk_move_pos *
ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)78 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
79 {
80 return &bulk->pos[res->mem_type][res->bo->priority];
81 }
82
83 /* Move the resource to the tail of the bulk move range */
ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos * pos,struct ttm_resource * res)84 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
85 struct ttm_resource *res)
86 {
87 if (pos->last != res) {
88 list_move(&res->lru, &pos->last->lru);
89 pos->last = res;
90 }
91 }
92
93 /* Add the resource to a bulk_move cursor */
ttm_lru_bulk_move_add(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)94 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
95 struct ttm_resource *res)
96 {
97 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
98
99 if (!pos->first) {
100 pos->first = res;
101 pos->last = res;
102 } else {
103 ttm_lru_bulk_move_pos_tail(pos, res);
104 }
105 }
106
107 /* Remove the resource from a bulk_move range */
ttm_lru_bulk_move_del(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)108 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
109 struct ttm_resource *res)
110 {
111 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
112
113 if (unlikely(pos->first == res && pos->last == res)) {
114 pos->first = NULL;
115 pos->last = NULL;
116 } else if (pos->first == res) {
117 pos->first = list_next_entry(res, lru);
118 } else if (pos->last == res) {
119 pos->last = list_prev_entry(res, lru);
120 } else {
121 list_move(&res->lru, &pos->last->lru);
122 }
123 }
124
125 /* Add the resource to a bulk move if the BO is configured for it */
ttm_resource_add_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)126 void ttm_resource_add_bulk_move(struct ttm_resource *res,
127 struct ttm_buffer_object *bo)
128 {
129 if (bo->bulk_move && !bo->pin_count)
130 ttm_lru_bulk_move_add(bo->bulk_move, res);
131 }
132
133 /* Remove the resource from a bulk move if the BO is configured for it */
ttm_resource_del_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)134 void ttm_resource_del_bulk_move(struct ttm_resource *res,
135 struct ttm_buffer_object *bo)
136 {
137 if (bo->bulk_move && !bo->pin_count)
138 ttm_lru_bulk_move_del(bo->bulk_move, res);
139 }
140
141 /* Move a resource to the LRU or bulk tail */
ttm_resource_move_to_lru_tail(struct ttm_resource * res)142 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
143 {
144 struct ttm_buffer_object *bo = res->bo;
145 struct ttm_device *bdev = bo->bdev;
146
147 lockdep_assert_held(&bo->bdev->lru_lock);
148
149 if (bo->pin_count) {
150 list_move_tail(&res->lru, &bdev->pinned);
151
152 } else if (bo->bulk_move) {
153 struct ttm_lru_bulk_move_pos *pos =
154 ttm_lru_bulk_move_pos(bo->bulk_move, res);
155
156 ttm_lru_bulk_move_pos_tail(pos, res);
157 } else {
158 struct ttm_resource_manager *man;
159
160 man = ttm_manager_type(bdev, res->mem_type);
161 list_move_tail(&res->lru, &man->lru[bo->priority]);
162 }
163 }
164
165 /**
166 * ttm_resource_init - resource object constructure
167 * @bo: buffer object this resources is allocated for
168 * @place: placement of the resource
169 * @res: the resource object to inistilize
170 *
171 * Initialize a new resource object. Counterpart of ttm_resource_fini().
172 */
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)173 void ttm_resource_init(struct ttm_buffer_object *bo,
174 const struct ttm_place *place,
175 struct ttm_resource *res)
176 {
177 struct ttm_resource_manager *man;
178
179 res->start = 0;
180 res->num_pages = PFN_UP(bo->base.size);
181 res->mem_type = place->mem_type;
182 res->placement = place->flags;
183 res->bus.addr = NULL;
184 res->bus.offset = 0;
185 res->bus.is_iomem = false;
186 res->bus.caching = ttm_cached;
187 res->bo = bo;
188
189 man = ttm_manager_type(bo->bdev, place->mem_type);
190 spin_lock(&bo->bdev->lru_lock);
191 if (bo->pin_count)
192 list_add_tail(&res->lru, &bo->bdev->pinned);
193 else
194 list_add_tail(&res->lru, &man->lru[bo->priority]);
195 man->usage += res->num_pages << PAGE_SHIFT;
196 spin_unlock(&bo->bdev->lru_lock);
197 }
198 EXPORT_SYMBOL(ttm_resource_init);
199
200 /**
201 * ttm_resource_fini - resource destructor
202 * @man: the resource manager this resource belongs to
203 * @res: the resource to clean up
204 *
205 * Should be used by resource manager backends to clean up the TTM resource
206 * objects before freeing the underlying structure. Makes sure the resource is
207 * removed from the LRU before destruction.
208 * Counterpart of ttm_resource_init().
209 */
ttm_resource_fini(struct ttm_resource_manager * man,struct ttm_resource * res)210 void ttm_resource_fini(struct ttm_resource_manager *man,
211 struct ttm_resource *res)
212 {
213 struct ttm_device *bdev = man->bdev;
214
215 spin_lock(&bdev->lru_lock);
216 list_del_init(&res->lru);
217 man->usage -= res->num_pages << PAGE_SHIFT;
218 spin_unlock(&bdev->lru_lock);
219 }
220 EXPORT_SYMBOL(ttm_resource_fini);
221
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)222 int ttm_resource_alloc(struct ttm_buffer_object *bo,
223 const struct ttm_place *place,
224 struct ttm_resource **res_ptr)
225 {
226 struct ttm_resource_manager *man =
227 ttm_manager_type(bo->bdev, place->mem_type);
228 int ret;
229
230 ret = man->func->alloc(man, bo, place, res_ptr);
231 if (ret)
232 return ret;
233
234 spin_lock(&bo->bdev->lru_lock);
235 ttm_resource_add_bulk_move(*res_ptr, bo);
236 spin_unlock(&bo->bdev->lru_lock);
237 return 0;
238 }
239
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)240 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
241 {
242 struct ttm_resource_manager *man;
243
244 if (!*res)
245 return;
246
247 spin_lock(&bo->bdev->lru_lock);
248 ttm_resource_del_bulk_move(*res, bo);
249 spin_unlock(&bo->bdev->lru_lock);
250 man = ttm_manager_type(bo->bdev, (*res)->mem_type);
251 man->func->free(man, *res);
252 *res = NULL;
253 }
254 EXPORT_SYMBOL(ttm_resource_free);
255
256 /**
257 * ttm_resource_intersects - test for intersection
258 *
259 * @bdev: TTM device structure
260 * @res: The resource to test
261 * @place: The placement to test
262 * @size: How many bytes the new allocation needs.
263 *
264 * Test if @res intersects with @place and @size. Used for testing if evictions
265 * are valueable or not.
266 *
267 * Returns true if the res placement intersects with @place and @size.
268 */
ttm_resource_intersects(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)269 bool ttm_resource_intersects(struct ttm_device *bdev,
270 struct ttm_resource *res,
271 const struct ttm_place *place,
272 size_t size)
273 {
274 struct ttm_resource_manager *man;
275
276 if (!res)
277 return false;
278
279 man = ttm_manager_type(bdev, res->mem_type);
280 if (!place || !man->func->intersects)
281 return true;
282
283 return man->func->intersects(man, res, place, size);
284 }
285
286 /**
287 * ttm_resource_compatible - test for compatibility
288 *
289 * @bdev: TTM device structure
290 * @res: The resource to test
291 * @place: The placement to test
292 * @size: How many bytes the new allocation needs.
293 *
294 * Test if @res compatible with @place and @size.
295 *
296 * Returns true if the res placement compatible with @place and @size.
297 */
ttm_resource_compatible(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)298 bool ttm_resource_compatible(struct ttm_device *bdev,
299 struct ttm_resource *res,
300 const struct ttm_place *place,
301 size_t size)
302 {
303 struct ttm_resource_manager *man;
304
305 if (!res || !place)
306 return false;
307
308 man = ttm_manager_type(bdev, res->mem_type);
309 if (!man->func->compatible)
310 return true;
311
312 return man->func->compatible(man, res, place, size);
313 }
314
ttm_resource_places_compat(struct ttm_resource * res,const struct ttm_place * places,unsigned num_placement)315 static bool ttm_resource_places_compat(struct ttm_resource *res,
316 const struct ttm_place *places,
317 unsigned num_placement)
318 {
319 struct ttm_buffer_object *bo = res->bo;
320 struct ttm_device *bdev = bo->bdev;
321 unsigned i;
322
323 if (res->placement & TTM_PL_FLAG_TEMPORARY)
324 return false;
325
326 for (i = 0; i < num_placement; i++) {
327 const struct ttm_place *heap = &places[i];
328
329 if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
330 continue;
331
332 if ((res->mem_type == heap->mem_type) &&
333 (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
334 (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
335 return true;
336 }
337 return false;
338 }
339
340 /**
341 * ttm_resource_compat - check if resource is compatible with placement
342 *
343 * @res: the resource to check
344 * @placement: the placement to check against
345 *
346 * Returns true if the placement is compatible.
347 */
ttm_resource_compat(struct ttm_resource * res,struct ttm_placement * placement)348 bool ttm_resource_compat(struct ttm_resource *res,
349 struct ttm_placement *placement)
350 {
351 if (ttm_resource_places_compat(res, placement->placement,
352 placement->num_placement))
353 return true;
354
355 if ((placement->busy_placement != placement->placement ||
356 placement->num_busy_placement > placement->num_placement) &&
357 ttm_resource_places_compat(res, placement->busy_placement,
358 placement->num_busy_placement))
359 return true;
360
361 return false;
362 }
363 EXPORT_SYMBOL(ttm_resource_compat);
364
ttm_resource_set_bo(struct ttm_resource * res,struct ttm_buffer_object * bo)365 void ttm_resource_set_bo(struct ttm_resource *res,
366 struct ttm_buffer_object *bo)
367 {
368 spin_lock(&bo->bdev->lru_lock);
369 res->bo = bo;
370 spin_unlock(&bo->bdev->lru_lock);
371 }
372
373 /**
374 * ttm_resource_manager_init
375 *
376 * @man: memory manager object to init
377 * @bdev: ttm device this manager belongs to
378 * @size: size of managed resources in arbitrary units
379 *
380 * Initialise core parts of a manager object.
381 */
ttm_resource_manager_init(struct ttm_resource_manager * man,struct ttm_device * bdev,uint64_t size)382 void ttm_resource_manager_init(struct ttm_resource_manager *man,
383 struct ttm_device *bdev,
384 uint64_t size)
385 {
386 unsigned i;
387
388 spin_lock_init(&man->move_lock);
389 man->bdev = bdev;
390 man->size = size;
391 man->usage = 0;
392
393 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
394 INIT_LIST_HEAD(&man->lru[i]);
395 man->move = NULL;
396 }
397 EXPORT_SYMBOL(ttm_resource_manager_init);
398
399 /*
400 * ttm_resource_manager_evict_all
401 *
402 * @bdev - device to use
403 * @man - manager to use
404 *
405 * Evict all the objects out of a memory manager until it is empty.
406 * Part of memory manager cleanup sequence.
407 */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)408 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
409 struct ttm_resource_manager *man)
410 {
411 struct ttm_operation_ctx ctx = {
412 .interruptible = false,
413 .no_wait_gpu = false,
414 .force_alloc = true
415 };
416 struct dma_fence *fence;
417 int ret;
418 unsigned i;
419
420 /*
421 * Can't use standard list traversal since we're unlocking.
422 */
423
424 spin_lock(&bdev->lru_lock);
425 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
426 while (!list_empty(&man->lru[i])) {
427 spin_unlock(&bdev->lru_lock);
428 ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
429 NULL);
430 if (ret)
431 return ret;
432 spin_lock(&bdev->lru_lock);
433 }
434 }
435 spin_unlock(&bdev->lru_lock);
436
437 spin_lock(&man->move_lock);
438 fence = dma_fence_get(man->move);
439 spin_unlock(&man->move_lock);
440
441 if (fence) {
442 ret = dma_fence_wait(fence, false);
443 dma_fence_put(fence);
444 if (ret)
445 return ret;
446 }
447
448 return 0;
449 }
450 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
451
452 /**
453 * ttm_resource_manager_usage
454 *
455 * @man: A memory manager object.
456 *
457 * Return how many resources are currently used.
458 */
ttm_resource_manager_usage(struct ttm_resource_manager * man)459 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
460 {
461 uint64_t usage;
462
463 spin_lock(&man->bdev->lru_lock);
464 usage = man->usage;
465 spin_unlock(&man->bdev->lru_lock);
466 return usage;
467 }
468 EXPORT_SYMBOL(ttm_resource_manager_usage);
469
470 /**
471 * ttm_resource_manager_debug
472 *
473 * @man: manager type to dump.
474 * @p: printer to use for debug.
475 */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)476 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
477 struct drm_printer *p)
478 {
479 drm_printf(p, " use_type: %d\n", man->use_type);
480 drm_printf(p, " use_tt: %d\n", man->use_tt);
481 drm_printf(p, " size: %llu\n", man->size);
482 drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man));
483 if (man->func->debug)
484 man->func->debug(man, p);
485 }
486 EXPORT_SYMBOL(ttm_resource_manager_debug);
487
488 /**
489 * ttm_resource_manager_first
490 *
491 * @man: resource manager to iterate over
492 * @cursor: cursor to record the position
493 *
494 * Returns the first resource from the resource manager.
495 */
496 struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor)497 ttm_resource_manager_first(struct ttm_resource_manager *man,
498 struct ttm_resource_cursor *cursor)
499 {
500 struct ttm_resource *res;
501
502 lockdep_assert_held(&man->bdev->lru_lock);
503
504 for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
505 ++cursor->priority)
506 list_for_each_entry(res, &man->lru[cursor->priority], lru)
507 return res;
508
509 return NULL;
510 }
511
512 /**
513 * ttm_resource_manager_next
514 *
515 * @man: resource manager to iterate over
516 * @cursor: cursor to record the position
517 * @res: the current resource pointer
518 *
519 * Returns the next resource from the resource manager.
520 */
521 struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor,struct ttm_resource * res)522 ttm_resource_manager_next(struct ttm_resource_manager *man,
523 struct ttm_resource_cursor *cursor,
524 struct ttm_resource *res)
525 {
526 lockdep_assert_held(&man->bdev->lru_lock);
527
528 list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
529 return res;
530
531 for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
532 ++cursor->priority)
533 list_for_each_entry(res, &man->lru[cursor->priority], lru)
534 return res;
535
536 return NULL;
537 }
538
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)539 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
540 struct iosys_map *dmap,
541 pgoff_t i)
542 {
543 struct ttm_kmap_iter_iomap *iter_io =
544 container_of(iter, typeof(*iter_io), base);
545 void __iomem *addr;
546
547 retry:
548 while (i >= iter_io->cache.end) {
549 iter_io->cache.sg = iter_io->cache.sg ?
550 sg_next(iter_io->cache.sg) : iter_io->st->sgl;
551 iter_io->cache.i = iter_io->cache.end;
552 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
553 PAGE_SHIFT;
554 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
555 iter_io->start;
556 }
557
558 if (i < iter_io->cache.i) {
559 iter_io->cache.end = 0;
560 iter_io->cache.sg = NULL;
561 goto retry;
562 }
563
564 addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
565 (((resource_size_t)i - iter_io->cache.i)
566 << PAGE_SHIFT));
567 iosys_map_set_vaddr_iomem(dmap, addr);
568 }
569
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct iosys_map * map)570 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
571 struct iosys_map *map)
572 {
573 io_mapping_unmap_local(map->vaddr_iomem);
574 }
575
576 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
577 .map_local = ttm_kmap_iter_iomap_map_local,
578 .unmap_local = ttm_kmap_iter_iomap_unmap_local,
579 .maps_tt = false,
580 };
581
582 /**
583 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
584 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
585 * @iomap: The struct io_mapping representing the underlying linear io_memory.
586 * @st: sg_table into @iomap, representing the memory of the struct
587 * ttm_resource.
588 * @start: Offset that needs to be subtracted from @st to make
589 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
590 *
591 * Return: Pointer to the embedded struct ttm_kmap_iter.
592 */
593 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)594 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
595 struct io_mapping *iomap,
596 struct sg_table *st,
597 resource_size_t start)
598 {
599 iter_io->base.ops = &ttm_kmap_iter_io_ops;
600 iter_io->iomap = iomap;
601 iter_io->st = st;
602 iter_io->start = start;
603 memset(&iter_io->cache, 0, sizeof(iter_io->cache));
604
605 return &iter_io->base;
606 }
607 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
608
609 /**
610 * DOC: Linear io iterator
611 *
612 * This code should die in the not too near future. Best would be if we could
613 * make io-mapping use memremap for all io memory, and have memremap
614 * implement a kmap_local functionality. We could then strip a huge amount of
615 * code. These linear io iterators are implemented to mimic old functionality,
616 * and they don't use kmap_local semantics at all internally. Rather ioremap or
617 * friends, and at least on 32-bit they add global TLB flushes and points
618 * of failure.
619 */
620
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)621 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
622 struct iosys_map *dmap,
623 pgoff_t i)
624 {
625 struct ttm_kmap_iter_linear_io *iter_io =
626 container_of(iter, typeof(*iter_io), base);
627
628 *dmap = iter_io->dmap;
629 iosys_map_incr(dmap, i * PAGE_SIZE);
630 }
631
632 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
633 .map_local = ttm_kmap_iter_linear_io_map_local,
634 .maps_tt = false,
635 };
636
637 /**
638 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
639 * @iter_io: The iterator to initialize
640 * @bdev: The TTM device
641 * @mem: The ttm resource representing the iomap.
642 *
643 * This function is for internal TTM use only. It sets up a memcpy kmap iterator
644 * pointing at a linear chunk of io memory.
645 *
646 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
647 * failure.
648 */
649 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)650 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
651 struct ttm_device *bdev,
652 struct ttm_resource *mem)
653 {
654 int ret;
655
656 ret = ttm_mem_io_reserve(bdev, mem);
657 if (ret)
658 goto out_err;
659 if (!mem->bus.is_iomem) {
660 ret = -EINVAL;
661 goto out_io_free;
662 }
663
664 if (mem->bus.addr) {
665 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
666 iter_io->needs_unmap = false;
667 } else {
668 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
669
670 iter_io->needs_unmap = true;
671 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
672 if (mem->bus.caching == ttm_write_combined)
673 iosys_map_set_vaddr_iomem(&iter_io->dmap,
674 ioremap_wc(mem->bus.offset,
675 bus_size));
676 else if (mem->bus.caching == ttm_cached)
677 iosys_map_set_vaddr(&iter_io->dmap,
678 memremap(mem->bus.offset, bus_size,
679 MEMREMAP_WB |
680 MEMREMAP_WT |
681 MEMREMAP_WC));
682
683 /* If uncached requested or if mapping cached or wc failed */
684 if (iosys_map_is_null(&iter_io->dmap))
685 iosys_map_set_vaddr_iomem(&iter_io->dmap,
686 ioremap(mem->bus.offset,
687 bus_size));
688
689 if (iosys_map_is_null(&iter_io->dmap)) {
690 ret = -ENOMEM;
691 goto out_io_free;
692 }
693 }
694
695 iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
696 return &iter_io->base;
697
698 out_io_free:
699 ttm_mem_io_free(bdev, mem);
700 out_err:
701 return ERR_PTR(ret);
702 }
703
704 /**
705 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
706 * @iter_io: The iterator to initialize
707 * @bdev: The TTM device
708 * @mem: The ttm resource representing the iomap.
709 *
710 * This function is for internal TTM use only. It cleans up a memcpy kmap
711 * iterator initialized by ttm_kmap_iter_linear_io_init.
712 */
713 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)714 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
715 struct ttm_device *bdev,
716 struct ttm_resource *mem)
717 {
718 if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
719 if (iter_io->dmap.is_iomem)
720 iounmap(iter_io->dmap.vaddr_iomem);
721 else
722 memunmap(iter_io->dmap.vaddr);
723 }
724
725 ttm_mem_io_free(bdev, mem);
726 }
727
728 #if defined(CONFIG_DEBUG_FS)
729
ttm_resource_manager_show(struct seq_file * m,void * unused)730 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
731 {
732 struct ttm_resource_manager *man =
733 (struct ttm_resource_manager *)m->private;
734 struct drm_printer p = drm_seq_file_printer(m);
735 ttm_resource_manager_debug(man, &p);
736 return 0;
737 }
738 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
739
740 #endif
741
742 /**
743 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
744 * resource manager.
745 * @man: The TTM resource manager for which the debugfs stats file be creates
746 * @parent: debugfs directory in which the file will reside
747 * @name: The filename to create.
748 *
749 * This function setups up a debugfs file that can be used to look
750 * at debug statistics of the specified ttm_resource_manager.
751 */
ttm_resource_manager_create_debugfs(struct ttm_resource_manager * man,struct dentry * parent,const char * name)752 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
753 struct dentry * parent,
754 const char *name)
755 {
756 #if defined(CONFIG_DEBUG_FS)
757 debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
758 #endif
759 }
760 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
761