1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/iosys-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28 
29 #include <drm/ttm/ttm_bo.h>
30 #include <drm/ttm/ttm_placement.h>
31 #include <drm/ttm/ttm_resource.h>
32 
33 /**
34  * ttm_lru_bulk_move_init - initialize a bulk move structure
35  * @bulk: the structure to init
36  *
37  * For now just memset the structure to zero.
38  */
ttm_lru_bulk_move_init(struct ttm_lru_bulk_move * bulk)39 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
40 {
41 	memset(bulk, 0, sizeof(*bulk));
42 }
43 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
44 
45 /**
46  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
47  *
48  * @bulk: bulk move structure
49  *
50  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
51  * resource order never changes. Should be called with &ttm_device.lru_lock held.
52  */
ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move * bulk)53 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
54 {
55 	unsigned i, j;
56 
57 	for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
58 		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
59 			struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
60 			struct ttm_resource_manager *man;
61 
62 			if (!pos->first)
63 				continue;
64 
65 			lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
66 			dma_resv_assert_held(pos->first->bo->base.resv);
67 			dma_resv_assert_held(pos->last->bo->base.resv);
68 
69 			man = ttm_manager_type(pos->first->bo->bdev, i);
70 			list_bulk_move_tail(&man->lru[j], &pos->first->lru,
71 					    &pos->last->lru);
72 		}
73 	}
74 }
75 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
76 
77 /* Return the bulk move pos object for this resource */
78 static struct ttm_lru_bulk_move_pos *
ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)79 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
80 {
81 	return &bulk->pos[res->mem_type][res->bo->priority];
82 }
83 
84 /* Move the resource to the tail of the bulk move range */
ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos * pos,struct ttm_resource * res)85 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
86 				       struct ttm_resource *res)
87 {
88 	if (pos->last != res) {
89 		if (pos->first == res)
90 			pos->first = list_next_entry(res, lru);
91 		list_move(&res->lru, &pos->last->lru);
92 		pos->last = res;
93 	}
94 }
95 
96 /* Add the resource to a bulk_move cursor */
ttm_lru_bulk_move_add(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)97 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
98 				  struct ttm_resource *res)
99 {
100 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
101 
102 	if (!pos->first) {
103 		pos->first = res;
104 		pos->last = res;
105 	} else {
106 		ttm_lru_bulk_move_pos_tail(pos, res);
107 	}
108 }
109 
110 /* Remove the resource from a bulk_move range */
ttm_lru_bulk_move_del(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)111 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
112 				  struct ttm_resource *res)
113 {
114 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
115 
116 	if (unlikely(WARN_ON(!pos->first || !pos->last) ||
117 		     (pos->first == res && pos->last == res))) {
118 		pos->first = NULL;
119 		pos->last = NULL;
120 	} else if (pos->first == res) {
121 		pos->first = list_next_entry(res, lru);
122 	} else if (pos->last == res) {
123 		pos->last = list_prev_entry(res, lru);
124 	} else {
125 		list_move(&res->lru, &pos->last->lru);
126 	}
127 }
128 
129 /* Add the resource to a bulk move if the BO is configured for it */
ttm_resource_add_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)130 void ttm_resource_add_bulk_move(struct ttm_resource *res,
131 				struct ttm_buffer_object *bo)
132 {
133 	if (bo->bulk_move && !bo->pin_count)
134 		ttm_lru_bulk_move_add(bo->bulk_move, res);
135 }
136 
137 /* Remove the resource from a bulk move if the BO is configured for it */
ttm_resource_del_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)138 void ttm_resource_del_bulk_move(struct ttm_resource *res,
139 				struct ttm_buffer_object *bo)
140 {
141 	if (bo->bulk_move && !bo->pin_count)
142 		ttm_lru_bulk_move_del(bo->bulk_move, res);
143 }
144 
145 /* Move a resource to the LRU or bulk tail */
ttm_resource_move_to_lru_tail(struct ttm_resource * res)146 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
147 {
148 	struct ttm_buffer_object *bo = res->bo;
149 	struct ttm_device *bdev = bo->bdev;
150 
151 	lockdep_assert_held(&bo->bdev->lru_lock);
152 
153 	if (bo->pin_count) {
154 		list_move_tail(&res->lru, &bdev->pinned);
155 
156 	} else	if (bo->bulk_move) {
157 		struct ttm_lru_bulk_move_pos *pos =
158 			ttm_lru_bulk_move_pos(bo->bulk_move, res);
159 
160 		ttm_lru_bulk_move_pos_tail(pos, res);
161 	} else {
162 		struct ttm_resource_manager *man;
163 
164 		man = ttm_manager_type(bdev, res->mem_type);
165 		list_move_tail(&res->lru, &man->lru[bo->priority]);
166 	}
167 }
168 
169 /**
170  * ttm_resource_init - resource object constructure
171  * @bo: buffer object this resources is allocated for
172  * @place: placement of the resource
173  * @res: the resource object to inistilize
174  *
175  * Initialize a new resource object. Counterpart of ttm_resource_fini().
176  */
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)177 void ttm_resource_init(struct ttm_buffer_object *bo,
178                        const struct ttm_place *place,
179                        struct ttm_resource *res)
180 {
181 	struct ttm_resource_manager *man;
182 
183 	res->start = 0;
184 	res->size = bo->base.size;
185 	res->mem_type = place->mem_type;
186 	res->placement = place->flags;
187 	res->bus.addr = NULL;
188 	res->bus.offset = 0;
189 	res->bus.is_iomem = false;
190 	res->bus.caching = ttm_cached;
191 	res->bo = bo;
192 
193 	man = ttm_manager_type(bo->bdev, place->mem_type);
194 	spin_lock(&bo->bdev->lru_lock);
195 	if (bo->pin_count)
196 		list_add_tail(&res->lru, &bo->bdev->pinned);
197 	else
198 		list_add_tail(&res->lru, &man->lru[bo->priority]);
199 	man->usage += res->size;
200 	spin_unlock(&bo->bdev->lru_lock);
201 }
202 EXPORT_SYMBOL(ttm_resource_init);
203 
204 /**
205  * ttm_resource_fini - resource destructor
206  * @man: the resource manager this resource belongs to
207  * @res: the resource to clean up
208  *
209  * Should be used by resource manager backends to clean up the TTM resource
210  * objects before freeing the underlying structure. Makes sure the resource is
211  * removed from the LRU before destruction.
212  * Counterpart of ttm_resource_init().
213  */
ttm_resource_fini(struct ttm_resource_manager * man,struct ttm_resource * res)214 void ttm_resource_fini(struct ttm_resource_manager *man,
215 		       struct ttm_resource *res)
216 {
217 	struct ttm_device *bdev = man->bdev;
218 
219 	spin_lock(&bdev->lru_lock);
220 	list_del_init(&res->lru);
221 	man->usage -= res->size;
222 	spin_unlock(&bdev->lru_lock);
223 }
224 EXPORT_SYMBOL(ttm_resource_fini);
225 
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)226 int ttm_resource_alloc(struct ttm_buffer_object *bo,
227 		       const struct ttm_place *place,
228 		       struct ttm_resource **res_ptr)
229 {
230 	struct ttm_resource_manager *man =
231 		ttm_manager_type(bo->bdev, place->mem_type);
232 	int ret;
233 
234 	ret = man->func->alloc(man, bo, place, res_ptr);
235 	if (ret)
236 		return ret;
237 
238 	spin_lock(&bo->bdev->lru_lock);
239 	ttm_resource_add_bulk_move(*res_ptr, bo);
240 	spin_unlock(&bo->bdev->lru_lock);
241 	return 0;
242 }
243 
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)244 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
245 {
246 	struct ttm_resource_manager *man;
247 
248 	if (!*res)
249 		return;
250 
251 	spin_lock(&bo->bdev->lru_lock);
252 	ttm_resource_del_bulk_move(*res, bo);
253 	spin_unlock(&bo->bdev->lru_lock);
254 	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
255 	man->func->free(man, *res);
256 	*res = NULL;
257 }
258 EXPORT_SYMBOL(ttm_resource_free);
259 
260 /**
261  * ttm_resource_intersects - test for intersection
262  *
263  * @bdev: TTM device structure
264  * @res: The resource to test
265  * @place: The placement to test
266  * @size: How many bytes the new allocation needs.
267  *
268  * Test if @res intersects with @place and @size. Used for testing if evictions
269  * are valueable or not.
270  *
271  * Returns true if the res placement intersects with @place and @size.
272  */
ttm_resource_intersects(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)273 bool ttm_resource_intersects(struct ttm_device *bdev,
274 			     struct ttm_resource *res,
275 			     const struct ttm_place *place,
276 			     size_t size)
277 {
278 	struct ttm_resource_manager *man;
279 
280 	if (!res)
281 		return false;
282 
283 	man = ttm_manager_type(bdev, res->mem_type);
284 	if (!place || !man->func->intersects)
285 		return true;
286 
287 	return man->func->intersects(man, res, place, size);
288 }
289 
290 /**
291  * ttm_resource_compatible - test for compatibility
292  *
293  * @bdev: TTM device structure
294  * @res: The resource to test
295  * @place: The placement to test
296  * @size: How many bytes the new allocation needs.
297  *
298  * Test if @res compatible with @place and @size.
299  *
300  * Returns true if the res placement compatible with @place and @size.
301  */
ttm_resource_compatible(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)302 bool ttm_resource_compatible(struct ttm_device *bdev,
303 			     struct ttm_resource *res,
304 			     const struct ttm_place *place,
305 			     size_t size)
306 {
307 	struct ttm_resource_manager *man;
308 
309 	if (!res || !place)
310 		return false;
311 
312 	man = ttm_manager_type(bdev, res->mem_type);
313 	if (!man->func->compatible)
314 		return true;
315 
316 	return man->func->compatible(man, res, place, size);
317 }
318 
ttm_resource_places_compat(struct ttm_resource * res,const struct ttm_place * places,unsigned num_placement)319 static bool ttm_resource_places_compat(struct ttm_resource *res,
320 				       const struct ttm_place *places,
321 				       unsigned num_placement)
322 {
323 	struct ttm_buffer_object *bo = res->bo;
324 	struct ttm_device *bdev = bo->bdev;
325 	unsigned i;
326 
327 	if (res->placement & TTM_PL_FLAG_TEMPORARY)
328 		return false;
329 
330 	for (i = 0; i < num_placement; i++) {
331 		const struct ttm_place *heap = &places[i];
332 
333 		if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
334 			continue;
335 
336 		if ((res->mem_type == heap->mem_type) &&
337 		    (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
338 		     (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
339 			return true;
340 	}
341 	return false;
342 }
343 
344 /**
345  * ttm_resource_compat - check if resource is compatible with placement
346  *
347  * @res: the resource to check
348  * @placement: the placement to check against
349  *
350  * Returns true if the placement is compatible.
351  */
ttm_resource_compat(struct ttm_resource * res,struct ttm_placement * placement)352 bool ttm_resource_compat(struct ttm_resource *res,
353 			 struct ttm_placement *placement)
354 {
355 	if (ttm_resource_places_compat(res, placement->placement,
356 				       placement->num_placement))
357 		return true;
358 
359 	if ((placement->busy_placement != placement->placement ||
360 	     placement->num_busy_placement > placement->num_placement) &&
361 	    ttm_resource_places_compat(res, placement->busy_placement,
362 				       placement->num_busy_placement))
363 		return true;
364 
365 	return false;
366 }
367 
ttm_resource_set_bo(struct ttm_resource * res,struct ttm_buffer_object * bo)368 void ttm_resource_set_bo(struct ttm_resource *res,
369 			 struct ttm_buffer_object *bo)
370 {
371 	spin_lock(&bo->bdev->lru_lock);
372 	res->bo = bo;
373 	spin_unlock(&bo->bdev->lru_lock);
374 }
375 
376 /**
377  * ttm_resource_manager_init
378  *
379  * @man: memory manager object to init
380  * @bdev: ttm device this manager belongs to
381  * @size: size of managed resources in arbitrary units
382  *
383  * Initialise core parts of a manager object.
384  */
ttm_resource_manager_init(struct ttm_resource_manager * man,struct ttm_device * bdev,uint64_t size)385 void ttm_resource_manager_init(struct ttm_resource_manager *man,
386 			       struct ttm_device *bdev,
387 			       uint64_t size)
388 {
389 	unsigned i;
390 
391 	spin_lock_init(&man->move_lock);
392 	man->bdev = bdev;
393 	man->size = size;
394 	man->usage = 0;
395 
396 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
397 		INIT_LIST_HEAD(&man->lru[i]);
398 	man->move = NULL;
399 }
400 EXPORT_SYMBOL(ttm_resource_manager_init);
401 
402 /*
403  * ttm_resource_manager_evict_all
404  *
405  * @bdev - device to use
406  * @man - manager to use
407  *
408  * Evict all the objects out of a memory manager until it is empty.
409  * Part of memory manager cleanup sequence.
410  */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)411 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
412 				   struct ttm_resource_manager *man)
413 {
414 	struct ttm_operation_ctx ctx = {
415 		.interruptible = false,
416 		.no_wait_gpu = false,
417 		.force_alloc = true
418 	};
419 	struct dma_fence *fence;
420 	int ret;
421 	unsigned i;
422 
423 	/*
424 	 * Can't use standard list traversal since we're unlocking.
425 	 */
426 
427 	spin_lock(&bdev->lru_lock);
428 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
429 		while (!list_empty(&man->lru[i])) {
430 			spin_unlock(&bdev->lru_lock);
431 			ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
432 						  NULL);
433 			if (ret)
434 				return ret;
435 			spin_lock(&bdev->lru_lock);
436 		}
437 	}
438 	spin_unlock(&bdev->lru_lock);
439 
440 	spin_lock(&man->move_lock);
441 	fence = dma_fence_get(man->move);
442 	spin_unlock(&man->move_lock);
443 
444 	if (fence) {
445 		ret = dma_fence_wait(fence, false);
446 		dma_fence_put(fence);
447 		if (ret)
448 			return ret;
449 	}
450 
451 	return 0;
452 }
453 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
454 
455 /**
456  * ttm_resource_manager_usage
457  *
458  * @man: A memory manager object.
459  *
460  * Return how many resources are currently used.
461  */
ttm_resource_manager_usage(struct ttm_resource_manager * man)462 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
463 {
464 	uint64_t usage;
465 
466 	spin_lock(&man->bdev->lru_lock);
467 	usage = man->usage;
468 	spin_unlock(&man->bdev->lru_lock);
469 	return usage;
470 }
471 EXPORT_SYMBOL(ttm_resource_manager_usage);
472 
473 /**
474  * ttm_resource_manager_debug
475  *
476  * @man: manager type to dump.
477  * @p: printer to use for debug.
478  */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)479 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
480 				struct drm_printer *p)
481 {
482 	drm_printf(p, "  use_type: %d\n", man->use_type);
483 	drm_printf(p, "  use_tt: %d\n", man->use_tt);
484 	drm_printf(p, "  size: %llu\n", man->size);
485 	drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
486 	if (man->func->debug)
487 		man->func->debug(man, p);
488 }
489 EXPORT_SYMBOL(ttm_resource_manager_debug);
490 
491 /**
492  * ttm_resource_manager_first
493  *
494  * @man: resource manager to iterate over
495  * @cursor: cursor to record the position
496  *
497  * Returns the first resource from the resource manager.
498  */
499 struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor)500 ttm_resource_manager_first(struct ttm_resource_manager *man,
501 			   struct ttm_resource_cursor *cursor)
502 {
503 	struct ttm_resource *res;
504 
505 	lockdep_assert_held(&man->bdev->lru_lock);
506 
507 	for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
508 	     ++cursor->priority)
509 		list_for_each_entry(res, &man->lru[cursor->priority], lru)
510 			return res;
511 
512 	return NULL;
513 }
514 
515 /**
516  * ttm_resource_manager_next
517  *
518  * @man: resource manager to iterate over
519  * @cursor: cursor to record the position
520  * @res: the current resource pointer
521  *
522  * Returns the next resource from the resource manager.
523  */
524 struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor,struct ttm_resource * res)525 ttm_resource_manager_next(struct ttm_resource_manager *man,
526 			  struct ttm_resource_cursor *cursor,
527 			  struct ttm_resource *res)
528 {
529 	lockdep_assert_held(&man->bdev->lru_lock);
530 
531 	list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
532 		return res;
533 
534 	for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
535 	     ++cursor->priority)
536 		list_for_each_entry(res, &man->lru[cursor->priority], lru)
537 			return res;
538 
539 	return NULL;
540 }
541 
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)542 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
543 					  struct iosys_map *dmap,
544 					  pgoff_t i)
545 {
546 	struct ttm_kmap_iter_iomap *iter_io =
547 		container_of(iter, typeof(*iter_io), base);
548 	void __iomem *addr;
549 
550 retry:
551 	while (i >= iter_io->cache.end) {
552 		iter_io->cache.sg = iter_io->cache.sg ?
553 			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
554 		iter_io->cache.i = iter_io->cache.end;
555 		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
556 			PAGE_SHIFT;
557 		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
558 			iter_io->start;
559 	}
560 
561 	if (i < iter_io->cache.i) {
562 		iter_io->cache.end = 0;
563 		iter_io->cache.sg = NULL;
564 		goto retry;
565 	}
566 
567 	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
568 				       (((resource_size_t)i - iter_io->cache.i)
569 					<< PAGE_SHIFT));
570 	iosys_map_set_vaddr_iomem(dmap, addr);
571 }
572 
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct iosys_map * map)573 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
574 					    struct iosys_map *map)
575 {
576 	io_mapping_unmap_local(map->vaddr_iomem);
577 }
578 
579 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
580 	.map_local =  ttm_kmap_iter_iomap_map_local,
581 	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
582 	.maps_tt = false,
583 };
584 
585 /**
586  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
587  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
588  * @iomap: The struct io_mapping representing the underlying linear io_memory.
589  * @st: sg_table into @iomap, representing the memory of the struct
590  * ttm_resource.
591  * @start: Offset that needs to be subtracted from @st to make
592  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
593  *
594  * Return: Pointer to the embedded struct ttm_kmap_iter.
595  */
596 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)597 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
598 			 struct io_mapping *iomap,
599 			 struct sg_table *st,
600 			 resource_size_t start)
601 {
602 	iter_io->base.ops = &ttm_kmap_iter_io_ops;
603 	iter_io->iomap = iomap;
604 	iter_io->st = st;
605 	iter_io->start = start;
606 	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
607 
608 	return &iter_io->base;
609 }
610 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
611 
612 /**
613  * DOC: Linear io iterator
614  *
615  * This code should die in the not too near future. Best would be if we could
616  * make io-mapping use memremap for all io memory, and have memremap
617  * implement a kmap_local functionality. We could then strip a huge amount of
618  * code. These linear io iterators are implemented to mimic old functionality,
619  * and they don't use kmap_local semantics at all internally. Rather ioremap or
620  * friends, and at least on 32-bit they add global TLB flushes and points
621  * of failure.
622  */
623 
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)624 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
625 					      struct iosys_map *dmap,
626 					      pgoff_t i)
627 {
628 	struct ttm_kmap_iter_linear_io *iter_io =
629 		container_of(iter, typeof(*iter_io), base);
630 
631 	*dmap = iter_io->dmap;
632 	iosys_map_incr(dmap, i * PAGE_SIZE);
633 }
634 
635 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
636 	.map_local =  ttm_kmap_iter_linear_io_map_local,
637 	.maps_tt = false,
638 };
639 
640 /**
641  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
642  * @iter_io: The iterator to initialize
643  * @bdev: The TTM device
644  * @mem: The ttm resource representing the iomap.
645  *
646  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
647  * pointing at a linear chunk of io memory.
648  *
649  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
650  * failure.
651  */
652 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)653 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
654 			     struct ttm_device *bdev,
655 			     struct ttm_resource *mem)
656 {
657 	int ret;
658 
659 	ret = ttm_mem_io_reserve(bdev, mem);
660 	if (ret)
661 		goto out_err;
662 	if (!mem->bus.is_iomem) {
663 		ret = -EINVAL;
664 		goto out_io_free;
665 	}
666 
667 	if (mem->bus.addr) {
668 		iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
669 		iter_io->needs_unmap = false;
670 	} else {
671 		iter_io->needs_unmap = true;
672 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
673 		if (mem->bus.caching == ttm_write_combined)
674 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
675 						  ioremap_wc(mem->bus.offset,
676 							     mem->size));
677 		else if (mem->bus.caching == ttm_cached)
678 			iosys_map_set_vaddr(&iter_io->dmap,
679 					    memremap(mem->bus.offset, mem->size,
680 						     MEMREMAP_WB |
681 						     MEMREMAP_WT |
682 						     MEMREMAP_WC));
683 
684 		/* If uncached requested or if mapping cached or wc failed */
685 		if (iosys_map_is_null(&iter_io->dmap))
686 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
687 						  ioremap(mem->bus.offset,
688 							  mem->size));
689 
690 		if (iosys_map_is_null(&iter_io->dmap)) {
691 			ret = -ENOMEM;
692 			goto out_io_free;
693 		}
694 	}
695 
696 	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
697 	return &iter_io->base;
698 
699 out_io_free:
700 	ttm_mem_io_free(bdev, mem);
701 out_err:
702 	return ERR_PTR(ret);
703 }
704 
705 /**
706  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
707  * @iter_io: The iterator to initialize
708  * @bdev: The TTM device
709  * @mem: The ttm resource representing the iomap.
710  *
711  * This function is for internal TTM use only. It cleans up a memcpy kmap
712  * iterator initialized by ttm_kmap_iter_linear_io_init.
713  */
714 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)715 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
716 			     struct ttm_device *bdev,
717 			     struct ttm_resource *mem)
718 {
719 	if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
720 		if (iter_io->dmap.is_iomem)
721 			iounmap(iter_io->dmap.vaddr_iomem);
722 		else
723 			memunmap(iter_io->dmap.vaddr);
724 	}
725 
726 	ttm_mem_io_free(bdev, mem);
727 }
728 
729 #if defined(CONFIG_DEBUG_FS)
730 
ttm_resource_manager_show(struct seq_file * m,void * unused)731 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
732 {
733 	struct ttm_resource_manager *man =
734 		(struct ttm_resource_manager *)m->private;
735 	struct drm_printer p = drm_seq_file_printer(m);
736 	ttm_resource_manager_debug(man, &p);
737 	return 0;
738 }
739 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
740 
741 #endif
742 
743 /**
744  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
745  * resource manager.
746  * @man: The TTM resource manager for which the debugfs stats file be creates
747  * @parent: debugfs directory in which the file will reside
748  * @name: The filename to create.
749  *
750  * This function setups up a debugfs file that can be used to look
751  * at debug statistics of the specified ttm_resource_manager.
752  */
ttm_resource_manager_create_debugfs(struct ttm_resource_manager * man,struct dentry * parent,const char * name)753 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
754 					 struct dentry * parent,
755 					 const char *name)
756 {
757 #if defined(CONFIG_DEBUG_FS)
758 	debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
759 #endif
760 }
761 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
762