1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/iosys-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28 
29 #include <drm/ttm/ttm_resource.h>
30 #include <drm/ttm/ttm_bo_driver.h>
31 
32 /**
33  * ttm_lru_bulk_move_init - initialize a bulk move structure
34  * @bulk: the structure to init
35  *
36  * For now just memset the structure to zero.
37  */
ttm_lru_bulk_move_init(struct ttm_lru_bulk_move * bulk)38 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
39 {
40 	memset(bulk, 0, sizeof(*bulk));
41 }
42 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
43 
44 /**
45  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
46  *
47  * @bulk: bulk move structure
48  *
49  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
50  * resource order never changes. Should be called with &ttm_device.lru_lock held.
51  */
ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move * bulk)52 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
53 {
54 	unsigned i, j;
55 
56 	for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
57 		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
58 			struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
59 			struct ttm_resource_manager *man;
60 
61 			if (!pos->first)
62 				continue;
63 
64 			lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
65 			dma_resv_assert_held(pos->first->bo->base.resv);
66 			dma_resv_assert_held(pos->last->bo->base.resv);
67 
68 			man = ttm_manager_type(pos->first->bo->bdev, i);
69 			list_bulk_move_tail(&man->lru[j], &pos->first->lru,
70 					    &pos->last->lru);
71 		}
72 	}
73 }
74 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
75 
76 /* Return the bulk move pos object for this resource */
77 static struct ttm_lru_bulk_move_pos *
ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)78 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
79 {
80 	return &bulk->pos[res->mem_type][res->bo->priority];
81 }
82 
83 /* Move the resource to the tail of the bulk move range */
ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos * pos,struct ttm_resource * res)84 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
85 				       struct ttm_resource *res)
86 {
87 	if (pos->last != res) {
88 		list_move(&res->lru, &pos->last->lru);
89 		pos->last = res;
90 	}
91 }
92 
93 /* Add the resource to a bulk_move cursor */
ttm_lru_bulk_move_add(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)94 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
95 				  struct ttm_resource *res)
96 {
97 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
98 
99 	if (!pos->first) {
100 		pos->first = res;
101 		pos->last = res;
102 	} else {
103 		ttm_lru_bulk_move_pos_tail(pos, res);
104 	}
105 }
106 
107 /* Remove the resource from a bulk_move range */
ttm_lru_bulk_move_del(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)108 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
109 				  struct ttm_resource *res)
110 {
111 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
112 
113 	if (unlikely(pos->first == res && pos->last == res)) {
114 		pos->first = NULL;
115 		pos->last = NULL;
116 	} else if (pos->first == res) {
117 		pos->first = list_next_entry(res, lru);
118 	} else if (pos->last == res) {
119 		pos->last = list_prev_entry(res, lru);
120 	} else {
121 		list_move(&res->lru, &pos->last->lru);
122 	}
123 }
124 
125 /* Add the resource to a bulk move if the BO is configured for it */
ttm_resource_add_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)126 void ttm_resource_add_bulk_move(struct ttm_resource *res,
127 				struct ttm_buffer_object *bo)
128 {
129 	if (bo->bulk_move && !bo->pin_count)
130 		ttm_lru_bulk_move_add(bo->bulk_move, res);
131 }
132 
133 /* Remove the resource from a bulk move if the BO is configured for it */
ttm_resource_del_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)134 void ttm_resource_del_bulk_move(struct ttm_resource *res,
135 				struct ttm_buffer_object *bo)
136 {
137 	if (bo->bulk_move && !bo->pin_count)
138 		ttm_lru_bulk_move_del(bo->bulk_move, res);
139 }
140 
141 /* Move a resource to the LRU or bulk tail */
ttm_resource_move_to_lru_tail(struct ttm_resource * res)142 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
143 {
144 	struct ttm_buffer_object *bo = res->bo;
145 	struct ttm_device *bdev = bo->bdev;
146 
147 	lockdep_assert_held(&bo->bdev->lru_lock);
148 
149 	if (bo->pin_count) {
150 		list_move_tail(&res->lru, &bdev->pinned);
151 
152 	} else	if (bo->bulk_move) {
153 		struct ttm_lru_bulk_move_pos *pos =
154 			ttm_lru_bulk_move_pos(bo->bulk_move, res);
155 
156 		ttm_lru_bulk_move_pos_tail(pos, res);
157 	} else {
158 		struct ttm_resource_manager *man;
159 
160 		man = ttm_manager_type(bdev, res->mem_type);
161 		list_move_tail(&res->lru, &man->lru[bo->priority]);
162 	}
163 }
164 
165 /**
166  * ttm_resource_init - resource object constructure
167  * @bo: buffer object this resources is allocated for
168  * @place: placement of the resource
169  * @res: the resource object to inistilize
170  *
171  * Initialize a new resource object. Counterpart of ttm_resource_fini().
172  */
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)173 void ttm_resource_init(struct ttm_buffer_object *bo,
174                        const struct ttm_place *place,
175                        struct ttm_resource *res)
176 {
177 	struct ttm_resource_manager *man;
178 
179 	res->start = 0;
180 	res->num_pages = PFN_UP(bo->base.size);
181 	res->mem_type = place->mem_type;
182 	res->placement = place->flags;
183 	res->bus.addr = NULL;
184 	res->bus.offset = 0;
185 	res->bus.is_iomem = false;
186 	res->bus.caching = ttm_cached;
187 	res->bo = bo;
188 
189 	man = ttm_manager_type(bo->bdev, place->mem_type);
190 	spin_lock(&bo->bdev->lru_lock);
191 	if (bo->pin_count)
192 		list_add_tail(&res->lru, &bo->bdev->pinned);
193 	else
194 		list_add_tail(&res->lru, &man->lru[bo->priority]);
195 	man->usage += res->num_pages << PAGE_SHIFT;
196 	spin_unlock(&bo->bdev->lru_lock);
197 }
198 EXPORT_SYMBOL(ttm_resource_init);
199 
200 /**
201  * ttm_resource_fini - resource destructor
202  * @man: the resource manager this resource belongs to
203  * @res: the resource to clean up
204  *
205  * Should be used by resource manager backends to clean up the TTM resource
206  * objects before freeing the underlying structure. Makes sure the resource is
207  * removed from the LRU before destruction.
208  * Counterpart of ttm_resource_init().
209  */
ttm_resource_fini(struct ttm_resource_manager * man,struct ttm_resource * res)210 void ttm_resource_fini(struct ttm_resource_manager *man,
211 		       struct ttm_resource *res)
212 {
213 	struct ttm_device *bdev = man->bdev;
214 
215 	spin_lock(&bdev->lru_lock);
216 	list_del_init(&res->lru);
217 	man->usage -= res->num_pages << PAGE_SHIFT;
218 	spin_unlock(&bdev->lru_lock);
219 }
220 EXPORT_SYMBOL(ttm_resource_fini);
221 
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)222 int ttm_resource_alloc(struct ttm_buffer_object *bo,
223 		       const struct ttm_place *place,
224 		       struct ttm_resource **res_ptr)
225 {
226 	struct ttm_resource_manager *man =
227 		ttm_manager_type(bo->bdev, place->mem_type);
228 	int ret;
229 
230 	ret = man->func->alloc(man, bo, place, res_ptr);
231 	if (ret)
232 		return ret;
233 
234 	spin_lock(&bo->bdev->lru_lock);
235 	ttm_resource_add_bulk_move(*res_ptr, bo);
236 	spin_unlock(&bo->bdev->lru_lock);
237 	return 0;
238 }
239 
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)240 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
241 {
242 	struct ttm_resource_manager *man;
243 
244 	if (!*res)
245 		return;
246 
247 	spin_lock(&bo->bdev->lru_lock);
248 	ttm_resource_del_bulk_move(*res, bo);
249 	spin_unlock(&bo->bdev->lru_lock);
250 	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
251 	man->func->free(man, *res);
252 	*res = NULL;
253 }
254 EXPORT_SYMBOL(ttm_resource_free);
255 
256 /**
257  * ttm_resource_intersects - test for intersection
258  *
259  * @bdev: TTM device structure
260  * @res: The resource to test
261  * @place: The placement to test
262  * @size: How many bytes the new allocation needs.
263  *
264  * Test if @res intersects with @place and @size. Used for testing if evictions
265  * are valueable or not.
266  *
267  * Returns true if the res placement intersects with @place and @size.
268  */
ttm_resource_intersects(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)269 bool ttm_resource_intersects(struct ttm_device *bdev,
270 			     struct ttm_resource *res,
271 			     const struct ttm_place *place,
272 			     size_t size)
273 {
274 	struct ttm_resource_manager *man;
275 
276 	if (!res)
277 		return false;
278 
279 	man = ttm_manager_type(bdev, res->mem_type);
280 	if (!place || !man->func->intersects)
281 		return true;
282 
283 	return man->func->intersects(man, res, place, size);
284 }
285 
286 /**
287  * ttm_resource_compatible - test for compatibility
288  *
289  * @bdev: TTM device structure
290  * @res: The resource to test
291  * @place: The placement to test
292  * @size: How many bytes the new allocation needs.
293  *
294  * Test if @res compatible with @place and @size.
295  *
296  * Returns true if the res placement compatible with @place and @size.
297  */
ttm_resource_compatible(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)298 bool ttm_resource_compatible(struct ttm_device *bdev,
299 			     struct ttm_resource *res,
300 			     const struct ttm_place *place,
301 			     size_t size)
302 {
303 	struct ttm_resource_manager *man;
304 
305 	if (!res || !place)
306 		return false;
307 
308 	man = ttm_manager_type(bdev, res->mem_type);
309 	if (!man->func->compatible)
310 		return true;
311 
312 	return man->func->compatible(man, res, place, size);
313 }
314 
ttm_resource_places_compat(struct ttm_resource * res,const struct ttm_place * places,unsigned num_placement)315 static bool ttm_resource_places_compat(struct ttm_resource *res,
316 				       const struct ttm_place *places,
317 				       unsigned num_placement)
318 {
319 	struct ttm_buffer_object *bo = res->bo;
320 	struct ttm_device *bdev = bo->bdev;
321 	unsigned i;
322 
323 	if (res->placement & TTM_PL_FLAG_TEMPORARY)
324 		return false;
325 
326 	for (i = 0; i < num_placement; i++) {
327 		const struct ttm_place *heap = &places[i];
328 
329 		if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
330 			continue;
331 
332 		if ((res->mem_type == heap->mem_type) &&
333 		    (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
334 		     (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
335 			return true;
336 	}
337 	return false;
338 }
339 
340 /**
341  * ttm_resource_compat - check if resource is compatible with placement
342  *
343  * @res: the resource to check
344  * @placement: the placement to check against
345  *
346  * Returns true if the placement is compatible.
347  */
ttm_resource_compat(struct ttm_resource * res,struct ttm_placement * placement)348 bool ttm_resource_compat(struct ttm_resource *res,
349 			 struct ttm_placement *placement)
350 {
351 	if (ttm_resource_places_compat(res, placement->placement,
352 				       placement->num_placement))
353 		return true;
354 
355 	if ((placement->busy_placement != placement->placement ||
356 	     placement->num_busy_placement > placement->num_placement) &&
357 	    ttm_resource_places_compat(res, placement->busy_placement,
358 				       placement->num_busy_placement))
359 		return true;
360 
361 	return false;
362 }
363 EXPORT_SYMBOL(ttm_resource_compat);
364 
ttm_resource_set_bo(struct ttm_resource * res,struct ttm_buffer_object * bo)365 void ttm_resource_set_bo(struct ttm_resource *res,
366 			 struct ttm_buffer_object *bo)
367 {
368 	spin_lock(&bo->bdev->lru_lock);
369 	res->bo = bo;
370 	spin_unlock(&bo->bdev->lru_lock);
371 }
372 
373 /**
374  * ttm_resource_manager_init
375  *
376  * @man: memory manager object to init
377  * @bdev: ttm device this manager belongs to
378  * @size: size of managed resources in arbitrary units
379  *
380  * Initialise core parts of a manager object.
381  */
ttm_resource_manager_init(struct ttm_resource_manager * man,struct ttm_device * bdev,uint64_t size)382 void ttm_resource_manager_init(struct ttm_resource_manager *man,
383 			       struct ttm_device *bdev,
384 			       uint64_t size)
385 {
386 	unsigned i;
387 
388 	spin_lock_init(&man->move_lock);
389 	man->bdev = bdev;
390 	man->size = size;
391 	man->usage = 0;
392 
393 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
394 		INIT_LIST_HEAD(&man->lru[i]);
395 	man->move = NULL;
396 }
397 EXPORT_SYMBOL(ttm_resource_manager_init);
398 
399 /*
400  * ttm_resource_manager_evict_all
401  *
402  * @bdev - device to use
403  * @man - manager to use
404  *
405  * Evict all the objects out of a memory manager until it is empty.
406  * Part of memory manager cleanup sequence.
407  */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)408 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
409 				   struct ttm_resource_manager *man)
410 {
411 	struct ttm_operation_ctx ctx = {
412 		.interruptible = false,
413 		.no_wait_gpu = false,
414 		.force_alloc = true
415 	};
416 	struct dma_fence *fence;
417 	int ret;
418 	unsigned i;
419 
420 	/*
421 	 * Can't use standard list traversal since we're unlocking.
422 	 */
423 
424 	spin_lock(&bdev->lru_lock);
425 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
426 		while (!list_empty(&man->lru[i])) {
427 			spin_unlock(&bdev->lru_lock);
428 			ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
429 						  NULL);
430 			if (ret)
431 				return ret;
432 			spin_lock(&bdev->lru_lock);
433 		}
434 	}
435 	spin_unlock(&bdev->lru_lock);
436 
437 	spin_lock(&man->move_lock);
438 	fence = dma_fence_get(man->move);
439 	spin_unlock(&man->move_lock);
440 
441 	if (fence) {
442 		ret = dma_fence_wait(fence, false);
443 		dma_fence_put(fence);
444 		if (ret)
445 			return ret;
446 	}
447 
448 	return 0;
449 }
450 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
451 
452 /**
453  * ttm_resource_manager_usage
454  *
455  * @man: A memory manager object.
456  *
457  * Return how many resources are currently used.
458  */
ttm_resource_manager_usage(struct ttm_resource_manager * man)459 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
460 {
461 	uint64_t usage;
462 
463 	spin_lock(&man->bdev->lru_lock);
464 	usage = man->usage;
465 	spin_unlock(&man->bdev->lru_lock);
466 	return usage;
467 }
468 EXPORT_SYMBOL(ttm_resource_manager_usage);
469 
470 /**
471  * ttm_resource_manager_debug
472  *
473  * @man: manager type to dump.
474  * @p: printer to use for debug.
475  */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)476 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
477 				struct drm_printer *p)
478 {
479 	drm_printf(p, "  use_type: %d\n", man->use_type);
480 	drm_printf(p, "  use_tt: %d\n", man->use_tt);
481 	drm_printf(p, "  size: %llu\n", man->size);
482 	drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
483 	if (man->func->debug)
484 		man->func->debug(man, p);
485 }
486 EXPORT_SYMBOL(ttm_resource_manager_debug);
487 
488 /**
489  * ttm_resource_manager_first
490  *
491  * @man: resource manager to iterate over
492  * @cursor: cursor to record the position
493  *
494  * Returns the first resource from the resource manager.
495  */
496 struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor)497 ttm_resource_manager_first(struct ttm_resource_manager *man,
498 			   struct ttm_resource_cursor *cursor)
499 {
500 	struct ttm_resource *res;
501 
502 	lockdep_assert_held(&man->bdev->lru_lock);
503 
504 	for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
505 	     ++cursor->priority)
506 		list_for_each_entry(res, &man->lru[cursor->priority], lru)
507 			return res;
508 
509 	return NULL;
510 }
511 
512 /**
513  * ttm_resource_manager_next
514  *
515  * @man: resource manager to iterate over
516  * @cursor: cursor to record the position
517  * @res: the current resource pointer
518  *
519  * Returns the next resource from the resource manager.
520  */
521 struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor,struct ttm_resource * res)522 ttm_resource_manager_next(struct ttm_resource_manager *man,
523 			  struct ttm_resource_cursor *cursor,
524 			  struct ttm_resource *res)
525 {
526 	lockdep_assert_held(&man->bdev->lru_lock);
527 
528 	list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
529 		return res;
530 
531 	for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
532 	     ++cursor->priority)
533 		list_for_each_entry(res, &man->lru[cursor->priority], lru)
534 			return res;
535 
536 	return NULL;
537 }
538 
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)539 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
540 					  struct iosys_map *dmap,
541 					  pgoff_t i)
542 {
543 	struct ttm_kmap_iter_iomap *iter_io =
544 		container_of(iter, typeof(*iter_io), base);
545 	void __iomem *addr;
546 
547 retry:
548 	while (i >= iter_io->cache.end) {
549 		iter_io->cache.sg = iter_io->cache.sg ?
550 			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
551 		iter_io->cache.i = iter_io->cache.end;
552 		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
553 			PAGE_SHIFT;
554 		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
555 			iter_io->start;
556 	}
557 
558 	if (i < iter_io->cache.i) {
559 		iter_io->cache.end = 0;
560 		iter_io->cache.sg = NULL;
561 		goto retry;
562 	}
563 
564 	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
565 				       (((resource_size_t)i - iter_io->cache.i)
566 					<< PAGE_SHIFT));
567 	iosys_map_set_vaddr_iomem(dmap, addr);
568 }
569 
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct iosys_map * map)570 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
571 					    struct iosys_map *map)
572 {
573 	io_mapping_unmap_local(map->vaddr_iomem);
574 }
575 
576 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
577 	.map_local =  ttm_kmap_iter_iomap_map_local,
578 	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
579 	.maps_tt = false,
580 };
581 
582 /**
583  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
584  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
585  * @iomap: The struct io_mapping representing the underlying linear io_memory.
586  * @st: sg_table into @iomap, representing the memory of the struct
587  * ttm_resource.
588  * @start: Offset that needs to be subtracted from @st to make
589  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
590  *
591  * Return: Pointer to the embedded struct ttm_kmap_iter.
592  */
593 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)594 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
595 			 struct io_mapping *iomap,
596 			 struct sg_table *st,
597 			 resource_size_t start)
598 {
599 	iter_io->base.ops = &ttm_kmap_iter_io_ops;
600 	iter_io->iomap = iomap;
601 	iter_io->st = st;
602 	iter_io->start = start;
603 	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
604 
605 	return &iter_io->base;
606 }
607 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
608 
609 /**
610  * DOC: Linear io iterator
611  *
612  * This code should die in the not too near future. Best would be if we could
613  * make io-mapping use memremap for all io memory, and have memremap
614  * implement a kmap_local functionality. We could then strip a huge amount of
615  * code. These linear io iterators are implemented to mimic old functionality,
616  * and they don't use kmap_local semantics at all internally. Rather ioremap or
617  * friends, and at least on 32-bit they add global TLB flushes and points
618  * of failure.
619  */
620 
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)621 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
622 					      struct iosys_map *dmap,
623 					      pgoff_t i)
624 {
625 	struct ttm_kmap_iter_linear_io *iter_io =
626 		container_of(iter, typeof(*iter_io), base);
627 
628 	*dmap = iter_io->dmap;
629 	iosys_map_incr(dmap, i * PAGE_SIZE);
630 }
631 
632 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
633 	.map_local =  ttm_kmap_iter_linear_io_map_local,
634 	.maps_tt = false,
635 };
636 
637 /**
638  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
639  * @iter_io: The iterator to initialize
640  * @bdev: The TTM device
641  * @mem: The ttm resource representing the iomap.
642  *
643  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
644  * pointing at a linear chunk of io memory.
645  *
646  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
647  * failure.
648  */
649 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)650 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
651 			     struct ttm_device *bdev,
652 			     struct ttm_resource *mem)
653 {
654 	int ret;
655 
656 	ret = ttm_mem_io_reserve(bdev, mem);
657 	if (ret)
658 		goto out_err;
659 	if (!mem->bus.is_iomem) {
660 		ret = -EINVAL;
661 		goto out_io_free;
662 	}
663 
664 	if (mem->bus.addr) {
665 		iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
666 		iter_io->needs_unmap = false;
667 	} else {
668 		size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
669 
670 		iter_io->needs_unmap = true;
671 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
672 		if (mem->bus.caching == ttm_write_combined)
673 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
674 						  ioremap_wc(mem->bus.offset,
675 							     bus_size));
676 		else if (mem->bus.caching == ttm_cached)
677 			iosys_map_set_vaddr(&iter_io->dmap,
678 					    memremap(mem->bus.offset, bus_size,
679 						     MEMREMAP_WB |
680 						     MEMREMAP_WT |
681 						     MEMREMAP_WC));
682 
683 		/* If uncached requested or if mapping cached or wc failed */
684 		if (iosys_map_is_null(&iter_io->dmap))
685 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
686 						  ioremap(mem->bus.offset,
687 							  bus_size));
688 
689 		if (iosys_map_is_null(&iter_io->dmap)) {
690 			ret = -ENOMEM;
691 			goto out_io_free;
692 		}
693 	}
694 
695 	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
696 	return &iter_io->base;
697 
698 out_io_free:
699 	ttm_mem_io_free(bdev, mem);
700 out_err:
701 	return ERR_PTR(ret);
702 }
703 
704 /**
705  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
706  * @iter_io: The iterator to initialize
707  * @bdev: The TTM device
708  * @mem: The ttm resource representing the iomap.
709  *
710  * This function is for internal TTM use only. It cleans up a memcpy kmap
711  * iterator initialized by ttm_kmap_iter_linear_io_init.
712  */
713 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)714 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
715 			     struct ttm_device *bdev,
716 			     struct ttm_resource *mem)
717 {
718 	if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
719 		if (iter_io->dmap.is_iomem)
720 			iounmap(iter_io->dmap.vaddr_iomem);
721 		else
722 			memunmap(iter_io->dmap.vaddr);
723 	}
724 
725 	ttm_mem_io_free(bdev, mem);
726 }
727 
728 #if defined(CONFIG_DEBUG_FS)
729 
ttm_resource_manager_show(struct seq_file * m,void * unused)730 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
731 {
732 	struct ttm_resource_manager *man =
733 		(struct ttm_resource_manager *)m->private;
734 	struct drm_printer p = drm_seq_file_printer(m);
735 	ttm_resource_manager_debug(man, &p);
736 	return 0;
737 }
738 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
739 
740 #endif
741 
742 /**
743  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
744  * resource manager.
745  * @man: The TTM resource manager for which the debugfs stats file be creates
746  * @parent: debugfs directory in which the file will reside
747  * @name: The filename to create.
748  *
749  * This function setups up a debugfs file that can be used to look
750  * at debug statistics of the specified ttm_resource_manager.
751  */
ttm_resource_manager_create_debugfs(struct ttm_resource_manager * man,struct dentry * parent,const char * name)752 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
753 					 struct dentry * parent,
754 					 const char *name)
755 {
756 #if defined(CONFIG_DEBUG_FS)
757 	debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
758 #endif
759 }
760 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
761