1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
3  *
4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sub license, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial portions
17  * of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  **************************************************************************/
28 /*
29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30  */
31 
32 #define pr_fmt(fmt) "[TTM] " fmt
33 
34 #include <drm/ttm/ttm_bo_driver.h>
35 #include <drm/ttm/ttm_placement.h>
36 #include <linux/jiffies.h>
37 #include <linux/slab.h>
38 #include <linux/sched.h>
39 #include <linux/mm.h>
40 #include <linux/file.h>
41 #include <linux/module.h>
42 #include <linux/atomic.h>
43 #include <linux/dma-resv.h>
44 
45 #include "ttm_module.h"
46 
47 /* default destructor */
ttm_bo_default_destroy(struct ttm_buffer_object * bo)48 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
49 {
50 	kfree(bo);
51 }
52 
ttm_bo_mem_space_debug(struct ttm_buffer_object * bo,struct ttm_placement * placement)53 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
54 					struct ttm_placement *placement)
55 {
56 	struct drm_printer p = drm_debug_printer(TTM_PFX);
57 	struct ttm_resource_manager *man;
58 	int i, mem_type;
59 
60 	drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n",
61 		   bo, bo->resource->num_pages, bo->base.size >> 10,
62 		   bo->base.size >> 20);
63 	for (i = 0; i < placement->num_placement; i++) {
64 		mem_type = placement->placement[i].mem_type;
65 		drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
66 			   i, placement->placement[i].flags, mem_type);
67 		man = ttm_manager_type(bo->bdev, mem_type);
68 		ttm_resource_manager_debug(man, &p);
69 	}
70 }
71 
72 /**
73  * ttm_bo_move_to_lru_tail
74  *
75  * @bo: The buffer object.
76  *
77  * Move this BO to the tail of all lru lists used to lookup and reserve an
78  * object. This function must be called with struct ttm_global::lru_lock
79  * held, and is used to make a BO less likely to be considered for eviction.
80  */
ttm_bo_move_to_lru_tail(struct ttm_buffer_object * bo)81 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
82 {
83 	dma_resv_assert_held(bo->base.resv);
84 
85 	if (bo->resource)
86 		ttm_resource_move_to_lru_tail(bo->resource);
87 }
88 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
89 
90 /**
91  * ttm_bo_set_bulk_move - update BOs bulk move object
92  *
93  * @bo: The buffer object.
94  *
95  * Update the BOs bulk move object, making sure that resources are added/removed
96  * as well. A bulk move allows to move many resource on the LRU at once,
97  * resulting in much less overhead of maintaining the LRU.
98  * The only requirement is that the resources stay together on the LRU and are
99  * never separated. This is enforces by setting the bulk_move structure on a BO.
100  * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
101  * their LRU list.
102  */
ttm_bo_set_bulk_move(struct ttm_buffer_object * bo,struct ttm_lru_bulk_move * bulk)103 void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
104 			  struct ttm_lru_bulk_move *bulk)
105 {
106 	dma_resv_assert_held(bo->base.resv);
107 
108 	if (bo->bulk_move == bulk)
109 		return;
110 
111 	spin_lock(&bo->bdev->lru_lock);
112 	if (bo->resource)
113 		ttm_resource_del_bulk_move(bo->resource, bo);
114 	bo->bulk_move = bulk;
115 	if (bo->resource)
116 		ttm_resource_add_bulk_move(bo->resource, bo);
117 	spin_unlock(&bo->bdev->lru_lock);
118 }
119 EXPORT_SYMBOL(ttm_bo_set_bulk_move);
120 
ttm_bo_handle_move_mem(struct ttm_buffer_object * bo,struct ttm_resource * mem,bool evict,struct ttm_operation_ctx * ctx,struct ttm_place * hop)121 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
122 				  struct ttm_resource *mem, bool evict,
123 				  struct ttm_operation_ctx *ctx,
124 				  struct ttm_place *hop)
125 {
126 	struct ttm_resource_manager *old_man, *new_man;
127 	struct ttm_device *bdev = bo->bdev;
128 	int ret;
129 
130 	old_man = ttm_manager_type(bdev, bo->resource->mem_type);
131 	new_man = ttm_manager_type(bdev, mem->mem_type);
132 
133 	ttm_bo_unmap_virtual(bo);
134 
135 	/*
136 	 * Create and bind a ttm if required.
137 	 */
138 
139 	if (new_man->use_tt) {
140 		/* Zero init the new TTM structure if the old location should
141 		 * have used one as well.
142 		 */
143 		ret = ttm_tt_create(bo, old_man->use_tt);
144 		if (ret)
145 			goto out_err;
146 
147 		if (mem->mem_type != TTM_PL_SYSTEM) {
148 			ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
149 			if (ret)
150 				goto out_err;
151 		}
152 	}
153 
154 	ret = dma_resv_reserve_fences(bo->base.resv, 1);
155 	if (ret)
156 		goto out_err;
157 
158 	ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
159 	if (ret) {
160 		if (ret == -EMULTIHOP)
161 			return ret;
162 		goto out_err;
163 	}
164 
165 	ctx->bytes_moved += bo->base.size;
166 	return 0;
167 
168 out_err:
169 	new_man = ttm_manager_type(bdev, bo->resource->mem_type);
170 	if (!new_man->use_tt)
171 		ttm_bo_tt_destroy(bo);
172 
173 	return ret;
174 }
175 
176 /*
177  * Call bo::reserved.
178  * Will release GPU memory type usage on destruction.
179  * This is the place to put in driver specific hooks to release
180  * driver private resources.
181  * Will release the bo::reserved lock.
182  */
183 
ttm_bo_cleanup_memtype_use(struct ttm_buffer_object * bo)184 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
185 {
186 	if (bo->bdev->funcs->delete_mem_notify)
187 		bo->bdev->funcs->delete_mem_notify(bo);
188 
189 	ttm_bo_tt_destroy(bo);
190 	ttm_resource_free(bo, &bo->resource);
191 }
192 
ttm_bo_individualize_resv(struct ttm_buffer_object * bo)193 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
194 {
195 	int r;
196 
197 	if (bo->base.resv == &bo->base._resv)
198 		return 0;
199 
200 	BUG_ON(!dma_resv_trylock(&bo->base._resv));
201 
202 	r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
203 	dma_resv_unlock(&bo->base._resv);
204 	if (r)
205 		return r;
206 
207 	if (bo->type != ttm_bo_type_sg) {
208 		/* This works because the BO is about to be destroyed and nobody
209 		 * reference it any more. The only tricky case is the trylock on
210 		 * the resv object while holding the lru_lock.
211 		 */
212 		spin_lock(&bo->bdev->lru_lock);
213 		bo->base.resv = &bo->base._resv;
214 		spin_unlock(&bo->bdev->lru_lock);
215 	}
216 
217 	return r;
218 }
219 
ttm_bo_flush_all_fences(struct ttm_buffer_object * bo)220 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
221 {
222 	struct dma_resv *resv = &bo->base._resv;
223 	struct dma_resv_iter cursor;
224 	struct dma_fence *fence;
225 
226 	dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
227 	dma_resv_for_each_fence_unlocked(&cursor, fence) {
228 		if (!fence->ops->signaled)
229 			dma_fence_enable_sw_signaling(fence);
230 	}
231 	dma_resv_iter_end(&cursor);
232 }
233 
234 /**
235  * ttm_bo_cleanup_refs
236  * If bo idle, remove from lru lists, and unref.
237  * If not idle, block if possible.
238  *
239  * Must be called with lru_lock and reservation held, this function
240  * will drop the lru lock and optionally the reservation lock before returning.
241  *
242  * @bo:                    The buffer object to clean-up
243  * @interruptible:         Any sleeps should occur interruptibly.
244  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
245  * @unlock_resv:           Unlock the reservation lock as well.
246  */
247 
ttm_bo_cleanup_refs(struct ttm_buffer_object * bo,bool interruptible,bool no_wait_gpu,bool unlock_resv)248 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
249 			       bool interruptible, bool no_wait_gpu,
250 			       bool unlock_resv)
251 {
252 	struct dma_resv *resv = &bo->base._resv;
253 	int ret;
254 
255 	if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
256 		ret = 0;
257 	else
258 		ret = -EBUSY;
259 
260 	if (ret && !no_wait_gpu) {
261 		long lret;
262 
263 		if (unlock_resv)
264 			dma_resv_unlock(bo->base.resv);
265 		spin_unlock(&bo->bdev->lru_lock);
266 
267 		lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
268 					     interruptible,
269 					     30 * HZ);
270 
271 		if (lret < 0)
272 			return lret;
273 		else if (lret == 0)
274 			return -EBUSY;
275 
276 		spin_lock(&bo->bdev->lru_lock);
277 		if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
278 			/*
279 			 * We raced, and lost, someone else holds the reservation now,
280 			 * and is probably busy in ttm_bo_cleanup_memtype_use.
281 			 *
282 			 * Even if it's not the case, because we finished waiting any
283 			 * delayed destruction would succeed, so just return success
284 			 * here.
285 			 */
286 			spin_unlock(&bo->bdev->lru_lock);
287 			return 0;
288 		}
289 		ret = 0;
290 	}
291 
292 	if (ret || unlikely(list_empty(&bo->ddestroy))) {
293 		if (unlock_resv)
294 			dma_resv_unlock(bo->base.resv);
295 		spin_unlock(&bo->bdev->lru_lock);
296 		return ret;
297 	}
298 
299 	list_del_init(&bo->ddestroy);
300 	spin_unlock(&bo->bdev->lru_lock);
301 	ttm_bo_cleanup_memtype_use(bo);
302 
303 	if (unlock_resv)
304 		dma_resv_unlock(bo->base.resv);
305 
306 	ttm_bo_put(bo);
307 
308 	return 0;
309 }
310 
311 /*
312  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
313  * encountered buffers.
314  */
ttm_bo_delayed_delete(struct ttm_device * bdev,bool remove_all)315 bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all)
316 {
317 	struct list_head removed;
318 	bool empty;
319 
320 	INIT_LIST_HEAD(&removed);
321 
322 	spin_lock(&bdev->lru_lock);
323 	while (!list_empty(&bdev->ddestroy)) {
324 		struct ttm_buffer_object *bo;
325 
326 		bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
327 				      ddestroy);
328 		list_move_tail(&bo->ddestroy, &removed);
329 		if (!ttm_bo_get_unless_zero(bo))
330 			continue;
331 
332 		if (remove_all || bo->base.resv != &bo->base._resv) {
333 			spin_unlock(&bdev->lru_lock);
334 			dma_resv_lock(bo->base.resv, NULL);
335 
336 			spin_lock(&bdev->lru_lock);
337 			ttm_bo_cleanup_refs(bo, false, !remove_all, true);
338 
339 		} else if (dma_resv_trylock(bo->base.resv)) {
340 			ttm_bo_cleanup_refs(bo, false, !remove_all, true);
341 		} else {
342 			spin_unlock(&bdev->lru_lock);
343 		}
344 
345 		ttm_bo_put(bo);
346 		spin_lock(&bdev->lru_lock);
347 	}
348 	list_splice_tail(&removed, &bdev->ddestroy);
349 	empty = list_empty(&bdev->ddestroy);
350 	spin_unlock(&bdev->lru_lock);
351 
352 	return empty;
353 }
354 
ttm_bo_release(struct kref * kref)355 static void ttm_bo_release(struct kref *kref)
356 {
357 	struct ttm_buffer_object *bo =
358 	    container_of(kref, struct ttm_buffer_object, kref);
359 	struct ttm_device *bdev = bo->bdev;
360 	int ret;
361 
362 	WARN_ON_ONCE(bo->pin_count);
363 	WARN_ON_ONCE(bo->bulk_move);
364 
365 	if (!bo->deleted) {
366 		ret = ttm_bo_individualize_resv(bo);
367 		if (ret) {
368 			/* Last resort, if we fail to allocate memory for the
369 			 * fences block for the BO to become idle
370 			 */
371 			dma_resv_wait_timeout(bo->base.resv,
372 					      DMA_RESV_USAGE_BOOKKEEP, false,
373 					      30 * HZ);
374 		}
375 
376 		if (bo->bdev->funcs->release_notify)
377 			bo->bdev->funcs->release_notify(bo);
378 
379 		drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
380 		ttm_mem_io_free(bdev, bo->resource);
381 	}
382 
383 	if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP) ||
384 	    !dma_resv_trylock(bo->base.resv)) {
385 		/* The BO is not idle, resurrect it for delayed destroy */
386 		ttm_bo_flush_all_fences(bo);
387 		bo->deleted = true;
388 
389 		spin_lock(&bo->bdev->lru_lock);
390 
391 		/*
392 		 * Make pinned bos immediately available to
393 		 * shrinkers, now that they are queued for
394 		 * destruction.
395 		 *
396 		 * FIXME: QXL is triggering this. Can be removed when the
397 		 * driver is fixed.
398 		 */
399 		if (bo->pin_count) {
400 			bo->pin_count = 0;
401 			ttm_resource_move_to_lru_tail(bo->resource);
402 		}
403 
404 		kref_init(&bo->kref);
405 		list_add_tail(&bo->ddestroy, &bdev->ddestroy);
406 		spin_unlock(&bo->bdev->lru_lock);
407 
408 		schedule_delayed_work(&bdev->wq,
409 				      ((HZ / 100) < 1) ? 1 : HZ / 100);
410 		return;
411 	}
412 
413 	spin_lock(&bo->bdev->lru_lock);
414 	list_del(&bo->ddestroy);
415 	spin_unlock(&bo->bdev->lru_lock);
416 
417 	ttm_bo_cleanup_memtype_use(bo);
418 	dma_resv_unlock(bo->base.resv);
419 
420 	atomic_dec(&ttm_glob.bo_count);
421 	bo->destroy(bo);
422 }
423 
ttm_bo_put(struct ttm_buffer_object * bo)424 void ttm_bo_put(struct ttm_buffer_object *bo)
425 {
426 	kref_put(&bo->kref, ttm_bo_release);
427 }
428 EXPORT_SYMBOL(ttm_bo_put);
429 
ttm_bo_lock_delayed_workqueue(struct ttm_device * bdev)430 int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev)
431 {
432 	return cancel_delayed_work_sync(&bdev->wq);
433 }
434 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
435 
ttm_bo_unlock_delayed_workqueue(struct ttm_device * bdev,int resched)436 void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched)
437 {
438 	if (resched)
439 		schedule_delayed_work(&bdev->wq,
440 				      ((HZ / 100) < 1) ? 1 : HZ / 100);
441 }
442 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
443 
ttm_bo_bounce_temp_buffer(struct ttm_buffer_object * bo,struct ttm_resource ** mem,struct ttm_operation_ctx * ctx,struct ttm_place * hop)444 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
445 				     struct ttm_resource **mem,
446 				     struct ttm_operation_ctx *ctx,
447 				     struct ttm_place *hop)
448 {
449 	struct ttm_placement hop_placement;
450 	struct ttm_resource *hop_mem;
451 	int ret;
452 
453 	hop_placement.num_placement = hop_placement.num_busy_placement = 1;
454 	hop_placement.placement = hop_placement.busy_placement = hop;
455 
456 	/* find space in the bounce domain */
457 	ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
458 	if (ret)
459 		return ret;
460 	/* move to the bounce domain */
461 	ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
462 	if (ret) {
463 		ttm_resource_free(bo, &hop_mem);
464 		return ret;
465 	}
466 	return 0;
467 }
468 
ttm_bo_evict(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx)469 static int ttm_bo_evict(struct ttm_buffer_object *bo,
470 			struct ttm_operation_ctx *ctx)
471 {
472 	struct ttm_device *bdev = bo->bdev;
473 	struct ttm_resource *evict_mem;
474 	struct ttm_placement placement;
475 	struct ttm_place hop;
476 	int ret = 0;
477 
478 	memset(&hop, 0, sizeof(hop));
479 
480 	dma_resv_assert_held(bo->base.resv);
481 
482 	placement.num_placement = 0;
483 	placement.num_busy_placement = 0;
484 	bdev->funcs->evict_flags(bo, &placement);
485 
486 	if (!placement.num_placement && !placement.num_busy_placement) {
487 		ret = ttm_bo_wait(bo, true, false);
488 		if (ret)
489 			return ret;
490 
491 		/*
492 		 * Since we've already synced, this frees backing store
493 		 * immediately.
494 		 */
495 		return ttm_bo_pipeline_gutting(bo);
496 	}
497 
498 	ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
499 	if (ret) {
500 		if (ret != -ERESTARTSYS) {
501 			pr_err("Failed to find memory space for buffer 0x%p eviction\n",
502 			       bo);
503 			ttm_bo_mem_space_debug(bo, &placement);
504 		}
505 		goto out;
506 	}
507 
508 bounce:
509 	ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
510 	if (ret == -EMULTIHOP) {
511 		ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
512 		if (ret) {
513 			pr_err("Buffer eviction failed\n");
514 			ttm_resource_free(bo, &evict_mem);
515 			goto out;
516 		}
517 		/* try and move to final place now. */
518 		goto bounce;
519 	}
520 out:
521 	return ret;
522 }
523 
ttm_bo_eviction_valuable(struct ttm_buffer_object * bo,const struct ttm_place * place)524 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
525 			      const struct ttm_place *place)
526 {
527 	dma_resv_assert_held(bo->base.resv);
528 	if (bo->resource->mem_type == TTM_PL_SYSTEM)
529 		return true;
530 
531 	/* Don't evict this BO if it's outside of the
532 	 * requested placement range
533 	 */
534 	if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) ||
535 	    (place->lpfn && place->lpfn <= bo->resource->start))
536 		return false;
537 
538 	return true;
539 }
540 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
541 
542 /*
543  * Check the target bo is allowable to be evicted or swapout, including cases:
544  *
545  * a. if share same reservation object with ctx->resv, have assumption
546  * reservation objects should already be locked, so not lock again and
547  * return true directly when either the opreation allow_reserved_eviction
548  * or the target bo already is in delayed free list;
549  *
550  * b. Otherwise, trylock it.
551  */
ttm_bo_evict_swapout_allowable(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,const struct ttm_place * place,bool * locked,bool * busy)552 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
553 					   struct ttm_operation_ctx *ctx,
554 					   const struct ttm_place *place,
555 					   bool *locked, bool *busy)
556 {
557 	bool ret = false;
558 
559 	if (bo->base.resv == ctx->resv) {
560 		dma_resv_assert_held(bo->base.resv);
561 		if (ctx->allow_res_evict)
562 			ret = true;
563 		*locked = false;
564 		if (busy)
565 			*busy = false;
566 	} else {
567 		ret = dma_resv_trylock(bo->base.resv);
568 		*locked = ret;
569 		if (busy)
570 			*busy = !ret;
571 	}
572 
573 	if (ret && place && (bo->resource->mem_type != place->mem_type ||
574 		!bo->bdev->funcs->eviction_valuable(bo, place))) {
575 		ret = false;
576 		if (*locked) {
577 			dma_resv_unlock(bo->base.resv);
578 			*locked = false;
579 		}
580 	}
581 
582 	return ret;
583 }
584 
585 /**
586  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
587  *
588  * @busy_bo: BO which couldn't be locked with trylock
589  * @ctx: operation context
590  * @ticket: acquire ticket
591  *
592  * Try to lock a busy buffer object to avoid failing eviction.
593  */
ttm_mem_evict_wait_busy(struct ttm_buffer_object * busy_bo,struct ttm_operation_ctx * ctx,struct ww_acquire_ctx * ticket)594 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
595 				   struct ttm_operation_ctx *ctx,
596 				   struct ww_acquire_ctx *ticket)
597 {
598 	int r;
599 
600 	if (!busy_bo || !ticket)
601 		return -EBUSY;
602 
603 	if (ctx->interruptible)
604 		r = dma_resv_lock_interruptible(busy_bo->base.resv,
605 							  ticket);
606 	else
607 		r = dma_resv_lock(busy_bo->base.resv, ticket);
608 
609 	/*
610 	 * TODO: It would be better to keep the BO locked until allocation is at
611 	 * least tried one more time, but that would mean a much larger rework
612 	 * of TTM.
613 	 */
614 	if (!r)
615 		dma_resv_unlock(busy_bo->base.resv);
616 
617 	return r == -EDEADLK ? -EBUSY : r;
618 }
619 
ttm_mem_evict_first(struct ttm_device * bdev,struct ttm_resource_manager * man,const struct ttm_place * place,struct ttm_operation_ctx * ctx,struct ww_acquire_ctx * ticket)620 int ttm_mem_evict_first(struct ttm_device *bdev,
621 			struct ttm_resource_manager *man,
622 			const struct ttm_place *place,
623 			struct ttm_operation_ctx *ctx,
624 			struct ww_acquire_ctx *ticket)
625 {
626 	struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
627 	struct ttm_resource_cursor cursor;
628 	struct ttm_resource *res;
629 	bool locked = false;
630 	int ret;
631 
632 	spin_lock(&bdev->lru_lock);
633 	ttm_resource_manager_for_each_res(man, &cursor, res) {
634 		bool busy;
635 
636 		if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
637 						    &locked, &busy)) {
638 			if (busy && !busy_bo && ticket !=
639 			    dma_resv_locking_ctx(res->bo->base.resv))
640 				busy_bo = res->bo;
641 			continue;
642 		}
643 
644 		if (ttm_bo_get_unless_zero(res->bo)) {
645 			bo = res->bo;
646 			break;
647 		}
648 		if (locked)
649 			dma_resv_unlock(res->bo->base.resv);
650 	}
651 
652 	if (!bo) {
653 		if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
654 			busy_bo = NULL;
655 		spin_unlock(&bdev->lru_lock);
656 		ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
657 		if (busy_bo)
658 			ttm_bo_put(busy_bo);
659 		return ret;
660 	}
661 
662 	if (bo->deleted) {
663 		ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
664 					  ctx->no_wait_gpu, locked);
665 		ttm_bo_put(bo);
666 		return ret;
667 	}
668 
669 	spin_unlock(&bdev->lru_lock);
670 
671 	ret = ttm_bo_evict(bo, ctx);
672 	if (locked)
673 		ttm_bo_unreserve(bo);
674 	else
675 		ttm_bo_move_to_lru_tail_unlocked(bo);
676 
677 	ttm_bo_put(bo);
678 	return ret;
679 }
680 
681 /**
682  * ttm_bo_pin - Pin the buffer object.
683  * @bo: The buffer object to pin
684  *
685  * Make sure the buffer is not evicted any more during memory pressure.
686  * @bo must be unpinned again by calling ttm_bo_unpin().
687  */
ttm_bo_pin(struct ttm_buffer_object * bo)688 void ttm_bo_pin(struct ttm_buffer_object *bo)
689 {
690 	dma_resv_assert_held(bo->base.resv);
691 	WARN_ON_ONCE(!kref_read(&bo->kref));
692 	spin_lock(&bo->bdev->lru_lock);
693 	if (bo->resource)
694 		ttm_resource_del_bulk_move(bo->resource, bo);
695 	++bo->pin_count;
696 	spin_unlock(&bo->bdev->lru_lock);
697 }
698 EXPORT_SYMBOL(ttm_bo_pin);
699 
700 /**
701  * ttm_bo_unpin - Unpin the buffer object.
702  * @bo: The buffer object to unpin
703  *
704  * Allows the buffer object to be evicted again during memory pressure.
705  */
ttm_bo_unpin(struct ttm_buffer_object * bo)706 void ttm_bo_unpin(struct ttm_buffer_object *bo)
707 {
708 	dma_resv_assert_held(bo->base.resv);
709 	WARN_ON_ONCE(!kref_read(&bo->kref));
710 	if (WARN_ON_ONCE(!bo->pin_count))
711 		return;
712 
713 	spin_lock(&bo->bdev->lru_lock);
714 	--bo->pin_count;
715 	if (bo->resource)
716 		ttm_resource_add_bulk_move(bo->resource, bo);
717 	spin_unlock(&bo->bdev->lru_lock);
718 }
719 EXPORT_SYMBOL(ttm_bo_unpin);
720 
721 /*
722  * Add the last move fence to the BO as kernel dependency and reserve a new
723  * fence slot.
724  */
ttm_bo_add_move_fence(struct ttm_buffer_object * bo,struct ttm_resource_manager * man,struct ttm_resource * mem,bool no_wait_gpu)725 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
726 				 struct ttm_resource_manager *man,
727 				 struct ttm_resource *mem,
728 				 bool no_wait_gpu)
729 {
730 	struct dma_fence *fence;
731 	int ret;
732 
733 	spin_lock(&man->move_lock);
734 	fence = dma_fence_get(man->move);
735 	spin_unlock(&man->move_lock);
736 
737 	if (!fence)
738 		return 0;
739 
740 	if (no_wait_gpu) {
741 		ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
742 		dma_fence_put(fence);
743 		return ret;
744 	}
745 
746 	dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
747 
748 	ret = dma_resv_reserve_fences(bo->base.resv, 1);
749 	dma_fence_put(fence);
750 	return ret;
751 }
752 
753 /*
754  * Repeatedly evict memory from the LRU for @mem_type until we create enough
755  * space, or we've evicted everything and there isn't enough space.
756  */
ttm_bo_mem_force_space(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** mem,struct ttm_operation_ctx * ctx)757 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
758 				  const struct ttm_place *place,
759 				  struct ttm_resource **mem,
760 				  struct ttm_operation_ctx *ctx)
761 {
762 	struct ttm_device *bdev = bo->bdev;
763 	struct ttm_resource_manager *man;
764 	struct ww_acquire_ctx *ticket;
765 	int ret;
766 
767 	man = ttm_manager_type(bdev, place->mem_type);
768 	ticket = dma_resv_locking_ctx(bo->base.resv);
769 	do {
770 		ret = ttm_resource_alloc(bo, place, mem);
771 		if (likely(!ret))
772 			break;
773 		if (unlikely(ret != -ENOSPC))
774 			return ret;
775 		ret = ttm_mem_evict_first(bdev, man, place, ctx,
776 					  ticket);
777 		if (unlikely(ret != 0))
778 			return ret;
779 	} while (1);
780 
781 	return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
782 }
783 
784 /*
785  * Creates space for memory region @mem according to its type.
786  *
787  * This function first searches for free space in compatible memory types in
788  * the priority order defined by the driver.  If free space isn't found, then
789  * ttm_bo_mem_force_space is attempted in priority order to evict and find
790  * space.
791  */
ttm_bo_mem_space(struct ttm_buffer_object * bo,struct ttm_placement * placement,struct ttm_resource ** mem,struct ttm_operation_ctx * ctx)792 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
793 			struct ttm_placement *placement,
794 			struct ttm_resource **mem,
795 			struct ttm_operation_ctx *ctx)
796 {
797 	struct ttm_device *bdev = bo->bdev;
798 	bool type_found = false;
799 	int i, ret;
800 
801 	ret = dma_resv_reserve_fences(bo->base.resv, 1);
802 	if (unlikely(ret))
803 		return ret;
804 
805 	for (i = 0; i < placement->num_placement; ++i) {
806 		const struct ttm_place *place = &placement->placement[i];
807 		struct ttm_resource_manager *man;
808 
809 		man = ttm_manager_type(bdev, place->mem_type);
810 		if (!man || !ttm_resource_manager_used(man))
811 			continue;
812 
813 		type_found = true;
814 		ret = ttm_resource_alloc(bo, place, mem);
815 		if (ret == -ENOSPC)
816 			continue;
817 		if (unlikely(ret))
818 			goto error;
819 
820 		ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
821 		if (unlikely(ret)) {
822 			ttm_resource_free(bo, mem);
823 			if (ret == -EBUSY)
824 				continue;
825 
826 			goto error;
827 		}
828 		return 0;
829 	}
830 
831 	for (i = 0; i < placement->num_busy_placement; ++i) {
832 		const struct ttm_place *place = &placement->busy_placement[i];
833 		struct ttm_resource_manager *man;
834 
835 		man = ttm_manager_type(bdev, place->mem_type);
836 		if (!man || !ttm_resource_manager_used(man))
837 			continue;
838 
839 		type_found = true;
840 		ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
841 		if (likely(!ret))
842 			return 0;
843 
844 		if (ret && ret != -EBUSY)
845 			goto error;
846 	}
847 
848 	ret = -ENOMEM;
849 	if (!type_found) {
850 		pr_err(TTM_PFX "No compatible memory type found\n");
851 		ret = -EINVAL;
852 	}
853 
854 error:
855 	return ret;
856 }
857 EXPORT_SYMBOL(ttm_bo_mem_space);
858 
ttm_bo_move_buffer(struct ttm_buffer_object * bo,struct ttm_placement * placement,struct ttm_operation_ctx * ctx)859 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
860 			      struct ttm_placement *placement,
861 			      struct ttm_operation_ctx *ctx)
862 {
863 	struct ttm_resource *mem;
864 	struct ttm_place hop;
865 	int ret;
866 
867 	dma_resv_assert_held(bo->base.resv);
868 
869 	/*
870 	 * Determine where to move the buffer.
871 	 *
872 	 * If driver determines move is going to need
873 	 * an extra step then it will return -EMULTIHOP
874 	 * and the buffer will be moved to the temporary
875 	 * stop and the driver will be called to make
876 	 * the second hop.
877 	 */
878 	ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
879 	if (ret)
880 		return ret;
881 bounce:
882 	ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
883 	if (ret == -EMULTIHOP) {
884 		ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
885 		if (ret)
886 			goto out;
887 		/* try and move to final place now. */
888 		goto bounce;
889 	}
890 out:
891 	if (ret)
892 		ttm_resource_free(bo, &mem);
893 	return ret;
894 }
895 
ttm_bo_validate(struct ttm_buffer_object * bo,struct ttm_placement * placement,struct ttm_operation_ctx * ctx)896 int ttm_bo_validate(struct ttm_buffer_object *bo,
897 		    struct ttm_placement *placement,
898 		    struct ttm_operation_ctx *ctx)
899 {
900 	int ret;
901 
902 	dma_resv_assert_held(bo->base.resv);
903 
904 	/*
905 	 * Remove the backing store if no placement is given.
906 	 */
907 	if (!placement->num_placement && !placement->num_busy_placement)
908 		return ttm_bo_pipeline_gutting(bo);
909 
910 	/*
911 	 * Check whether we need to move buffer.
912 	 */
913 	if (!ttm_resource_compat(bo->resource, placement)) {
914 		ret = ttm_bo_move_buffer(bo, placement, ctx);
915 		if (ret)
916 			return ret;
917 	}
918 	/*
919 	 * We might need to add a TTM.
920 	 */
921 	if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) {
922 		ret = ttm_tt_create(bo, true);
923 		if (ret)
924 			return ret;
925 	}
926 	return 0;
927 }
928 EXPORT_SYMBOL(ttm_bo_validate);
929 
ttm_bo_init_reserved(struct ttm_device * bdev,struct ttm_buffer_object * bo,size_t size,enum ttm_bo_type type,struct ttm_placement * placement,uint32_t page_alignment,struct ttm_operation_ctx * ctx,struct sg_table * sg,struct dma_resv * resv,void (* destroy)(struct ttm_buffer_object *))930 int ttm_bo_init_reserved(struct ttm_device *bdev,
931 			 struct ttm_buffer_object *bo,
932 			 size_t size,
933 			 enum ttm_bo_type type,
934 			 struct ttm_placement *placement,
935 			 uint32_t page_alignment,
936 			 struct ttm_operation_ctx *ctx,
937 			 struct sg_table *sg,
938 			 struct dma_resv *resv,
939 			 void (*destroy) (struct ttm_buffer_object *))
940 {
941 	static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
942 	bool locked;
943 	int ret;
944 
945 	bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
946 
947 	kref_init(&bo->kref);
948 	INIT_LIST_HEAD(&bo->ddestroy);
949 	bo->bdev = bdev;
950 	bo->type = type;
951 	bo->page_alignment = page_alignment;
952 	bo->pin_count = 0;
953 	bo->sg = sg;
954 	bo->bulk_move = NULL;
955 	if (resv) {
956 		bo->base.resv = resv;
957 		dma_resv_assert_held(bo->base.resv);
958 	} else {
959 		bo->base.resv = &bo->base._resv;
960 	}
961 	atomic_inc(&ttm_glob.bo_count);
962 
963 	ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
964 	if (unlikely(ret)) {
965 		ttm_bo_put(bo);
966 		return ret;
967 	}
968 
969 	/*
970 	 * For ttm_bo_type_device buffers, allocate
971 	 * address space from the device.
972 	 */
973 	if (bo->type == ttm_bo_type_device ||
974 	    bo->type == ttm_bo_type_sg)
975 		ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
976 					 bo->resource->num_pages);
977 
978 	/* passed reservation objects should already be locked,
979 	 * since otherwise lockdep will be angered in radeon.
980 	 */
981 	if (!resv) {
982 		locked = dma_resv_trylock(bo->base.resv);
983 		WARN_ON(!locked);
984 	}
985 
986 	if (likely(!ret))
987 		ret = ttm_bo_validate(bo, placement, ctx);
988 
989 	if (unlikely(ret)) {
990 		if (!resv)
991 			ttm_bo_unreserve(bo);
992 
993 		ttm_bo_put(bo);
994 		return ret;
995 	}
996 
997 	return ret;
998 }
999 EXPORT_SYMBOL(ttm_bo_init_reserved);
1000 
ttm_bo_init(struct ttm_device * bdev,struct ttm_buffer_object * bo,size_t size,enum ttm_bo_type type,struct ttm_placement * placement,uint32_t page_alignment,bool interruptible,struct sg_table * sg,struct dma_resv * resv,void (* destroy)(struct ttm_buffer_object *))1001 int ttm_bo_init(struct ttm_device *bdev,
1002 		struct ttm_buffer_object *bo,
1003 		size_t size,
1004 		enum ttm_bo_type type,
1005 		struct ttm_placement *placement,
1006 		uint32_t page_alignment,
1007 		bool interruptible,
1008 		struct sg_table *sg,
1009 		struct dma_resv *resv,
1010 		void (*destroy) (struct ttm_buffer_object *))
1011 {
1012 	struct ttm_operation_ctx ctx = { interruptible, false };
1013 	int ret;
1014 
1015 	ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1016 				   page_alignment, &ctx, sg, resv, destroy);
1017 	if (ret)
1018 		return ret;
1019 
1020 	if (!resv)
1021 		ttm_bo_unreserve(bo);
1022 
1023 	return 0;
1024 }
1025 EXPORT_SYMBOL(ttm_bo_init);
1026 
1027 /*
1028  * buffer object vm functions.
1029  */
1030 
ttm_bo_unmap_virtual(struct ttm_buffer_object * bo)1031 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1032 {
1033 	struct ttm_device *bdev = bo->bdev;
1034 
1035 	drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1036 	ttm_mem_io_free(bdev, bo->resource);
1037 }
1038 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1039 
ttm_bo_wait(struct ttm_buffer_object * bo,bool interruptible,bool no_wait)1040 int ttm_bo_wait(struct ttm_buffer_object *bo,
1041 		bool interruptible, bool no_wait)
1042 {
1043 	long timeout = 15 * HZ;
1044 
1045 	if (no_wait) {
1046 		if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP))
1047 			return 0;
1048 		else
1049 			return -EBUSY;
1050 	}
1051 
1052 	timeout = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1053 					interruptible, timeout);
1054 	if (timeout < 0)
1055 		return timeout;
1056 
1057 	if (timeout == 0)
1058 		return -EBUSY;
1059 
1060 	return 0;
1061 }
1062 EXPORT_SYMBOL(ttm_bo_wait);
1063 
ttm_bo_swapout(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,gfp_t gfp_flags)1064 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1065 		   gfp_t gfp_flags)
1066 {
1067 	struct ttm_place place;
1068 	bool locked;
1069 	int ret;
1070 
1071 	/*
1072 	 * While the bo may already reside in SYSTEM placement, set
1073 	 * SYSTEM as new placement to cover also the move further below.
1074 	 * The driver may use the fact that we're moving from SYSTEM
1075 	 * as an indication that we're about to swap out.
1076 	 */
1077 	memset(&place, 0, sizeof(place));
1078 	place.mem_type = bo->resource->mem_type;
1079 	if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1080 		return -EBUSY;
1081 
1082 	if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1083 	    bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1084 	    bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1085 	    !ttm_bo_get_unless_zero(bo)) {
1086 		if (locked)
1087 			dma_resv_unlock(bo->base.resv);
1088 		return -EBUSY;
1089 	}
1090 
1091 	if (bo->deleted) {
1092 		ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1093 		ttm_bo_put(bo);
1094 		return ret == -EBUSY ? -ENOSPC : ret;
1095 	}
1096 
1097 	/* TODO: Cleanup the locking */
1098 	spin_unlock(&bo->bdev->lru_lock);
1099 
1100 	/*
1101 	 * Move to system cached
1102 	 */
1103 	if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1104 		struct ttm_operation_ctx ctx = { false, false };
1105 		struct ttm_resource *evict_mem;
1106 		struct ttm_place hop;
1107 
1108 		memset(&hop, 0, sizeof(hop));
1109 		place.mem_type = TTM_PL_SYSTEM;
1110 		ret = ttm_resource_alloc(bo, &place, &evict_mem);
1111 		if (unlikely(ret))
1112 			goto out;
1113 
1114 		ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop);
1115 		if (unlikely(ret != 0)) {
1116 			WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1117 			goto out;
1118 		}
1119 	}
1120 
1121 	/*
1122 	 * Make sure BO is idle.
1123 	 */
1124 	ret = ttm_bo_wait(bo, false, false);
1125 	if (unlikely(ret != 0))
1126 		goto out;
1127 
1128 	ttm_bo_unmap_virtual(bo);
1129 
1130 	/*
1131 	 * Swap out. Buffer will be swapped in again as soon as
1132 	 * anyone tries to access a ttm page.
1133 	 */
1134 	if (bo->bdev->funcs->swap_notify)
1135 		bo->bdev->funcs->swap_notify(bo);
1136 
1137 	if (ttm_tt_is_populated(bo->ttm))
1138 		ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1139 out:
1140 
1141 	/*
1142 	 * Unreserve without putting on LRU to avoid swapping out an
1143 	 * already swapped buffer.
1144 	 */
1145 	if (locked)
1146 		dma_resv_unlock(bo->base.resv);
1147 	ttm_bo_put(bo);
1148 	return ret == -EBUSY ? -ENOSPC : ret;
1149 }
1150 
ttm_bo_tt_destroy(struct ttm_buffer_object * bo)1151 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1152 {
1153 	if (bo->ttm == NULL)
1154 		return;
1155 
1156 	ttm_tt_unpopulate(bo->bdev, bo->ttm);
1157 	ttm_tt_destroy(bo->bdev, bo->ttm);
1158 	bo->ttm = NULL;
1159 }
1160