56 static int radeon_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm,
58 static void radeon_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm);
205 		r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, new_mem);  in radeon_bo_move()
220 	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {  in radeon_bo_move()
232 		radeon_ttm_tt_unbind(bo->bdev, bo->ttm);  in radeon_bo_move()
325 	struct ttm_tt		ttm;  member
335 static int radeon_ttm_tt_pin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_tt_pin_userptr()  argument
338 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_tt_pin_userptr()
352 		unsigned long end = gtt->userptr + (u64)ttm->num_pages * PAGE_SIZE;  in radeon_ttm_tt_pin_userptr()
360 		unsigned num_pages = ttm->num_pages - pinned;  in radeon_ttm_tt_pin_userptr()
362 		struct page **pages = ttm->pages + pinned;  in radeon_ttm_tt_pin_userptr()
371 	} while (pinned < ttm->num_pages);  in radeon_ttm_tt_pin_userptr()
373 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,  in radeon_ttm_tt_pin_userptr()
374 				      (u64)ttm->num_pages << PAGE_SHIFT,  in radeon_ttm_tt_pin_userptr()
379 	r = dma_map_sgtable(rdev->dev, ttm->sg, direction, 0);  in radeon_ttm_tt_pin_userptr()
383 	drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,  in radeon_ttm_tt_pin_userptr()
384 				       ttm->num_pages);  in radeon_ttm_tt_pin_userptr()
389 	kfree(ttm->sg);  in radeon_ttm_tt_pin_userptr()
392 	release_pages(ttm->pages, pinned);  in radeon_ttm_tt_pin_userptr()
396 static void radeon_ttm_tt_unpin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_tt_unpin_userptr()  argument
399 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_tt_unpin_userptr()
407 	if (!ttm->sg || !ttm->sg->sgl)  in radeon_ttm_tt_unpin_userptr()
411 	dma_unmap_sgtable(rdev->dev, ttm->sg, direction, 0);  in radeon_ttm_tt_unpin_userptr()
413 	for_each_sgtable_page(ttm->sg, &sg_iter, 0) {  in radeon_ttm_tt_unpin_userptr()
422 	sg_free_table(ttm->sg);  in radeon_ttm_tt_unpin_userptr()
425 static bool radeon_ttm_backend_is_bound(struct ttm_tt *ttm)  in radeon_ttm_backend_is_bound()  argument
427 	struct radeon_ttm_tt *gtt = (void*)ttm;  in radeon_ttm_backend_is_bound()
433 				   struct ttm_tt *ttm,  in radeon_ttm_backend_bind()  argument
436 	struct radeon_ttm_tt *gtt = (void*)ttm;  in radeon_ttm_backend_bind()
446 		radeon_ttm_tt_pin_userptr(bdev, ttm);  in radeon_ttm_backend_bind()
451 	if (!ttm->num_pages) {  in radeon_ttm_backend_bind()
453 		     ttm->num_pages, bo_mem, ttm);  in radeon_ttm_backend_bind()
455 	if (ttm->caching == ttm_cached)  in radeon_ttm_backend_bind()
457 	r = radeon_gart_bind(rdev, gtt->offset, ttm->num_pages,  in radeon_ttm_backend_bind()
458 			     ttm->pages, gtt->ttm.dma_address, flags);  in radeon_ttm_backend_bind()
461 			  ttm->num_pages, (unsigned)gtt->offset);  in radeon_ttm_backend_bind()
468 static void radeon_ttm_backend_unbind(struct ttm_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_backend_unbind()  argument
470 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_backend_unbind()
474 		radeon_ttm_tt_unpin_userptr(bdev, ttm);  in radeon_ttm_backend_unbind()
479 	radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages);  in radeon_ttm_backend_unbind()
484 static void radeon_ttm_backend_destroy(struct ttm_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_backend_destroy()  argument
486 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_backend_destroy()
488 	ttm_tt_fini(&gtt->ttm);  in radeon_ttm_backend_destroy()
519 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags, caching)) {  in radeon_ttm_tt_create()
523 	return &gtt->ttm;  in radeon_ttm_tt_create()
527 						  struct ttm_tt *ttm)  in radeon_ttm_tt_to_gtt()  argument
534 	if (!ttm)  in radeon_ttm_tt_to_gtt()
536 	return container_of(ttm, struct radeon_ttm_tt, ttm);  in radeon_ttm_tt_to_gtt()
540 				  struct ttm_tt *ttm,  in radeon_ttm_tt_populate()  argument
544 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_populate()
545 	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);  in radeon_ttm_tt_populate()
548 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);  in radeon_ttm_tt_populate()
549 		if (!ttm->sg)  in radeon_ttm_tt_populate()
552 		ttm->page_flags |= TTM_TT_FLAG_EXTERNAL;  in radeon_ttm_tt_populate()
556 	if (slave && ttm->sg) {  in radeon_ttm_tt_populate()
557 		drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,  in radeon_ttm_tt_populate()
558 					       ttm->num_pages);  in radeon_ttm_tt_populate()
562 	return ttm_pool_alloc(&rdev->mman.bdev.pool, ttm, ctx);  in radeon_ttm_tt_populate()
565 static void radeon_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_tt_unpopulate()  argument
568 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_unpopulate()
569 	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);  in radeon_ttm_tt_unpopulate()
571 	radeon_ttm_tt_unbind(bdev, ttm);  in radeon_ttm_tt_unpopulate()
574 		kfree(ttm->sg);  in radeon_ttm_tt_unpopulate()
575 		ttm->page_flags &= ~TTM_TT_FLAG_EXTERNAL;  in radeon_ttm_tt_unpopulate()
582 	return ttm_pool_free(&rdev->mman.bdev.pool, ttm);  in radeon_ttm_tt_unpopulate()
586 			      struct ttm_tt *ttm, uint64_t addr,  in radeon_ttm_tt_set_userptr()  argument
589 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_set_userptr()
601 			    struct ttm_tt *ttm)  in radeon_ttm_tt_is_bound()  argument
606 		return ttm_agp_is_bound(ttm);  in radeon_ttm_tt_is_bound()
608 	return radeon_ttm_backend_is_bound(ttm);  in radeon_ttm_tt_is_bound()
612 			      struct ttm_tt *ttm,  in radeon_ttm_tt_bind()  argument
623 		return ttm_agp_bind(ttm, bo_mem);  in radeon_ttm_tt_bind()
626 	return radeon_ttm_backend_bind(bdev, ttm, bo_mem);  in radeon_ttm_tt_bind()
630 				 struct ttm_tt *ttm)  in radeon_ttm_tt_unbind()  argument
636 		ttm_agp_unbind(ttm);  in radeon_ttm_tt_unbind()
640 	radeon_ttm_backend_unbind(bdev, ttm);  in radeon_ttm_tt_unbind()
644 				  struct ttm_tt *ttm)  in radeon_ttm_tt_destroy()  argument
650 		ttm_agp_destroy(ttm);  in radeon_ttm_tt_destroy()
654 	radeon_ttm_backend_destroy(bdev, ttm);  in radeon_ttm_tt_destroy()
658 			       struct ttm_tt *ttm)  in radeon_ttm_tt_has_userptr()  argument
660 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_has_userptr()
669 			       struct ttm_tt *ttm)  in radeon_ttm_tt_is_readonly()  argument
671 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_is_readonly()