111 static void drm_gem_vram_cleanup(struct drm_gem_vram_object *gbo)  in drm_gem_vram_cleanup()  argument
118 	WARN_ON(gbo->vmap_use_count);  in drm_gem_vram_cleanup()
119 	WARN_ON(iosys_map_is_set(&gbo->map));  in drm_gem_vram_cleanup()
121 	drm_gem_object_release(&gbo->bo.base);  in drm_gem_vram_cleanup()
124 static void drm_gem_vram_destroy(struct drm_gem_vram_object *gbo)  in drm_gem_vram_destroy()  argument
126 	drm_gem_vram_cleanup(gbo);  in drm_gem_vram_destroy()
127 	kfree(gbo);  in drm_gem_vram_destroy()
132 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo);  in ttm_buffer_object_destroy()  local
134 	drm_gem_vram_destroy(gbo);  in ttm_buffer_object_destroy()
137 static void drm_gem_vram_placement(struct drm_gem_vram_object *gbo,  in drm_gem_vram_placement()  argument
147 	gbo->placement.placement = gbo->placements;  in drm_gem_vram_placement()
148 	gbo->placement.busy_placement = gbo->placements;  in drm_gem_vram_placement()
151 		gbo->placements[c].mem_type = TTM_PL_VRAM;  in drm_gem_vram_placement()
152 		gbo->placements[c++].flags = invariant_flags;  in drm_gem_vram_placement()
156 		gbo->placements[c].mem_type = TTM_PL_SYSTEM;  in drm_gem_vram_placement()
157 		gbo->placements[c++].flags = invariant_flags;  in drm_gem_vram_placement()
160 	gbo->placement.num_placement = c;  in drm_gem_vram_placement()
161 	gbo->placement.num_busy_placement = c;  in drm_gem_vram_placement()
164 		gbo->placements[i].fpfn = 0;  in drm_gem_vram_placement()
165 		gbo->placements[i].lpfn = 0;  in drm_gem_vram_placement()
189 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_create()  local
202 		gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_create()
204 		gbo = kzalloc(sizeof(*gbo), GFP_KERNEL);  in drm_gem_vram_create()
205 		if (!gbo)  in drm_gem_vram_create()
207 		gem = &gbo->bo.base;  in drm_gem_vram_create()
215 		kfree(gbo);  in drm_gem_vram_create()
221 	gbo->bo.bdev = bdev;  in drm_gem_vram_create()
222 	drm_gem_vram_placement(gbo, DRM_GEM_VRAM_PL_FLAG_SYSTEM);  in drm_gem_vram_create()
228 	ret = ttm_bo_init(bdev, &gbo->bo, size, ttm_bo_type_device,  in drm_gem_vram_create()
229 			  &gbo->placement, pg_align, false, NULL, NULL,  in drm_gem_vram_create()
234 	return gbo;  in drm_gem_vram_create()
244 void drm_gem_vram_put(struct drm_gem_vram_object *gbo)  in drm_gem_vram_put()  argument
246 	ttm_bo_put(&gbo->bo);  in drm_gem_vram_put()
250 static u64 drm_gem_vram_pg_offset(struct drm_gem_vram_object *gbo)  in drm_gem_vram_pg_offset()  argument
253 	if (WARN_ON_ONCE(!gbo->bo.resource ||  in drm_gem_vram_pg_offset()
254 			 gbo->bo.resource->mem_type == TTM_PL_SYSTEM))  in drm_gem_vram_pg_offset()
257 	return gbo->bo.resource->start;  in drm_gem_vram_pg_offset()
272 s64 drm_gem_vram_offset(struct drm_gem_vram_object *gbo)  in drm_gem_vram_offset()  argument
274 	if (WARN_ON_ONCE(!gbo->bo.pin_count))  in drm_gem_vram_offset()
276 	return drm_gem_vram_pg_offset(gbo) << PAGE_SHIFT;  in drm_gem_vram_offset()
280 static int drm_gem_vram_pin_locked(struct drm_gem_vram_object *gbo,  in drm_gem_vram_pin_locked()  argument
286 	if (gbo->bo.pin_count)  in drm_gem_vram_pin_locked()
290 		drm_gem_vram_placement(gbo, pl_flag);  in drm_gem_vram_pin_locked()
292 	ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx);  in drm_gem_vram_pin_locked()
297 	ttm_bo_pin(&gbo->bo);  in drm_gem_vram_pin_locked()
325 int drm_gem_vram_pin(struct drm_gem_vram_object *gbo, unsigned long pl_flag)  in drm_gem_vram_pin()  argument
329 	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);  in drm_gem_vram_pin()
332 	ret = drm_gem_vram_pin_locked(gbo, pl_flag);  in drm_gem_vram_pin()
333 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_pin()
339 static void drm_gem_vram_unpin_locked(struct drm_gem_vram_object *gbo)  in drm_gem_vram_unpin_locked()  argument
341 	ttm_bo_unpin(&gbo->bo);  in drm_gem_vram_unpin_locked()
352 int drm_gem_vram_unpin(struct drm_gem_vram_object *gbo)  in drm_gem_vram_unpin()  argument
356 	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);  in drm_gem_vram_unpin()
360 	drm_gem_vram_unpin_locked(gbo);  in drm_gem_vram_unpin()
361 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_unpin()
367 static int drm_gem_vram_kmap_locked(struct drm_gem_vram_object *gbo,  in drm_gem_vram_kmap_locked()  argument
372 	if (gbo->vmap_use_count > 0)  in drm_gem_vram_kmap_locked()
380 	if (iosys_map_is_null(&gbo->map)) {  in drm_gem_vram_kmap_locked()
381 		ret = ttm_bo_vmap(&gbo->bo, &gbo->map);  in drm_gem_vram_kmap_locked()
387 	++gbo->vmap_use_count;  in drm_gem_vram_kmap_locked()
388 	*map = gbo->map;  in drm_gem_vram_kmap_locked()
393 static void drm_gem_vram_kunmap_locked(struct drm_gem_vram_object *gbo,  in drm_gem_vram_kunmap_locked()  argument
396 	struct drm_device *dev = gbo->bo.base.dev;  in drm_gem_vram_kunmap_locked()
398 	if (drm_WARN_ON_ONCE(dev, !gbo->vmap_use_count))  in drm_gem_vram_kunmap_locked()
401 	if (drm_WARN_ON_ONCE(dev, !iosys_map_is_equal(&gbo->map, map)))  in drm_gem_vram_kunmap_locked()
404 	if (--gbo->vmap_use_count > 0)  in drm_gem_vram_kunmap_locked()
431 int drm_gem_vram_vmap(struct drm_gem_vram_object *gbo, struct iosys_map *map)  in drm_gem_vram_vmap()  argument
435 	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);  in drm_gem_vram_vmap()
439 	ret = drm_gem_vram_pin_locked(gbo, 0);  in drm_gem_vram_vmap()
442 	ret = drm_gem_vram_kmap_locked(gbo, map);  in drm_gem_vram_vmap()
446 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_vmap()
451 	drm_gem_vram_unpin_locked(gbo);  in drm_gem_vram_vmap()
453 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_vmap()
466 void drm_gem_vram_vunmap(struct drm_gem_vram_object *gbo,  in drm_gem_vram_vunmap()  argument
471 	ret = ttm_bo_reserve(&gbo->bo, false, false, NULL);  in drm_gem_vram_vunmap()
475 	drm_gem_vram_kunmap_locked(gbo, map);  in drm_gem_vram_vunmap()
476 	drm_gem_vram_unpin_locked(gbo);  in drm_gem_vram_vunmap()
478 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_vunmap()
508 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_fill_create_dumb()  local
524 	gbo = drm_gem_vram_create(dev, size, pg_align);  in drm_gem_vram_fill_create_dumb()
525 	if (IS_ERR(gbo))  in drm_gem_vram_fill_create_dumb()
526 		return PTR_ERR(gbo);  in drm_gem_vram_fill_create_dumb()
528 	ret = drm_gem_handle_create(file, &gbo->bo.base, &handle);  in drm_gem_vram_fill_create_dumb()
532 	drm_gem_object_put(&gbo->bo.base);  in drm_gem_vram_fill_create_dumb()
541 	drm_gem_object_put(&gbo->bo.base);  in drm_gem_vram_fill_create_dumb()
555 static void drm_gem_vram_bo_driver_evict_flags(struct drm_gem_vram_object *gbo,  in drm_gem_vram_bo_driver_evict_flags()  argument
558 	drm_gem_vram_placement(gbo, DRM_GEM_VRAM_PL_FLAG_SYSTEM);  in drm_gem_vram_bo_driver_evict_flags()
559 	*pl = gbo->placement;  in drm_gem_vram_bo_driver_evict_flags()
562 static void drm_gem_vram_bo_driver_move_notify(struct drm_gem_vram_object *gbo)  in drm_gem_vram_bo_driver_move_notify()  argument
564 	struct ttm_buffer_object *bo = &gbo->bo;  in drm_gem_vram_bo_driver_move_notify()
567 	if (drm_WARN_ON_ONCE(dev, gbo->vmap_use_count))  in drm_gem_vram_bo_driver_move_notify()
570 	ttm_bo_vunmap(bo, &gbo->map);  in drm_gem_vram_bo_driver_move_notify()
571 	iosys_map_clear(&gbo->map); /* explicitly clear mapping for next vmap call */  in drm_gem_vram_bo_driver_move_notify()
574 static int drm_gem_vram_bo_driver_move(struct drm_gem_vram_object *gbo,  in drm_gem_vram_bo_driver_move()  argument
579 	drm_gem_vram_bo_driver_move_notify(gbo);  in drm_gem_vram_bo_driver_move()
580 	return ttm_bo_move_memcpy(&gbo->bo, ctx, new_mem);  in drm_gem_vram_bo_driver_move()
594 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_free()  local
596 	drm_gem_vram_put(gbo);  in drm_gem_vram_object_free()
652 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_plane_helper_prepare_fb()  local
661 		gbo = drm_gem_vram_of_gem(new_state->fb->obj[i]);  in drm_gem_vram_plane_helper_prepare_fb()
662 		ret = drm_gem_vram_pin(gbo, DRM_GEM_VRAM_PL_FLAG_VRAM);  in drm_gem_vram_plane_helper_prepare_fb()
676 		gbo = drm_gem_vram_of_gem(new_state->fb->obj[i]);  in drm_gem_vram_plane_helper_prepare_fb()
677 		drm_gem_vram_unpin(gbo);  in drm_gem_vram_plane_helper_prepare_fb()
698 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_plane_helper_cleanup_fb()  local
706 		gbo = drm_gem_vram_of_gem(old_state->fb->obj[i]);  in drm_gem_vram_plane_helper_cleanup_fb()
707 		drm_gem_vram_unpin(gbo);  in drm_gem_vram_plane_helper_cleanup_fb()
771 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_pin()  local
781 	return drm_gem_vram_pin(gbo, 0);  in drm_gem_vram_object_pin()
791 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_unpin()  local
793 	drm_gem_vram_unpin(gbo);  in drm_gem_vram_object_unpin()
809 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_vmap()  local
811 	return drm_gem_vram_vmap(gbo, map);  in drm_gem_vram_object_vmap()
823 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_vunmap()  local
825 	drm_gem_vram_vunmap(gbo, map);  in drm_gem_vram_object_vunmap()
884 	struct drm_gem_vram_object *gbo;  in bo_driver_evict_flags()  local
890 	gbo = drm_gem_vram_of_bo(bo);  in bo_driver_evict_flags()
892 	drm_gem_vram_bo_driver_evict_flags(gbo, placement);  in bo_driver_evict_flags()
897 	struct drm_gem_vram_object *gbo;  in bo_driver_delete_mem_notify()  local
903 	gbo = drm_gem_vram_of_bo(bo);  in bo_driver_delete_mem_notify()
905 	drm_gem_vram_bo_driver_move_notify(gbo);  in bo_driver_delete_mem_notify()
914 	struct drm_gem_vram_object *gbo;  in bo_driver_move()  local
916 	gbo = drm_gem_vram_of_bo(bo);  in bo_driver_move()
918 	return drm_gem_vram_bo_driver_move(gbo, evict, ctx, new_mem);  in bo_driver_move()