55 	struct drm_gem_shmem_object *shmem;  in __drm_gem_shmem_create()  local
65 		shmem = to_drm_gem_shmem_obj(obj);  in __drm_gem_shmem_create()
67 		shmem = kzalloc(sizeof(*shmem), GFP_KERNEL);  in __drm_gem_shmem_create()
68 		if (!shmem)  in __drm_gem_shmem_create()
70 		obj = &shmem->base;  in __drm_gem_shmem_create()
78 		shmem->map_wc = false; /* dma-buf mappings use always writecombine */  in __drm_gem_shmem_create()
91 	INIT_LIST_HEAD(&shmem->madv_list);  in __drm_gem_shmem_create()
105 	return shmem;  in __drm_gem_shmem_create()
138 void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_free()  argument
140 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_free()
143 		drm_prime_gem_destroy(obj, shmem->sgt);  in drm_gem_shmem_free()
145 		dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_free()
147 		drm_WARN_ON(obj->dev, shmem->vmap_use_count);  in drm_gem_shmem_free()
149 		if (shmem->sgt) {  in drm_gem_shmem_free()
150 			dma_unmap_sgtable(obj->dev->dev, shmem->sgt,  in drm_gem_shmem_free()
152 			sg_free_table(shmem->sgt);  in drm_gem_shmem_free()
153 			kfree(shmem->sgt);  in drm_gem_shmem_free()
155 		if (shmem->pages)  in drm_gem_shmem_free()
156 			drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_free()
158 		drm_WARN_ON(obj->dev, shmem->pages_use_count);  in drm_gem_shmem_free()
160 		dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_free()
164 	kfree(shmem);  in drm_gem_shmem_free()
168 static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages()  argument
170 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages()
173 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_get_pages()
175 	if (shmem->pages_use_count++ > 0)  in drm_gem_shmem_get_pages()
182 		shmem->pages_use_count = 0;  in drm_gem_shmem_get_pages()
192 	if (shmem->map_wc)  in drm_gem_shmem_get_pages()
196 	shmem->pages = pages;  in drm_gem_shmem_get_pages()
207 void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_put_pages()  argument
209 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_put_pages()
211 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_put_pages()
213 	if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count))  in drm_gem_shmem_put_pages()
216 	if (--shmem->pages_use_count > 0)  in drm_gem_shmem_put_pages()
220 	if (shmem->map_wc)  in drm_gem_shmem_put_pages()
221 		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_put_pages()
224 	drm_gem_put_pages(obj, shmem->pages,  in drm_gem_shmem_put_pages()
225 			  shmem->pages_mark_dirty_on_put,  in drm_gem_shmem_put_pages()
226 			  shmem->pages_mark_accessed_on_put);  in drm_gem_shmem_put_pages()
227 	shmem->pages = NULL;  in drm_gem_shmem_put_pages()
231 static int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_pin_locked()  argument
235 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_pin_locked()
237 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_pin_locked()
242 static void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_unpin_locked()  argument
244 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_unpin_locked()
246 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_unpin_locked()
259 int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_pin()  argument
261 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_pin()
266 	ret = dma_resv_lock_interruptible(shmem->base.resv, NULL);  in drm_gem_shmem_pin()
269 	ret = drm_gem_shmem_pin_locked(shmem);  in drm_gem_shmem_pin()
270 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_pin()
283 void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_unpin()  argument
285 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_unpin()
289 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_unpin()
290 	drm_gem_shmem_unpin_locked(shmem);  in drm_gem_shmem_unpin()
291 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_unpin()
310 int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_vmap()  argument
313 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vmap()
327 		dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_vmap()
329 		if (shmem->vmap_use_count++ > 0) {  in drm_gem_shmem_vmap()
330 			iosys_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap()
334 		ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_vmap()
338 		if (shmem->map_wc)  in drm_gem_shmem_vmap()
340 		shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT,  in drm_gem_shmem_vmap()
342 		if (!shmem->vaddr)  in drm_gem_shmem_vmap()
345 			iosys_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap()
357 		drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vmap()
359 	shmem->vmap_use_count = 0;  in drm_gem_shmem_vmap()
377 void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_vunmap()  argument
380 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vunmap()
385 		dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_vunmap()
387 		if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count))  in drm_gem_shmem_vunmap()
390 		if (--shmem->vmap_use_count > 0)  in drm_gem_shmem_vunmap()
393 		vunmap(shmem->vaddr);  in drm_gem_shmem_vunmap()
394 		drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vunmap()
397 	shmem->vaddr = NULL;  in drm_gem_shmem_vunmap()
406 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_create_with_handle()  local
409 	shmem = drm_gem_shmem_create(dev, size);  in drm_gem_shmem_create_with_handle()
410 	if (IS_ERR(shmem))  in drm_gem_shmem_create_with_handle()
411 		return PTR_ERR(shmem);  in drm_gem_shmem_create_with_handle()
417 	ret = drm_gem_handle_create(file_priv, &shmem->base, handle);  in drm_gem_shmem_create_with_handle()
419 	drm_gem_object_put(&shmem->base);  in drm_gem_shmem_create_with_handle()
427 int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv)  in drm_gem_shmem_madvise()  argument
429 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_madvise()
431 	if (shmem->madv >= 0)  in drm_gem_shmem_madvise()
432 		shmem->madv = madv;  in drm_gem_shmem_madvise()
434 	madv = shmem->madv;  in drm_gem_shmem_madvise()
440 void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_purge()  argument
442 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_purge()
445 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_purge()
447 	drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem));  in drm_gem_shmem_purge()
449 	dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0);  in drm_gem_shmem_purge()
450 	sg_free_table(shmem->sgt);  in drm_gem_shmem_purge()
451 	kfree(shmem->sgt);  in drm_gem_shmem_purge()
452 	shmem->sgt = NULL;  in drm_gem_shmem_purge()
454 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_purge()
456 	shmem->madv = -1;  in drm_gem_shmem_purge()
513 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_fault()  local
522 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_fault()
525 	    drm_WARN_ON_ONCE(obj->dev, !shmem->pages) ||  in drm_gem_shmem_fault()
526 	    shmem->madv < 0) {  in drm_gem_shmem_fault()
529 		page = shmem->pages[page_offset];  in drm_gem_shmem_fault()
534 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_fault()
542 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vm_open()  local
546 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_vm_open()
553 	if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count))  in drm_gem_shmem_vm_open()
554 		shmem->pages_use_count++;  in drm_gem_shmem_vm_open()
556 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_vm_open()
564 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vm_close()  local
566 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_vm_close()
567 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vm_close()
568 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_vm_close()
591 int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct *vma)  in drm_gem_shmem_mmap()  argument
593 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_mmap()
613 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_mmap()
614 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_mmap()
615 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_mmap()
622 	if (shmem->map_wc)  in drm_gem_shmem_mmap()
635 void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_print_info()  argument
638 	if (shmem->base.import_attach)  in drm_gem_shmem_print_info()
641 	drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);  in drm_gem_shmem_print_info()
642 	drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);  in drm_gem_shmem_print_info()
643 	drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);  in drm_gem_shmem_print_info()
661 struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_sg_table()  argument
663 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_sg_table()
667 	return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_get_sg_table()
671 static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages_sgt_locked()  argument
673 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages_sgt_locked()
677 	if (shmem->sgt)  in drm_gem_shmem_get_pages_sgt_locked()
678 		return shmem->sgt;  in drm_gem_shmem_get_pages_sgt_locked()
682 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_get_pages_sgt_locked()
686 	sgt = drm_gem_shmem_get_sg_table(shmem);  in drm_gem_shmem_get_pages_sgt_locked()
696 	shmem->sgt = sgt;  in drm_gem_shmem_get_pages_sgt_locked()
704 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_get_pages_sgt_locked()
724 struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages_sgt()  argument
729 	ret = dma_resv_lock_interruptible(shmem->base.resv, NULL);  in drm_gem_shmem_get_pages_sgt()
732 	sgt = drm_gem_shmem_get_pages_sgt_locked(shmem);  in drm_gem_shmem_get_pages_sgt()
733 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_get_pages_sgt()
760 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_prime_import_sg_table()  local
762 	shmem = __drm_gem_shmem_create(dev, size, true);  in drm_gem_shmem_prime_import_sg_table()
763 	if (IS_ERR(shmem))  in drm_gem_shmem_prime_import_sg_table()
764 		return ERR_CAST(shmem);  in drm_gem_shmem_prime_import_sg_table()
766 	shmem->sgt = sgt;  in drm_gem_shmem_prime_import_sg_table()
770 	return &shmem->base;  in drm_gem_shmem_prime_import_sg_table()