21 static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_map()  argument
23 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_iommu_map()
29 	ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm,  in rockchip_gem_iommu_map()
30 					 rk_obj->base.size, PAGE_SIZE,  in rockchip_gem_iommu_map()
39 	rk_obj->dma_addr = rk_obj->mm.start;  in rockchip_gem_iommu_map()
41 	ret = iommu_map_sgtable(private->domain, rk_obj->dma_addr, rk_obj->sgt,  in rockchip_gem_iommu_map()
43 	if (ret < (ssize_t)rk_obj->base.size) {  in rockchip_gem_iommu_map()
45 			  ret, rk_obj->base.size);  in rockchip_gem_iommu_map()
50 	rk_obj->size = ret;  in rockchip_gem_iommu_map()
56 	drm_mm_remove_node(&rk_obj->mm);  in rockchip_gem_iommu_map()
62 static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_unmap()  argument
64 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_iommu_unmap()
67 	iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size);  in rockchip_gem_iommu_unmap()
71 	drm_mm_remove_node(&rk_obj->mm);  in rockchip_gem_iommu_unmap()
78 static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj)  in rockchip_gem_get_pages()  argument
80 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_get_pages()
84 	rk_obj->pages = drm_gem_get_pages(&rk_obj->base);  in rockchip_gem_get_pages()
85 	if (IS_ERR(rk_obj->pages))  in rockchip_gem_get_pages()
86 		return PTR_ERR(rk_obj->pages);  in rockchip_gem_get_pages()
88 	rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT;  in rockchip_gem_get_pages()
90 	rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->base.dev,  in rockchip_gem_get_pages()
91 					    rk_obj->pages, rk_obj->num_pages);  in rockchip_gem_get_pages()
92 	if (IS_ERR(rk_obj->sgt)) {  in rockchip_gem_get_pages()
93 		ret = PTR_ERR(rk_obj->sgt);  in rockchip_gem_get_pages()
104 	for_each_sgtable_sg(rk_obj->sgt, s, i)  in rockchip_gem_get_pages()
107 	dma_sync_sgtable_for_device(drm->dev, rk_obj->sgt, DMA_TO_DEVICE);  in rockchip_gem_get_pages()
112 	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false);  in rockchip_gem_get_pages()
116 static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj)  in rockchip_gem_put_pages()  argument
118 	sg_free_table(rk_obj->sgt);  in rockchip_gem_put_pages()
119 	kfree(rk_obj->sgt);  in rockchip_gem_put_pages()
120 	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true);  in rockchip_gem_put_pages()
123 static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_iommu()  argument
128 	ret = rockchip_gem_get_pages(rk_obj);  in rockchip_gem_alloc_iommu()
132 	ret = rockchip_gem_iommu_map(rk_obj);  in rockchip_gem_alloc_iommu()
137 		rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,  in rockchip_gem_alloc_iommu()
139 		if (!rk_obj->kvaddr) {  in rockchip_gem_alloc_iommu()
149 	rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_alloc_iommu()
151 	rockchip_gem_put_pages(rk_obj);  in rockchip_gem_alloc_iommu()
156 static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_dma()  argument
159 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_alloc_dma()
162 	rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;  in rockchip_gem_alloc_dma()
165 		rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;  in rockchip_gem_alloc_dma()
167 	rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size,  in rockchip_gem_alloc_dma()
168 					 &rk_obj->dma_addr, GFP_KERNEL,  in rockchip_gem_alloc_dma()
169 					 rk_obj->dma_attrs);  in rockchip_gem_alloc_dma()
170 	if (!rk_obj->kvaddr) {  in rockchip_gem_alloc_dma()
178 static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_buf()  argument
181 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_alloc_buf()
186 		return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap);  in rockchip_gem_alloc_buf()
188 		return rockchip_gem_alloc_dma(rk_obj, alloc_kmap);  in rockchip_gem_alloc_buf()
191 static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_iommu()  argument
193 	vunmap(rk_obj->kvaddr);  in rockchip_gem_free_iommu()
194 	rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_free_iommu()
195 	rockchip_gem_put_pages(rk_obj);  in rockchip_gem_free_iommu()
198 static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_dma()  argument
200 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_free_dma()
203 	dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr,  in rockchip_gem_free_dma()
204 		       rk_obj->dma_attrs);  in rockchip_gem_free_dma()
207 static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_buf()  argument
209 	if (rk_obj->pages)  in rockchip_gem_free_buf()
210 		rockchip_gem_free_iommu(rk_obj);  in rockchip_gem_free_buf()
212 		rockchip_gem_free_dma(rk_obj);  in rockchip_gem_free_buf()
218 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap_iommu()  local
225 	return vm_map_pages(vma, rk_obj->pages, count);  in rockchip_drm_gem_object_mmap_iommu()
231 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap_dma()  local
234 	return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr,  in rockchip_drm_gem_object_mmap_dma()
235 			      obj->size, rk_obj->dma_attrs);  in rockchip_drm_gem_object_mmap_dma()
242 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap()  local
259 	if (rk_obj->pages)  in rockchip_drm_gem_object_mmap()
267 static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj)  in rockchip_gem_release_object()  argument
269 	drm_gem_object_release(&rk_obj->base);  in rockchip_gem_release_object()
270 	kfree(rk_obj);  in rockchip_gem_release_object()
285 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_alloc_object()  local
290 	rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL);  in rockchip_gem_alloc_object()
291 	if (!rk_obj)  in rockchip_gem_alloc_object()
294 	obj = &rk_obj->base;  in rockchip_gem_alloc_object()
300 	return rk_obj;  in rockchip_gem_alloc_object()
307 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_create_object()  local
310 	rk_obj = rockchip_gem_alloc_object(drm, size);  in rockchip_gem_create_object()
311 	if (IS_ERR(rk_obj))  in rockchip_gem_create_object()
312 		return rk_obj;  in rockchip_gem_create_object()
314 	ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap);  in rockchip_gem_create_object()
318 	return rk_obj;  in rockchip_gem_create_object()
321 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_create_object()
333 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_free_object()  local
337 			rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_free_object()
339 			dma_unmap_sgtable(drm->dev, rk_obj->sgt,  in rockchip_gem_free_object()
342 		drm_prime_gem_destroy(obj, rk_obj->sgt);  in rockchip_gem_free_object()
344 		rockchip_gem_free_buf(rk_obj);  in rockchip_gem_free_object()
347 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_free_object()
362 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_create_with_handle()  local
369 	rk_obj = rockchip_gem_create_object(drm, size, is_framebuffer);  in rockchip_gem_create_with_handle()
370 	if (IS_ERR(rk_obj))  in rockchip_gem_create_with_handle()
371 		return ERR_CAST(rk_obj);  in rockchip_gem_create_with_handle()
373 	obj = &rk_obj->base;  in rockchip_gem_create_with_handle()
386 	return rk_obj;  in rockchip_gem_create_with_handle()
405 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_dumb_create()  local
414 	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,  in rockchip_gem_dumb_create()
417 	return PTR_ERR_OR_ZERO(rk_obj);  in rockchip_gem_dumb_create()
428 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_get_sg_table()  local
433 	if (rk_obj->pages)  in rockchip_gem_prime_get_sg_table()
434 		return drm_prime_pages_to_sg(obj->dev, rk_obj->pages, rk_obj->num_pages);  in rockchip_gem_prime_get_sg_table()
440 	ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr,  in rockchip_gem_prime_get_sg_table()
441 				    rk_obj->dma_addr, obj->size,  in rockchip_gem_prime_get_sg_table()
442 				    rk_obj->dma_attrs);  in rockchip_gem_prime_get_sg_table()
456 			  struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_map_sg()  argument
458 	rk_obj->sgt = sg;  in rockchip_gem_iommu_map_sg()
459 	return rockchip_gem_iommu_map(rk_obj);  in rockchip_gem_iommu_map_sg()
466 			struct rockchip_gem_object *rk_obj)  in rockchip_gem_dma_map_sg()  argument
478 	rk_obj->dma_addr = sg_dma_address(sg->sgl);  in rockchip_gem_dma_map_sg()
479 	rk_obj->sgt = sg;  in rockchip_gem_dma_map_sg()
489 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_prime_import_sg_table()  local
492 	rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size);  in rockchip_gem_prime_import_sg_table()
493 	if (IS_ERR(rk_obj))  in rockchip_gem_prime_import_sg_table()
494 		return ERR_CAST(rk_obj);  in rockchip_gem_prime_import_sg_table()
497 		ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj);  in rockchip_gem_prime_import_sg_table()
499 		ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj);  in rockchip_gem_prime_import_sg_table()
506 	return &rk_obj->base;  in rockchip_gem_prime_import_sg_table()
509 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_prime_import_sg_table()
515 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_vmap()  local
517 	if (rk_obj->pages) {  in rockchip_gem_prime_vmap()
520 		if (rk_obj->kvaddr)  in rockchip_gem_prime_vmap()
521 			vaddr = rk_obj->kvaddr;  in rockchip_gem_prime_vmap()
523 			vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,  in rockchip_gem_prime_vmap()
532 	if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)  in rockchip_gem_prime_vmap()
534 	iosys_map_set_vaddr(map, rk_obj->kvaddr);  in rockchip_gem_prime_vmap()
542 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_vunmap()  local
544 	if (rk_obj->pages) {  in rockchip_gem_prime_vunmap()
545 		if (map->vaddr != rk_obj->kvaddr)  in rockchip_gem_prime_vunmap()