1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27 
28 #include <linux/dma-buf.h>
29 #include <linux/file.h>
30 #include <linux/fs.h>
31 #include <linux/iosys-map.h>
32 #include <linux/mem_encrypt.h>
33 #include <linux/mm.h>
34 #include <linux/mman.h>
35 #include <linux/module.h>
36 #include <linux/pagemap.h>
37 #include <linux/pagevec.h>
38 #include <linux/shmem_fs.h>
39 #include <linux/slab.h>
40 #include <linux/string_helpers.h>
41 #include <linux/types.h>
42 #include <linux/uaccess.h>
43 
44 #include <drm/drm.h>
45 #include <drm/drm_device.h>
46 #include <drm/drm_drv.h>
47 #include <drm/drm_file.h>
48 #include <drm/drm_gem.h>
49 #include <drm/drm_managed.h>
50 #include <drm/drm_print.h>
51 #include <drm/drm_vma_manager.h>
52 
53 #include "drm_internal.h"
54 
55 /** @file drm_gem.c
56  *
57  * This file provides some of the base ioctls and library routines for
58  * the graphics memory manager implemented by each device driver.
59  *
60  * Because various devices have different requirements in terms of
61  * synchronization and migration strategies, implementing that is left up to
62  * the driver, and all that the general API provides should be generic --
63  * allocating objects, reading/writing data with the cpu, freeing objects.
64  * Even there, platform-dependent optimizations for reading/writing data with
65  * the CPU mean we'll likely hook those out to driver-specific calls.  However,
66  * the DRI2 implementation wants to have at least allocate/mmap be generic.
67  *
68  * The goal was to have swap-backed object allocation managed through
69  * struct file.  However, file descriptors as handles to a struct file have
70  * two major failings:
71  * - Process limits prevent more than 1024 or so being used at a time by
72  *   default.
73  * - Inability to allocate high fds will aggravate the X Server's select()
74  *   handling, and likely that of many GL client applications as well.
75  *
76  * This led to a plan of using our own integer IDs (called handles, following
77  * DRM terminology) to mimic fds, and implement the fd syscalls we need as
78  * ioctls.  The objects themselves will still include the struct file so
79  * that we can transition to fds if the required kernel infrastructure shows
80  * up at a later date, and as our interface with shmfs for memory allocation.
81  */
82 
83 static void
drm_gem_init_release(struct drm_device * dev,void * ptr)84 drm_gem_init_release(struct drm_device *dev, void *ptr)
85 {
86 	drm_vma_offset_manager_destroy(dev->vma_offset_manager);
87 }
88 
89 /**
90  * drm_gem_init - Initialize the GEM device fields
91  * @dev: drm_devic structure to initialize
92  */
93 int
drm_gem_init(struct drm_device * dev)94 drm_gem_init(struct drm_device *dev)
95 {
96 	struct drm_vma_offset_manager *vma_offset_manager;
97 
98 	mutex_init(&dev->object_name_lock);
99 	idr_init_base(&dev->object_name_idr, 1);
100 
101 	vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager),
102 					  GFP_KERNEL);
103 	if (!vma_offset_manager) {
104 		DRM_ERROR("out of memory\n");
105 		return -ENOMEM;
106 	}
107 
108 	dev->vma_offset_manager = vma_offset_manager;
109 	drm_vma_offset_manager_init(vma_offset_manager,
110 				    DRM_FILE_PAGE_OFFSET_START,
111 				    DRM_FILE_PAGE_OFFSET_SIZE);
112 
113 	return drmm_add_action(dev, drm_gem_init_release, NULL);
114 }
115 
116 /**
117  * drm_gem_object_init - initialize an allocated shmem-backed GEM object
118  * @dev: drm_device the object should be initialized for
119  * @obj: drm_gem_object to initialize
120  * @size: object size
121  *
122  * Initialize an already allocated GEM object of the specified size with
123  * shmfs backing store.
124  */
drm_gem_object_init(struct drm_device * dev,struct drm_gem_object * obj,size_t size)125 int drm_gem_object_init(struct drm_device *dev,
126 			struct drm_gem_object *obj, size_t size)
127 {
128 	struct file *filp;
129 
130 	drm_gem_private_object_init(dev, obj, size);
131 
132 	filp = shmem_file_setup("drm mm object", size, VM_NORESERVE);
133 	if (IS_ERR(filp))
134 		return PTR_ERR(filp);
135 
136 	obj->filp = filp;
137 
138 	return 0;
139 }
140 EXPORT_SYMBOL(drm_gem_object_init);
141 
142 /**
143  * drm_gem_private_object_init - initialize an allocated private GEM object
144  * @dev: drm_device the object should be initialized for
145  * @obj: drm_gem_object to initialize
146  * @size: object size
147  *
148  * Initialize an already allocated GEM object of the specified size with
149  * no GEM provided backing store. Instead the caller is responsible for
150  * backing the object and handling it.
151  */
drm_gem_private_object_init(struct drm_device * dev,struct drm_gem_object * obj,size_t size)152 void drm_gem_private_object_init(struct drm_device *dev,
153 				 struct drm_gem_object *obj, size_t size)
154 {
155 	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
156 
157 	obj->dev = dev;
158 	obj->filp = NULL;
159 
160 	kref_init(&obj->refcount);
161 	obj->handle_count = 0;
162 	obj->size = size;
163 	dma_resv_init(&obj->_resv);
164 	if (!obj->resv)
165 		obj->resv = &obj->_resv;
166 
167 	drm_vma_node_reset(&obj->vma_node);
168 }
169 EXPORT_SYMBOL(drm_gem_private_object_init);
170 
171 /**
172  * drm_gem_object_handle_free - release resources bound to userspace handles
173  * @obj: GEM object to clean up.
174  *
175  * Called after the last handle to the object has been closed
176  *
177  * Removes any name for the object. Note that this must be
178  * called before drm_gem_object_free or we'll be touching
179  * freed memory
180  */
drm_gem_object_handle_free(struct drm_gem_object * obj)181 static void drm_gem_object_handle_free(struct drm_gem_object *obj)
182 {
183 	struct drm_device *dev = obj->dev;
184 
185 	/* Remove any name for this object */
186 	if (obj->name) {
187 		idr_remove(&dev->object_name_idr, obj->name);
188 		obj->name = 0;
189 	}
190 }
191 
drm_gem_object_exported_dma_buf_free(struct drm_gem_object * obj)192 static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
193 {
194 	/* Unbreak the reference cycle if we have an exported dma_buf. */
195 	if (obj->dma_buf) {
196 		dma_buf_put(obj->dma_buf);
197 		obj->dma_buf = NULL;
198 	}
199 }
200 
201 static void
drm_gem_object_handle_put_unlocked(struct drm_gem_object * obj)202 drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
203 {
204 	struct drm_device *dev = obj->dev;
205 	bool final = false;
206 
207 	if (WARN_ON(READ_ONCE(obj->handle_count) == 0))
208 		return;
209 
210 	/*
211 	* Must bump handle count first as this may be the last
212 	* ref, in which case the object would disappear before we
213 	* checked for a name
214 	*/
215 
216 	mutex_lock(&dev->object_name_lock);
217 	if (--obj->handle_count == 0) {
218 		drm_gem_object_handle_free(obj);
219 		drm_gem_object_exported_dma_buf_free(obj);
220 		final = true;
221 	}
222 	mutex_unlock(&dev->object_name_lock);
223 
224 	if (final)
225 		drm_gem_object_put(obj);
226 }
227 
228 /*
229  * Called at device or object close to release the file's
230  * handle references on objects.
231  */
232 static int
drm_gem_object_release_handle(int id,void * ptr,void * data)233 drm_gem_object_release_handle(int id, void *ptr, void *data)
234 {
235 	struct drm_file *file_priv = data;
236 	struct drm_gem_object *obj = ptr;
237 
238 	if (obj->funcs->close)
239 		obj->funcs->close(obj, file_priv);
240 
241 	drm_prime_remove_buf_handle(&file_priv->prime, id);
242 	drm_vma_node_revoke(&obj->vma_node, file_priv);
243 
244 	drm_gem_object_handle_put_unlocked(obj);
245 
246 	return 0;
247 }
248 
249 /**
250  * drm_gem_handle_delete - deletes the given file-private handle
251  * @filp: drm file-private structure to use for the handle look up
252  * @handle: userspace handle to delete
253  *
254  * Removes the GEM handle from the @filp lookup table which has been added with
255  * drm_gem_handle_create(). If this is the last handle also cleans up linked
256  * resources like GEM names.
257  */
258 int
drm_gem_handle_delete(struct drm_file * filp,u32 handle)259 drm_gem_handle_delete(struct drm_file *filp, u32 handle)
260 {
261 	struct drm_gem_object *obj;
262 
263 	spin_lock(&filp->table_lock);
264 
265 	/* Check if we currently have a reference on the object */
266 	obj = idr_replace(&filp->object_idr, NULL, handle);
267 	spin_unlock(&filp->table_lock);
268 	if (IS_ERR_OR_NULL(obj))
269 		return -EINVAL;
270 
271 	/* Release driver's reference and decrement refcount. */
272 	drm_gem_object_release_handle(handle, obj, filp);
273 
274 	/* And finally make the handle available for future allocations. */
275 	spin_lock(&filp->table_lock);
276 	idr_remove(&filp->object_idr, handle);
277 	spin_unlock(&filp->table_lock);
278 
279 	return 0;
280 }
281 EXPORT_SYMBOL(drm_gem_handle_delete);
282 
283 /**
284  * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object
285  * @file: drm file-private structure containing the gem object
286  * @dev: corresponding drm_device
287  * @handle: gem object handle
288  * @offset: return location for the fake mmap offset
289  *
290  * This implements the &drm_driver.dumb_map_offset kms driver callback for
291  * drivers which use gem to manage their backing storage.
292  *
293  * Returns:
294  * 0 on success or a negative error code on failure.
295  */
drm_gem_dumb_map_offset(struct drm_file * file,struct drm_device * dev,u32 handle,u64 * offset)296 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
297 			    u32 handle, u64 *offset)
298 {
299 	struct drm_gem_object *obj;
300 	int ret;
301 
302 	obj = drm_gem_object_lookup(file, handle);
303 	if (!obj)
304 		return -ENOENT;
305 
306 	/* Don't allow imported objects to be mapped */
307 	if (obj->import_attach) {
308 		ret = -EINVAL;
309 		goto out;
310 	}
311 
312 	ret = drm_gem_create_mmap_offset(obj);
313 	if (ret)
314 		goto out;
315 
316 	*offset = drm_vma_node_offset_addr(&obj->vma_node);
317 out:
318 	drm_gem_object_put(obj);
319 
320 	return ret;
321 }
322 EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
323 
drm_gem_dumb_destroy(struct drm_file * file,struct drm_device * dev,u32 handle)324 int drm_gem_dumb_destroy(struct drm_file *file,
325 			 struct drm_device *dev,
326 			 u32 handle)
327 {
328 	return drm_gem_handle_delete(file, handle);
329 }
330 
331 /**
332  * drm_gem_handle_create_tail - internal functions to create a handle
333  * @file_priv: drm file-private structure to register the handle for
334  * @obj: object to register
335  * @handlep: pointer to return the created handle to the caller
336  *
337  * This expects the &drm_device.object_name_lock to be held already and will
338  * drop it before returning. Used to avoid races in establishing new handles
339  * when importing an object from either an flink name or a dma-buf.
340  *
341  * Handles must be release again through drm_gem_handle_delete(). This is done
342  * when userspace closes @file_priv for all attached handles, or through the
343  * GEM_CLOSE ioctl for individual handles.
344  */
345 int
drm_gem_handle_create_tail(struct drm_file * file_priv,struct drm_gem_object * obj,u32 * handlep)346 drm_gem_handle_create_tail(struct drm_file *file_priv,
347 			   struct drm_gem_object *obj,
348 			   u32 *handlep)
349 {
350 	struct drm_device *dev = obj->dev;
351 	u32 handle;
352 	int ret;
353 
354 	WARN_ON(!mutex_is_locked(&dev->object_name_lock));
355 	if (obj->handle_count++ == 0)
356 		drm_gem_object_get(obj);
357 
358 	/*
359 	 * Get the user-visible handle using idr.  Preload and perform
360 	 * allocation under our spinlock.
361 	 */
362 	idr_preload(GFP_KERNEL);
363 	spin_lock(&file_priv->table_lock);
364 
365 	ret = idr_alloc(&file_priv->object_idr, obj, 1, 0, GFP_NOWAIT);
366 
367 	spin_unlock(&file_priv->table_lock);
368 	idr_preload_end();
369 
370 	mutex_unlock(&dev->object_name_lock);
371 	if (ret < 0)
372 		goto err_unref;
373 
374 	handle = ret;
375 
376 	ret = drm_vma_node_allow(&obj->vma_node, file_priv);
377 	if (ret)
378 		goto err_remove;
379 
380 	if (obj->funcs->open) {
381 		ret = obj->funcs->open(obj, file_priv);
382 		if (ret)
383 			goto err_revoke;
384 	}
385 
386 	*handlep = handle;
387 	return 0;
388 
389 err_revoke:
390 	drm_vma_node_revoke(&obj->vma_node, file_priv);
391 err_remove:
392 	spin_lock(&file_priv->table_lock);
393 	idr_remove(&file_priv->object_idr, handle);
394 	spin_unlock(&file_priv->table_lock);
395 err_unref:
396 	drm_gem_object_handle_put_unlocked(obj);
397 	return ret;
398 }
399 
400 /**
401  * drm_gem_handle_create - create a gem handle for an object
402  * @file_priv: drm file-private structure to register the handle for
403  * @obj: object to register
404  * @handlep: pointer to return the created handle to the caller
405  *
406  * Create a handle for this object. This adds a handle reference to the object,
407  * which includes a regular reference count. Callers will likely want to
408  * dereference the object afterwards.
409  *
410  * Since this publishes @obj to userspace it must be fully set up by this point,
411  * drivers must call this last in their buffer object creation callbacks.
412  */
drm_gem_handle_create(struct drm_file * file_priv,struct drm_gem_object * obj,u32 * handlep)413 int drm_gem_handle_create(struct drm_file *file_priv,
414 			  struct drm_gem_object *obj,
415 			  u32 *handlep)
416 {
417 	mutex_lock(&obj->dev->object_name_lock);
418 
419 	return drm_gem_handle_create_tail(file_priv, obj, handlep);
420 }
421 EXPORT_SYMBOL(drm_gem_handle_create);
422 
423 
424 /**
425  * drm_gem_free_mmap_offset - release a fake mmap offset for an object
426  * @obj: obj in question
427  *
428  * This routine frees fake offsets allocated by drm_gem_create_mmap_offset().
429  *
430  * Note that drm_gem_object_release() already calls this function, so drivers
431  * don't have to take care of releasing the mmap offset themselves when freeing
432  * the GEM object.
433  */
434 void
drm_gem_free_mmap_offset(struct drm_gem_object * obj)435 drm_gem_free_mmap_offset(struct drm_gem_object *obj)
436 {
437 	struct drm_device *dev = obj->dev;
438 
439 	drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node);
440 }
441 EXPORT_SYMBOL(drm_gem_free_mmap_offset);
442 
443 /**
444  * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object
445  * @obj: obj in question
446  * @size: the virtual size
447  *
448  * GEM memory mapping works by handing back to userspace a fake mmap offset
449  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
450  * up the object based on the offset and sets up the various memory mapping
451  * structures.
452  *
453  * This routine allocates and attaches a fake offset for @obj, in cases where
454  * the virtual size differs from the physical size (ie. &drm_gem_object.size).
455  * Otherwise just use drm_gem_create_mmap_offset().
456  *
457  * This function is idempotent and handles an already allocated mmap offset
458  * transparently. Drivers do not need to check for this case.
459  */
460 int
drm_gem_create_mmap_offset_size(struct drm_gem_object * obj,size_t size)461 drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)
462 {
463 	struct drm_device *dev = obj->dev;
464 
465 	return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node,
466 				  size / PAGE_SIZE);
467 }
468 EXPORT_SYMBOL(drm_gem_create_mmap_offset_size);
469 
470 /**
471  * drm_gem_create_mmap_offset - create a fake mmap offset for an object
472  * @obj: obj in question
473  *
474  * GEM memory mapping works by handing back to userspace a fake mmap offset
475  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
476  * up the object based on the offset and sets up the various memory mapping
477  * structures.
478  *
479  * This routine allocates and attaches a fake offset for @obj.
480  *
481  * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release
482  * the fake offset again.
483  */
drm_gem_create_mmap_offset(struct drm_gem_object * obj)484 int drm_gem_create_mmap_offset(struct drm_gem_object *obj)
485 {
486 	return drm_gem_create_mmap_offset_size(obj, obj->size);
487 }
488 EXPORT_SYMBOL(drm_gem_create_mmap_offset);
489 
490 /*
491  * Move pages to appropriate lru and release the pagevec, decrementing the
492  * ref count of those pages.
493  */
drm_gem_check_release_pagevec(struct pagevec * pvec)494 static void drm_gem_check_release_pagevec(struct pagevec *pvec)
495 {
496 	check_move_unevictable_pages(pvec);
497 	__pagevec_release(pvec);
498 	cond_resched();
499 }
500 
501 /**
502  * drm_gem_get_pages - helper to allocate backing pages for a GEM object
503  * from shmem
504  * @obj: obj in question
505  *
506  * This reads the page-array of the shmem-backing storage of the given gem
507  * object. An array of pages is returned. If a page is not allocated or
508  * swapped-out, this will allocate/swap-in the required pages. Note that the
509  * whole object is covered by the page-array and pinned in memory.
510  *
511  * Use drm_gem_put_pages() to release the array and unpin all pages.
512  *
513  * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()).
514  * If you require other GFP-masks, you have to do those allocations yourself.
515  *
516  * Note that you are not allowed to change gfp-zones during runtime. That is,
517  * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as
518  * set during initialization. If you have special zone constraints, set them
519  * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care
520  * to keep pages in the required zone during swap-in.
521  *
522  * This function is only valid on objects initialized with
523  * drm_gem_object_init(), but not for those initialized with
524  * drm_gem_private_object_init() only.
525  */
drm_gem_get_pages(struct drm_gem_object * obj)526 struct page **drm_gem_get_pages(struct drm_gem_object *obj)
527 {
528 	struct address_space *mapping;
529 	struct page *p, **pages;
530 	struct pagevec pvec;
531 	int i, npages;
532 
533 
534 	if (WARN_ON(!obj->filp))
535 		return ERR_PTR(-EINVAL);
536 
537 	/* This is the shared memory object that backs the GEM resource */
538 	mapping = obj->filp->f_mapping;
539 
540 	/* We already BUG_ON() for non-page-aligned sizes in
541 	 * drm_gem_object_init(), so we should never hit this unless
542 	 * driver author is doing something really wrong:
543 	 */
544 	WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
545 
546 	npages = obj->size >> PAGE_SHIFT;
547 
548 	pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
549 	if (pages == NULL)
550 		return ERR_PTR(-ENOMEM);
551 
552 	mapping_set_unevictable(mapping);
553 
554 	for (i = 0; i < npages; i++) {
555 		p = shmem_read_mapping_page(mapping, i);
556 		if (IS_ERR(p))
557 			goto fail;
558 		pages[i] = p;
559 
560 		/* Make sure shmem keeps __GFP_DMA32 allocated pages in the
561 		 * correct region during swapin. Note that this requires
562 		 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping)
563 		 * so shmem can relocate pages during swapin if required.
564 		 */
565 		BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) &&
566 				(page_to_pfn(p) >= 0x00100000UL));
567 	}
568 
569 	return pages;
570 
571 fail:
572 	mapping_clear_unevictable(mapping);
573 	pagevec_init(&pvec);
574 	while (i--) {
575 		if (!pagevec_add(&pvec, pages[i]))
576 			drm_gem_check_release_pagevec(&pvec);
577 	}
578 	if (pagevec_count(&pvec))
579 		drm_gem_check_release_pagevec(&pvec);
580 
581 	kvfree(pages);
582 	return ERR_CAST(p);
583 }
584 EXPORT_SYMBOL(drm_gem_get_pages);
585 
586 /**
587  * drm_gem_put_pages - helper to free backing pages for a GEM object
588  * @obj: obj in question
589  * @pages: pages to free
590  * @dirty: if true, pages will be marked as dirty
591  * @accessed: if true, the pages will be marked as accessed
592  */
drm_gem_put_pages(struct drm_gem_object * obj,struct page ** pages,bool dirty,bool accessed)593 void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
594 		bool dirty, bool accessed)
595 {
596 	int i, npages;
597 	struct address_space *mapping;
598 	struct pagevec pvec;
599 
600 	mapping = file_inode(obj->filp)->i_mapping;
601 	mapping_clear_unevictable(mapping);
602 
603 	/* We already BUG_ON() for non-page-aligned sizes in
604 	 * drm_gem_object_init(), so we should never hit this unless
605 	 * driver author is doing something really wrong:
606 	 */
607 	WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
608 
609 	npages = obj->size >> PAGE_SHIFT;
610 
611 	pagevec_init(&pvec);
612 	for (i = 0; i < npages; i++) {
613 		if (!pages[i])
614 			continue;
615 
616 		if (dirty)
617 			set_page_dirty(pages[i]);
618 
619 		if (accessed)
620 			mark_page_accessed(pages[i]);
621 
622 		/* Undo the reference we took when populating the table */
623 		if (!pagevec_add(&pvec, pages[i]))
624 			drm_gem_check_release_pagevec(&pvec);
625 	}
626 	if (pagevec_count(&pvec))
627 		drm_gem_check_release_pagevec(&pvec);
628 
629 	kvfree(pages);
630 }
631 EXPORT_SYMBOL(drm_gem_put_pages);
632 
objects_lookup(struct drm_file * filp,u32 * handle,int count,struct drm_gem_object ** objs)633 static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
634 			  struct drm_gem_object **objs)
635 {
636 	int i, ret = 0;
637 	struct drm_gem_object *obj;
638 
639 	spin_lock(&filp->table_lock);
640 
641 	for (i = 0; i < count; i++) {
642 		/* Check if we currently have a reference on the object */
643 		obj = idr_find(&filp->object_idr, handle[i]);
644 		if (!obj) {
645 			ret = -ENOENT;
646 			break;
647 		}
648 		drm_gem_object_get(obj);
649 		objs[i] = obj;
650 	}
651 	spin_unlock(&filp->table_lock);
652 
653 	return ret;
654 }
655 
656 /**
657  * drm_gem_objects_lookup - look up GEM objects from an array of handles
658  * @filp: DRM file private date
659  * @bo_handles: user pointer to array of userspace handle
660  * @count: size of handle array
661  * @objs_out: returned pointer to array of drm_gem_object pointers
662  *
663  * Takes an array of userspace handles and returns a newly allocated array of
664  * GEM objects.
665  *
666  * For a single handle lookup, use drm_gem_object_lookup().
667  *
668  * Returns:
669  *
670  * @objs filled in with GEM object pointers. Returned GEM objects need to be
671  * released with drm_gem_object_put(). -ENOENT is returned on a lookup
672  * failure. 0 is returned on success.
673  *
674  */
drm_gem_objects_lookup(struct drm_file * filp,void __user * bo_handles,int count,struct drm_gem_object *** objs_out)675 int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
676 			   int count, struct drm_gem_object ***objs_out)
677 {
678 	int ret;
679 	u32 *handles;
680 	struct drm_gem_object **objs;
681 
682 	if (!count)
683 		return 0;
684 
685 	objs = kvmalloc_array(count, sizeof(struct drm_gem_object *),
686 			     GFP_KERNEL | __GFP_ZERO);
687 	if (!objs)
688 		return -ENOMEM;
689 
690 	*objs_out = objs;
691 
692 	handles = kvmalloc_array(count, sizeof(u32), GFP_KERNEL);
693 	if (!handles) {
694 		ret = -ENOMEM;
695 		goto out;
696 	}
697 
698 	if (copy_from_user(handles, bo_handles, count * sizeof(u32))) {
699 		ret = -EFAULT;
700 		DRM_DEBUG("Failed to copy in GEM handles\n");
701 		goto out;
702 	}
703 
704 	ret = objects_lookup(filp, handles, count, objs);
705 out:
706 	kvfree(handles);
707 	return ret;
708 
709 }
710 EXPORT_SYMBOL(drm_gem_objects_lookup);
711 
712 /**
713  * drm_gem_object_lookup - look up a GEM object from its handle
714  * @filp: DRM file private date
715  * @handle: userspace handle
716  *
717  * Returns:
718  *
719  * A reference to the object named by the handle if such exists on @filp, NULL
720  * otherwise.
721  *
722  * If looking up an array of handles, use drm_gem_objects_lookup().
723  */
724 struct drm_gem_object *
drm_gem_object_lookup(struct drm_file * filp,u32 handle)725 drm_gem_object_lookup(struct drm_file *filp, u32 handle)
726 {
727 	struct drm_gem_object *obj = NULL;
728 
729 	objects_lookup(filp, &handle, 1, &obj);
730 	return obj;
731 }
732 EXPORT_SYMBOL(drm_gem_object_lookup);
733 
734 /**
735  * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects
736  * shared and/or exclusive fences.
737  * @filep: DRM file private date
738  * @handle: userspace handle
739  * @wait_all: if true, wait on all fences, else wait on just exclusive fence
740  * @timeout: timeout value in jiffies or zero to return immediately
741  *
742  * Returns:
743  *
744  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
745  * greater than 0 on success.
746  */
drm_gem_dma_resv_wait(struct drm_file * filep,u32 handle,bool wait_all,unsigned long timeout)747 long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle,
748 				    bool wait_all, unsigned long timeout)
749 {
750 	long ret;
751 	struct drm_gem_object *obj;
752 
753 	obj = drm_gem_object_lookup(filep, handle);
754 	if (!obj) {
755 		DRM_DEBUG("Failed to look up GEM BO %d\n", handle);
756 		return -EINVAL;
757 	}
758 
759 	ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all),
760 				    true, timeout);
761 	if (ret == 0)
762 		ret = -ETIME;
763 	else if (ret > 0)
764 		ret = 0;
765 
766 	drm_gem_object_put(obj);
767 
768 	return ret;
769 }
770 EXPORT_SYMBOL(drm_gem_dma_resv_wait);
771 
772 /**
773  * drm_gem_close_ioctl - implementation of the GEM_CLOSE ioctl
774  * @dev: drm_device
775  * @data: ioctl data
776  * @file_priv: drm file-private structure
777  *
778  * Releases the handle to an mm object.
779  */
780 int
drm_gem_close_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)781 drm_gem_close_ioctl(struct drm_device *dev, void *data,
782 		    struct drm_file *file_priv)
783 {
784 	struct drm_gem_close *args = data;
785 	int ret;
786 
787 	if (!drm_core_check_feature(dev, DRIVER_GEM))
788 		return -EOPNOTSUPP;
789 
790 	ret = drm_gem_handle_delete(file_priv, args->handle);
791 
792 	return ret;
793 }
794 
795 /**
796  * drm_gem_flink_ioctl - implementation of the GEM_FLINK ioctl
797  * @dev: drm_device
798  * @data: ioctl data
799  * @file_priv: drm file-private structure
800  *
801  * Create a global name for an object, returning the name.
802  *
803  * Note that the name does not hold a reference; when the object
804  * is freed, the name goes away.
805  */
806 int
drm_gem_flink_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)807 drm_gem_flink_ioctl(struct drm_device *dev, void *data,
808 		    struct drm_file *file_priv)
809 {
810 	struct drm_gem_flink *args = data;
811 	struct drm_gem_object *obj;
812 	int ret;
813 
814 	if (!drm_core_check_feature(dev, DRIVER_GEM))
815 		return -EOPNOTSUPP;
816 
817 	obj = drm_gem_object_lookup(file_priv, args->handle);
818 	if (obj == NULL)
819 		return -ENOENT;
820 
821 	mutex_lock(&dev->object_name_lock);
822 	/* prevent races with concurrent gem_close. */
823 	if (obj->handle_count == 0) {
824 		ret = -ENOENT;
825 		goto err;
826 	}
827 
828 	if (!obj->name) {
829 		ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
830 		if (ret < 0)
831 			goto err;
832 
833 		obj->name = ret;
834 	}
835 
836 	args->name = (uint64_t) obj->name;
837 	ret = 0;
838 
839 err:
840 	mutex_unlock(&dev->object_name_lock);
841 	drm_gem_object_put(obj);
842 	return ret;
843 }
844 
845 /**
846  * drm_gem_open_ioctl - implementation of the GEM_OPEN ioctl
847  * @dev: drm_device
848  * @data: ioctl data
849  * @file_priv: drm file-private structure
850  *
851  * Open an object using the global name, returning a handle and the size.
852  *
853  * This handle (of course) holds a reference to the object, so the object
854  * will not go away until the handle is deleted.
855  */
856 int
drm_gem_open_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)857 drm_gem_open_ioctl(struct drm_device *dev, void *data,
858 		   struct drm_file *file_priv)
859 {
860 	struct drm_gem_open *args = data;
861 	struct drm_gem_object *obj;
862 	int ret;
863 	u32 handle;
864 
865 	if (!drm_core_check_feature(dev, DRIVER_GEM))
866 		return -EOPNOTSUPP;
867 
868 	mutex_lock(&dev->object_name_lock);
869 	obj = idr_find(&dev->object_name_idr, (int) args->name);
870 	if (obj) {
871 		drm_gem_object_get(obj);
872 	} else {
873 		mutex_unlock(&dev->object_name_lock);
874 		return -ENOENT;
875 	}
876 
877 	/* drm_gem_handle_create_tail unlocks dev->object_name_lock. */
878 	ret = drm_gem_handle_create_tail(file_priv, obj, &handle);
879 	if (ret)
880 		goto err;
881 
882 	args->handle = handle;
883 	args->size = obj->size;
884 
885 err:
886 	drm_gem_object_put(obj);
887 	return ret;
888 }
889 
890 /**
891  * drm_gem_open - initializes GEM file-private structures at devnode open time
892  * @dev: drm_device which is being opened by userspace
893  * @file_private: drm file-private structure to set up
894  *
895  * Called at device open time, sets up the structure for handling refcounting
896  * of mm objects.
897  */
898 void
drm_gem_open(struct drm_device * dev,struct drm_file * file_private)899 drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
900 {
901 	idr_init_base(&file_private->object_idr, 1);
902 	spin_lock_init(&file_private->table_lock);
903 }
904 
905 /**
906  * drm_gem_release - release file-private GEM resources
907  * @dev: drm_device which is being closed by userspace
908  * @file_private: drm file-private structure to clean up
909  *
910  * Called at close time when the filp is going away.
911  *
912  * Releases any remaining references on objects by this filp.
913  */
914 void
drm_gem_release(struct drm_device * dev,struct drm_file * file_private)915 drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
916 {
917 	idr_for_each(&file_private->object_idr,
918 		     &drm_gem_object_release_handle, file_private);
919 	idr_destroy(&file_private->object_idr);
920 }
921 
922 /**
923  * drm_gem_object_release - release GEM buffer object resources
924  * @obj: GEM buffer object
925  *
926  * This releases any structures and resources used by @obj and is the inverse of
927  * drm_gem_object_init().
928  */
929 void
drm_gem_object_release(struct drm_gem_object * obj)930 drm_gem_object_release(struct drm_gem_object *obj)
931 {
932 	WARN_ON(obj->dma_buf);
933 
934 	if (obj->filp)
935 		fput(obj->filp);
936 
937 	dma_resv_fini(&obj->_resv);
938 	drm_gem_free_mmap_offset(obj);
939 }
940 EXPORT_SYMBOL(drm_gem_object_release);
941 
942 /**
943  * drm_gem_object_free - free a GEM object
944  * @kref: kref of the object to free
945  *
946  * Called after the last reference to the object has been lost.
947  *
948  * Frees the object
949  */
950 void
drm_gem_object_free(struct kref * kref)951 drm_gem_object_free(struct kref *kref)
952 {
953 	struct drm_gem_object *obj =
954 		container_of(kref, struct drm_gem_object, refcount);
955 
956 	if (WARN_ON(!obj->funcs->free))
957 		return;
958 
959 	obj->funcs->free(obj);
960 }
961 EXPORT_SYMBOL(drm_gem_object_free);
962 
963 /**
964  * drm_gem_vm_open - vma->ops->open implementation for GEM
965  * @vma: VM area structure
966  *
967  * This function implements the #vm_operations_struct open() callback for GEM
968  * drivers. This must be used together with drm_gem_vm_close().
969  */
drm_gem_vm_open(struct vm_area_struct * vma)970 void drm_gem_vm_open(struct vm_area_struct *vma)
971 {
972 	struct drm_gem_object *obj = vma->vm_private_data;
973 
974 	drm_gem_object_get(obj);
975 }
976 EXPORT_SYMBOL(drm_gem_vm_open);
977 
978 /**
979  * drm_gem_vm_close - vma->ops->close implementation for GEM
980  * @vma: VM area structure
981  *
982  * This function implements the #vm_operations_struct close() callback for GEM
983  * drivers. This must be used together with drm_gem_vm_open().
984  */
drm_gem_vm_close(struct vm_area_struct * vma)985 void drm_gem_vm_close(struct vm_area_struct *vma)
986 {
987 	struct drm_gem_object *obj = vma->vm_private_data;
988 
989 	drm_gem_object_put(obj);
990 }
991 EXPORT_SYMBOL(drm_gem_vm_close);
992 
993 /**
994  * drm_gem_mmap_obj - memory map a GEM object
995  * @obj: the GEM object to map
996  * @obj_size: the object size to be mapped, in bytes
997  * @vma: VMA for the area to be mapped
998  *
999  * Set up the VMA to prepare mapping of the GEM object using the GEM object's
1000  * vm_ops. Depending on their requirements, GEM objects can either
1001  * provide a fault handler in their vm_ops (in which case any accesses to
1002  * the object will be trapped, to perform migration, GTT binding, surface
1003  * register allocation, or performance monitoring), or mmap the buffer memory
1004  * synchronously after calling drm_gem_mmap_obj.
1005  *
1006  * This function is mainly intended to implement the DMABUF mmap operation, when
1007  * the GEM object is not looked up based on its fake offset. To implement the
1008  * DRM mmap operation, drivers should use the drm_gem_mmap() function.
1009  *
1010  * drm_gem_mmap_obj() assumes the user is granted access to the buffer while
1011  * drm_gem_mmap() prevents unprivileged users from mapping random objects. So
1012  * callers must verify access restrictions before calling this helper.
1013  *
1014  * Return 0 or success or -EINVAL if the object size is smaller than the VMA
1015  * size, or if no vm_ops are provided.
1016  */
drm_gem_mmap_obj(struct drm_gem_object * obj,unsigned long obj_size,struct vm_area_struct * vma)1017 int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
1018 		     struct vm_area_struct *vma)
1019 {
1020 	int ret;
1021 
1022 	/* Check for valid size. */
1023 	if (obj_size < vma->vm_end - vma->vm_start)
1024 		return -EINVAL;
1025 
1026 	/* Take a ref for this mapping of the object, so that the fault
1027 	 * handler can dereference the mmap offset's pointer to the object.
1028 	 * This reference is cleaned up by the corresponding vm_close
1029 	 * (which should happen whether the vma was created by this call, or
1030 	 * by a vm_open due to mremap or partial unmap or whatever).
1031 	 */
1032 	drm_gem_object_get(obj);
1033 
1034 	vma->vm_private_data = obj;
1035 	vma->vm_ops = obj->funcs->vm_ops;
1036 
1037 	if (obj->funcs->mmap) {
1038 		ret = obj->funcs->mmap(obj, vma);
1039 		if (ret)
1040 			goto err_drm_gem_object_put;
1041 		WARN_ON(!(vma->vm_flags & VM_DONTEXPAND));
1042 	} else {
1043 		if (!vma->vm_ops) {
1044 			ret = -EINVAL;
1045 			goto err_drm_gem_object_put;
1046 		}
1047 
1048 		vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1049 		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1050 		vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1051 	}
1052 
1053 	return 0;
1054 
1055 err_drm_gem_object_put:
1056 	drm_gem_object_put(obj);
1057 	return ret;
1058 }
1059 EXPORT_SYMBOL(drm_gem_mmap_obj);
1060 
1061 /**
1062  * drm_gem_mmap - memory map routine for GEM objects
1063  * @filp: DRM file pointer
1064  * @vma: VMA for the area to be mapped
1065  *
1066  * If a driver supports GEM object mapping, mmap calls on the DRM file
1067  * descriptor will end up here.
1068  *
1069  * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
1070  * contain the fake offset we created when the GTT map ioctl was called on
1071  * the object) and map it with a call to drm_gem_mmap_obj().
1072  *
1073  * If the caller is not granted access to the buffer object, the mmap will fail
1074  * with EACCES. Please see the vma manager for more information.
1075  */
drm_gem_mmap(struct file * filp,struct vm_area_struct * vma)1076 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1077 {
1078 	struct drm_file *priv = filp->private_data;
1079 	struct drm_device *dev = priv->minor->dev;
1080 	struct drm_gem_object *obj = NULL;
1081 	struct drm_vma_offset_node *node;
1082 	int ret;
1083 
1084 	if (drm_dev_is_unplugged(dev))
1085 		return -ENODEV;
1086 
1087 	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1088 	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
1089 						  vma->vm_pgoff,
1090 						  vma_pages(vma));
1091 	if (likely(node)) {
1092 		obj = container_of(node, struct drm_gem_object, vma_node);
1093 		/*
1094 		 * When the object is being freed, after it hits 0-refcnt it
1095 		 * proceeds to tear down the object. In the process it will
1096 		 * attempt to remove the VMA offset and so acquire this
1097 		 * mgr->vm_lock.  Therefore if we find an object with a 0-refcnt
1098 		 * that matches our range, we know it is in the process of being
1099 		 * destroyed and will be freed as soon as we release the lock -
1100 		 * so we have to check for the 0-refcnted object and treat it as
1101 		 * invalid.
1102 		 */
1103 		if (!kref_get_unless_zero(&obj->refcount))
1104 			obj = NULL;
1105 	}
1106 	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1107 
1108 	if (!obj)
1109 		return -EINVAL;
1110 
1111 	if (!drm_vma_node_is_allowed(node, priv)) {
1112 		drm_gem_object_put(obj);
1113 		return -EACCES;
1114 	}
1115 
1116 	ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
1117 			       vma);
1118 
1119 	drm_gem_object_put(obj);
1120 
1121 	return ret;
1122 }
1123 EXPORT_SYMBOL(drm_gem_mmap);
1124 
drm_gem_print_info(struct drm_printer * p,unsigned int indent,const struct drm_gem_object * obj)1125 void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
1126 			const struct drm_gem_object *obj)
1127 {
1128 	drm_printf_indent(p, indent, "name=%d\n", obj->name);
1129 	drm_printf_indent(p, indent, "refcount=%u\n",
1130 			  kref_read(&obj->refcount));
1131 	drm_printf_indent(p, indent, "start=%08lx\n",
1132 			  drm_vma_node_start(&obj->vma_node));
1133 	drm_printf_indent(p, indent, "size=%zu\n", obj->size);
1134 	drm_printf_indent(p, indent, "imported=%s\n",
1135 			  str_yes_no(obj->import_attach));
1136 
1137 	if (obj->funcs->print_info)
1138 		obj->funcs->print_info(p, indent, obj);
1139 }
1140 
drm_gem_pin(struct drm_gem_object * obj)1141 int drm_gem_pin(struct drm_gem_object *obj)
1142 {
1143 	if (obj->funcs->pin)
1144 		return obj->funcs->pin(obj);
1145 	else
1146 		return 0;
1147 }
1148 
drm_gem_unpin(struct drm_gem_object * obj)1149 void drm_gem_unpin(struct drm_gem_object *obj)
1150 {
1151 	if (obj->funcs->unpin)
1152 		obj->funcs->unpin(obj);
1153 }
1154 
drm_gem_vmap(struct drm_gem_object * obj,struct iosys_map * map)1155 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1156 {
1157 	int ret;
1158 
1159 	if (!obj->funcs->vmap)
1160 		return -EOPNOTSUPP;
1161 
1162 	ret = obj->funcs->vmap(obj, map);
1163 	if (ret)
1164 		return ret;
1165 	else if (iosys_map_is_null(map))
1166 		return -ENOMEM;
1167 
1168 	return 0;
1169 }
1170 EXPORT_SYMBOL(drm_gem_vmap);
1171 
drm_gem_vunmap(struct drm_gem_object * obj,struct iosys_map * map)1172 void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1173 {
1174 	if (iosys_map_is_null(map))
1175 		return;
1176 
1177 	if (obj->funcs->vunmap)
1178 		obj->funcs->vunmap(obj, map);
1179 
1180 	/* Always set the mapping to NULL. Callers may rely on this. */
1181 	iosys_map_clear(map);
1182 }
1183 EXPORT_SYMBOL(drm_gem_vunmap);
1184 
1185 /**
1186  * drm_gem_lock_reservations - Sets up the ww context and acquires
1187  * the lock on an array of GEM objects.
1188  *
1189  * Once you've locked your reservations, you'll want to set up space
1190  * for your shared fences (if applicable), submit your job, then
1191  * drm_gem_unlock_reservations().
1192  *
1193  * @objs: drm_gem_objects to lock
1194  * @count: Number of objects in @objs
1195  * @acquire_ctx: struct ww_acquire_ctx that will be initialized as
1196  * part of tracking this set of locked reservations.
1197  */
1198 int
drm_gem_lock_reservations(struct drm_gem_object ** objs,int count,struct ww_acquire_ctx * acquire_ctx)1199 drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
1200 			  struct ww_acquire_ctx *acquire_ctx)
1201 {
1202 	int contended = -1;
1203 	int i, ret;
1204 
1205 	ww_acquire_init(acquire_ctx, &reservation_ww_class);
1206 
1207 retry:
1208 	if (contended != -1) {
1209 		struct drm_gem_object *obj = objs[contended];
1210 
1211 		ret = dma_resv_lock_slow_interruptible(obj->resv,
1212 								 acquire_ctx);
1213 		if (ret) {
1214 			ww_acquire_fini(acquire_ctx);
1215 			return ret;
1216 		}
1217 	}
1218 
1219 	for (i = 0; i < count; i++) {
1220 		if (i == contended)
1221 			continue;
1222 
1223 		ret = dma_resv_lock_interruptible(objs[i]->resv,
1224 							    acquire_ctx);
1225 		if (ret) {
1226 			int j;
1227 
1228 			for (j = 0; j < i; j++)
1229 				dma_resv_unlock(objs[j]->resv);
1230 
1231 			if (contended != -1 && contended >= i)
1232 				dma_resv_unlock(objs[contended]->resv);
1233 
1234 			if (ret == -EDEADLK) {
1235 				contended = i;
1236 				goto retry;
1237 			}
1238 
1239 			ww_acquire_fini(acquire_ctx);
1240 			return ret;
1241 		}
1242 	}
1243 
1244 	ww_acquire_done(acquire_ctx);
1245 
1246 	return 0;
1247 }
1248 EXPORT_SYMBOL(drm_gem_lock_reservations);
1249 
1250 void
drm_gem_unlock_reservations(struct drm_gem_object ** objs,int count,struct ww_acquire_ctx * acquire_ctx)1251 drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
1252 			    struct ww_acquire_ctx *acquire_ctx)
1253 {
1254 	int i;
1255 
1256 	for (i = 0; i < count; i++)
1257 		dma_resv_unlock(objs[i]->resv);
1258 
1259 	ww_acquire_fini(acquire_ctx);
1260 }
1261 EXPORT_SYMBOL(drm_gem_unlock_reservations);
1262