1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022 Red Hat.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  * Authors:
24  *     Danilo Krummrich <dakr@redhat.com>
25  *
26  */
27 
28 #include <drm/drm_gpuva_mgr.h>
29 
30 #include <linux/interval_tree_generic.h>
31 #include <linux/mm.h>
32 
33 /**
34  * DOC: Overview
35  *
36  * The DRM GPU VA Manager, represented by struct drm_gpuva_manager keeps track
37  * of a GPU's virtual address (VA) space and manages the corresponding virtual
38  * mappings represented by &drm_gpuva objects. It also keeps track of the
39  * mapping's backing &drm_gem_object buffers.
40  *
41  * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing
42  * all existent GPU VA mappings using this &drm_gem_object as backing buffer.
43  *
44  * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also
45  * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'.
46  *
47  * The GPU VA manager internally uses a rb-tree to manage the
48  * &drm_gpuva mappings within a GPU's virtual address space.
49  *
50  * The &drm_gpuva_manager contains a special &drm_gpuva representing the
51  * portion of VA space reserved by the kernel. This node is initialized together
52  * with the GPU VA manager instance and removed when the GPU VA manager is
53  * destroyed.
54  *
55  * In a typical application drivers would embed struct drm_gpuva_manager and
56  * struct drm_gpuva within their own driver specific structures, there won't be
57  * any memory allocations of its own nor memory allocations of &drm_gpuva
58  * entries.
59  *
60  * The data structures needed to store &drm_gpuvas within the &drm_gpuva_manager
61  * are contained within struct drm_gpuva already. Hence, for inserting
62  * &drm_gpuva entries from within dma-fence signalling critical sections it is
63  * enough to pre-allocate the &drm_gpuva structures.
64  */
65 
66 /**
67  * DOC: Split and Merge
68  *
69  * Besides its capability to manage and represent a GPU VA space, the
70  * &drm_gpuva_manager also provides functions to let the &drm_gpuva_manager
71  * calculate a sequence of operations to satisfy a given map or unmap request.
72  *
73  * Therefore the DRM GPU VA manager provides an algorithm implementing splitting
74  * and merging of existent GPU VA mappings with the ones that are requested to
75  * be mapped or unmapped. This feature is required by the Vulkan API to
76  * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this
77  * as VM BIND.
78  *
79  * Drivers can call drm_gpuva_sm_map() to receive a sequence of callbacks
80  * containing map, unmap and remap operations for a given newly requested
81  * mapping. The sequence of callbacks represents the set of operations to
82  * execute in order to integrate the new mapping cleanly into the current state
83  * of the GPU VA space.
84  *
85  * Depending on how the new GPU VA mapping intersects with the existent mappings
86  * of the GPU VA space the &drm_gpuva_fn_ops callbacks contain an arbitrary
87  * amount of unmap operations, a maximum of two remap operations and a single
88  * map operation. The caller might receive no callback at all if no operation is
89  * required, e.g. if the requested mapping already exists in the exact same way.
90  *
91  * The single map operation represents the original map operation requested by
92  * the caller.
93  *
94  * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the
95  * &drm_gpuva to unmap is physically contiguous with the original mapping
96  * request. Optionally, if 'keep' is set, drivers may keep the actual page table
97  * entries for this &drm_gpuva, adding the missing page table entries only and
98  * update the &drm_gpuva_manager's view of things accordingly.
99  *
100  * Drivers may do the same optimization, namely delta page table updates, also
101  * for remap operations. This is possible since &drm_gpuva_op_remap consists of
102  * one unmap operation and one or two map operations, such that drivers can
103  * derive the page table update delta accordingly.
104  *
105  * Note that there can't be more than two existent mappings to split up, one at
106  * the beginning and one at the end of the new mapping, hence there is a
107  * maximum of two remap operations.
108  *
109  * Analogous to drm_gpuva_sm_map() drm_gpuva_sm_unmap() uses &drm_gpuva_fn_ops
110  * to call back into the driver in order to unmap a range of GPU VA space. The
111  * logic behind this function is way simpler though: For all existent mappings
112  * enclosed by the given range unmap operations are created. For mappings which
113  * are only partically located within the given range, remap operations are
114  * created such that those mappings are split up and re-mapped partically.
115  *
116  * As an alternative to drm_gpuva_sm_map() and drm_gpuva_sm_unmap(),
117  * drm_gpuva_sm_map_ops_create() and drm_gpuva_sm_unmap_ops_create() can be used
118  * to directly obtain an instance of struct drm_gpuva_ops containing a list of
119  * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list
120  * contains the &drm_gpuva_ops analogous to the callbacks one would receive when
121  * calling drm_gpuva_sm_map() or drm_gpuva_sm_unmap(). While this way requires
122  * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to
123  * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory
124  * allocations are possible (e.g. to allocate GPU page tables) and once in the
125  * dma-fence signalling critical path.
126  *
127  * To update the &drm_gpuva_manager's view of the GPU VA space
128  * drm_gpuva_insert() and drm_gpuva_remove() may be used. These functions can
129  * safely be used from &drm_gpuva_fn_ops callbacks originating from
130  * drm_gpuva_sm_map() or drm_gpuva_sm_unmap(). However, it might be more
131  * convenient to use the provided helper functions drm_gpuva_map(),
132  * drm_gpuva_remap() and drm_gpuva_unmap() instead.
133  *
134  * The following diagram depicts the basic relationships of existent GPU VA
135  * mappings, a newly requested mapping and the resulting mappings as implemented
136  * by drm_gpuva_sm_map() - it doesn't cover any arbitrary combinations of these.
137  *
138  * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs
139  *    could be kept.
140  *
141  *    ::
142  *
143  *	     0     a     1
144  *	old: |-----------| (bo_offset=n)
145  *
146  *	     0     a     1
147  *	req: |-----------| (bo_offset=n)
148  *
149  *	     0     a     1
150  *	new: |-----------| (bo_offset=n)
151  *
152  *
153  * 2) Requested mapping is identical, except for the BO offset, hence replace
154  *    the mapping.
155  *
156  *    ::
157  *
158  *	     0     a     1
159  *	old: |-----------| (bo_offset=n)
160  *
161  *	     0     a     1
162  *	req: |-----------| (bo_offset=m)
163  *
164  *	     0     a     1
165  *	new: |-----------| (bo_offset=m)
166  *
167  *
168  * 3) Requested mapping is identical, except for the backing BO, hence replace
169  *    the mapping.
170  *
171  *    ::
172  *
173  *	     0     a     1
174  *	old: |-----------| (bo_offset=n)
175  *
176  *	     0     b     1
177  *	req: |-----------| (bo_offset=n)
178  *
179  *	     0     b     1
180  *	new: |-----------| (bo_offset=n)
181  *
182  *
183  * 4) Existent mapping is a left aligned subset of the requested one, hence
184  *    replace the existent one.
185  *
186  *    ::
187  *
188  *	     0  a  1
189  *	old: |-----|       (bo_offset=n)
190  *
191  *	     0     a     2
192  *	req: |-----------| (bo_offset=n)
193  *
194  *	     0     a     2
195  *	new: |-----------| (bo_offset=n)
196  *
197  *    .. note::
198  *       We expect to see the same result for a request with a different BO
199  *       and/or non-contiguous BO offset.
200  *
201  *
202  * 5) Requested mapping's range is a left aligned subset of the existent one,
203  *    but backed by a different BO. Hence, map the requested mapping and split
204  *    the existent one adjusting its BO offset.
205  *
206  *    ::
207  *
208  *	     0     a     2
209  *	old: |-----------| (bo_offset=n)
210  *
211  *	     0  b  1
212  *	req: |-----|       (bo_offset=n)
213  *
214  *	     0  b  1  a' 2
215  *	new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1)
216  *
217  *    .. note::
218  *       We expect to see the same result for a request with a different BO
219  *       and/or non-contiguous BO offset.
220  *
221  *
222  * 6) Existent mapping is a superset of the requested mapping. Split it up, but
223  *    indicate that the backing PTEs could be kept.
224  *
225  *    ::
226  *
227  *	     0     a     2
228  *	old: |-----------| (bo_offset=n)
229  *
230  *	     0  a  1
231  *	req: |-----|       (bo_offset=n)
232  *
233  *	     0  a  1  a' 2
234  *	new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1)
235  *
236  *
237  * 7) Requested mapping's range is a right aligned subset of the existent one,
238  *    but backed by a different BO. Hence, map the requested mapping and split
239  *    the existent one, without adjusting the BO offset.
240  *
241  *    ::
242  *
243  *	     0     a     2
244  *	old: |-----------| (bo_offset=n)
245  *
246  *	           1  b  2
247  *	req:       |-----| (bo_offset=m)
248  *
249  *	     0  a  1  b  2
250  *	new: |-----|-----| (a.bo_offset=n,b.bo_offset=m)
251  *
252  *
253  * 8) Existent mapping is a superset of the requested mapping. Split it up, but
254  *    indicate that the backing PTEs could be kept.
255  *
256  *    ::
257  *
258  *	      0     a     2
259  *	old: |-----------| (bo_offset=n)
260  *
261  *	           1  a  2
262  *	req:       |-----| (bo_offset=n+1)
263  *
264  *	     0  a' 1  a  2
265  *	new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1)
266  *
267  *
268  * 9) Existent mapping is overlapped at the end by the requested mapping backed
269  *    by a different BO. Hence, map the requested mapping and split up the
270  *    existent one, without adjusting the BO offset.
271  *
272  *    ::
273  *
274  *	     0     a     2
275  *	old: |-----------|       (bo_offset=n)
276  *
277  *	           1     b     3
278  *	req:       |-----------| (bo_offset=m)
279  *
280  *	     0  a  1     b     3
281  *	new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m)
282  *
283  *
284  * 10) Existent mapping is overlapped by the requested mapping, both having the
285  *     same backing BO with a contiguous offset. Indicate the backing PTEs of
286  *     the old mapping could be kept.
287  *
288  *     ::
289  *
290  *	      0     a     2
291  *	 old: |-----------|       (bo_offset=n)
292  *
293  *	            1     a     3
294  *	 req:       |-----------| (bo_offset=n+1)
295  *
296  *	      0  a' 1     a     3
297  *	 new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1)
298  *
299  *
300  * 11) Requested mapping's range is a centered subset of the existent one
301  *     having a different backing BO. Hence, map the requested mapping and split
302  *     up the existent one in two mappings, adjusting the BO offset of the right
303  *     one accordingly.
304  *
305  *     ::
306  *
307  *	      0        a        3
308  *	 old: |-----------------| (bo_offset=n)
309  *
310  *	            1  b  2
311  *	 req:       |-----|       (bo_offset=m)
312  *
313  *	      0  a  1  b  2  a' 3
314  *	 new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2)
315  *
316  *
317  * 12) Requested mapping is a contiguous subset of the existent one. Split it
318  *     up, but indicate that the backing PTEs could be kept.
319  *
320  *     ::
321  *
322  *	      0        a        3
323  *	 old: |-----------------| (bo_offset=n)
324  *
325  *	            1  a  2
326  *	 req:       |-----|       (bo_offset=n+1)
327  *
328  *	      0  a' 1  a  2 a'' 3
329  *	 old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2)
330  *
331  *
332  * 13) Existent mapping is a right aligned subset of the requested one, hence
333  *     replace the existent one.
334  *
335  *     ::
336  *
337  *	            1  a  2
338  *	 old:       |-----| (bo_offset=n+1)
339  *
340  *	      0     a     2
341  *	 req: |-----------| (bo_offset=n)
342  *
343  *	      0     a     2
344  *	 new: |-----------| (bo_offset=n)
345  *
346  *     .. note::
347  *        We expect to see the same result for a request with a different bo
348  *        and/or non-contiguous bo_offset.
349  *
350  *
351  * 14) Existent mapping is a centered subset of the requested one, hence
352  *     replace the existent one.
353  *
354  *     ::
355  *
356  *	            1  a  2
357  *	 old:       |-----| (bo_offset=n+1)
358  *
359  *	      0        a       3
360  *	 req: |----------------| (bo_offset=n)
361  *
362  *	      0        a       3
363  *	 new: |----------------| (bo_offset=n)
364  *
365  *     .. note::
366  *        We expect to see the same result for a request with a different bo
367  *        and/or non-contiguous bo_offset.
368  *
369  *
370  * 15) Existent mappings is overlapped at the beginning by the requested mapping
371  *     backed by a different BO. Hence, map the requested mapping and split up
372  *     the existent one, adjusting its BO offset accordingly.
373  *
374  *     ::
375  *
376  *	            1     a     3
377  *	 old:       |-----------| (bo_offset=n)
378  *
379  *	      0     b     2
380  *	 req: |-----------|       (bo_offset=m)
381  *
382  *	      0     b     2  a' 3
383  *	 new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2)
384  */
385 
386 /**
387  * DOC: Locking
388  *
389  * Generally, the GPU VA manager does not take care of locking itself, it is
390  * the drivers responsibility to take care about locking. Drivers might want to
391  * protect the following operations: inserting, removing and iterating
392  * &drm_gpuva objects as well as generating all kinds of operations, such as
393  * split / merge or prefetch.
394  *
395  * The GPU VA manager also does not take care of the locking of the backing
396  * &drm_gem_object buffers GPU VA lists by itself; drivers are responsible to
397  * enforce mutual exclusion using either the GEMs dma_resv lock or alternatively
398  * a driver specific external lock. For the latter see also
399  * drm_gem_gpuva_set_lock().
400  *
401  * However, the GPU VA manager contains lockdep checks to ensure callers of its
402  * API hold the corresponding lock whenever the &drm_gem_objects GPU VA list is
403  * accessed by functions such as drm_gpuva_link() or drm_gpuva_unlink().
404  */
405 
406 /**
407  * DOC: Examples
408  *
409  * This section gives two examples on how to let the DRM GPUVA Manager generate
410  * &drm_gpuva_op in order to satisfy a given map or unmap request and how to
411  * make use of them.
412  *
413  * The below code is strictly limited to illustrate the generic usage pattern.
414  * To maintain simplicitly, it doesn't make use of any abstractions for common
415  * code, different (asyncronous) stages with fence signalling critical paths,
416  * any other helpers or error handling in terms of freeing memory and dropping
417  * previously taken locks.
418  *
419  * 1) Obtain a list of &drm_gpuva_op to create a new mapping::
420  *
421  *	// Allocates a new &drm_gpuva.
422  *	struct drm_gpuva * driver_gpuva_alloc(void);
423  *
424  *	// Typically drivers would embedd the &drm_gpuva_manager and &drm_gpuva
425  *	// structure in individual driver structures and lock the dma-resv with
426  *	// drm_exec or similar helpers.
427  *	int driver_mapping_create(struct drm_gpuva_manager *mgr,
428  *				  u64 addr, u64 range,
429  *				  struct drm_gem_object *obj, u64 offset)
430  *	{
431  *		struct drm_gpuva_ops *ops;
432  *		struct drm_gpuva_op *op
433  *
434  *		driver_lock_va_space();
435  *		ops = drm_gpuva_sm_map_ops_create(mgr, addr, range,
436  *						  obj, offset);
437  *		if (IS_ERR(ops))
438  *			return PTR_ERR(ops);
439  *
440  *		drm_gpuva_for_each_op(op, ops) {
441  *			struct drm_gpuva *va;
442  *
443  *			switch (op->op) {
444  *			case DRM_GPUVA_OP_MAP:
445  *				va = driver_gpuva_alloc();
446  *				if (!va)
447  *					; // unwind previous VA space updates,
448  *					  // free memory and unlock
449  *
450  *				driver_vm_map();
451  *				drm_gpuva_map(mgr, va, &op->map);
452  *				drm_gpuva_link(va);
453  *
454  *				break;
455  *			case DRM_GPUVA_OP_REMAP: {
456  *				struct drm_gpuva *prev = NULL, *next = NULL;
457  *
458  *				va = op->remap.unmap->va;
459  *
460  *				if (op->remap.prev) {
461  *					prev = driver_gpuva_alloc();
462  *					if (!prev)
463  *						; // unwind previous VA space
464  *						  // updates, free memory and
465  *						  // unlock
466  *				}
467  *
468  *				if (op->remap.next) {
469  *					next = driver_gpuva_alloc();
470  *					if (!next)
471  *						; // unwind previous VA space
472  *						  // updates, free memory and
473  *						  // unlock
474  *				}
475  *
476  *				driver_vm_remap();
477  *				drm_gpuva_remap(prev, next, &op->remap);
478  *
479  *				drm_gpuva_unlink(va);
480  *				if (prev)
481  *					drm_gpuva_link(prev);
482  *				if (next)
483  *					drm_gpuva_link(next);
484  *
485  *				break;
486  *			}
487  *			case DRM_GPUVA_OP_UNMAP:
488  *				va = op->unmap->va;
489  *
490  *				driver_vm_unmap();
491  *				drm_gpuva_unlink(va);
492  *				drm_gpuva_unmap(&op->unmap);
493  *
494  *				break;
495  *			default:
496  *				break;
497  *			}
498  *		}
499  *		driver_unlock_va_space();
500  *
501  *		return 0;
502  *	}
503  *
504  * 2) Receive a callback for each &drm_gpuva_op to create a new mapping::
505  *
506  *	struct driver_context {
507  *		struct drm_gpuva_manager *mgr;
508  *		struct drm_gpuva *new_va;
509  *		struct drm_gpuva *prev_va;
510  *		struct drm_gpuva *next_va;
511  *	};
512  *
513  *	// ops to pass to drm_gpuva_manager_init()
514  *	static const struct drm_gpuva_fn_ops driver_gpuva_ops = {
515  *		.sm_step_map = driver_gpuva_map,
516  *		.sm_step_remap = driver_gpuva_remap,
517  *		.sm_step_unmap = driver_gpuva_unmap,
518  *	};
519  *
520  *	// Typically drivers would embedd the &drm_gpuva_manager and &drm_gpuva
521  *	// structure in individual driver structures and lock the dma-resv with
522  *	// drm_exec or similar helpers.
523  *	int driver_mapping_create(struct drm_gpuva_manager *mgr,
524  *				  u64 addr, u64 range,
525  *				  struct drm_gem_object *obj, u64 offset)
526  *	{
527  *		struct driver_context ctx;
528  *		struct drm_gpuva_ops *ops;
529  *		struct drm_gpuva_op *op;
530  *		int ret = 0;
531  *
532  *		ctx.mgr = mgr;
533  *
534  *		ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);
535  *		ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);
536  *		ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);
537  *		if (!ctx.new_va || !ctx.prev_va || !ctx.next_va) {
538  *			ret = -ENOMEM;
539  *			goto out;
540  *		}
541  *
542  *		driver_lock_va_space();
543  *		ret = drm_gpuva_sm_map(mgr, &ctx, addr, range, obj, offset);
544  *		driver_unlock_va_space();
545  *
546  *	out:
547  *		kfree(ctx.new_va);
548  *		kfree(ctx.prev_va);
549  *		kfree(ctx.next_va);
550  *		return ret;
551  *	}
552  *
553  *	int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx)
554  *	{
555  *		struct driver_context *ctx = __ctx;
556  *
557  *		drm_gpuva_map(ctx->mgr, ctx->new_va, &op->map);
558  *
559  *		drm_gpuva_link(ctx->new_va);
560  *
561  *		// prevent the new GPUVA from being freed in
562  *		// driver_mapping_create()
563  *		ctx->new_va = NULL;
564  *
565  *		return 0;
566  *	}
567  *
568  *	int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)
569  *	{
570  *		struct driver_context *ctx = __ctx;
571  *
572  *		drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);
573  *
574  *		drm_gpuva_unlink(op->remap.unmap->va);
575  *		kfree(op->remap.unmap->va);
576  *
577  *		if (op->remap.prev) {
578  *			drm_gpuva_link(ctx->prev_va);
579  *			ctx->prev_va = NULL;
580  *		}
581  *
582  *		if (op->remap.next) {
583  *			drm_gpuva_link(ctx->next_va);
584  *			ctx->next_va = NULL;
585  *		}
586  *
587  *		return 0;
588  *	}
589  *
590  *	int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx)
591  *	{
592  *		drm_gpuva_unlink(op->unmap.va);
593  *		drm_gpuva_unmap(&op->unmap);
594  *		kfree(op->unmap.va);
595  *
596  *		return 0;
597  *	}
598  */
599 
600 #define to_drm_gpuva(__node)	container_of((__node), struct drm_gpuva, rb.node)
601 
602 #define GPUVA_START(node) ((node)->va.addr)
603 #define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
604 
605 /* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain
606  * about this.
607  */
608 INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last,
609 		     GPUVA_START, GPUVA_LAST, static __maybe_unused,
610 		     drm_gpuva_it)
611 
612 static int __drm_gpuva_insert(struct drm_gpuva_manager *mgr,
613 			      struct drm_gpuva *va);
614 static void __drm_gpuva_remove(struct drm_gpuva *va);
615 
616 static bool
drm_gpuva_check_overflow(u64 addr,u64 range)617 drm_gpuva_check_overflow(u64 addr, u64 range)
618 {
619 	u64 end;
620 
621 	return WARN(check_add_overflow(addr, range, &end),
622 		    "GPUVA address limited to %zu bytes.\n", sizeof(end));
623 }
624 
625 static bool
drm_gpuva_in_mm_range(struct drm_gpuva_manager * mgr,u64 addr,u64 range)626 drm_gpuva_in_mm_range(struct drm_gpuva_manager *mgr, u64 addr, u64 range)
627 {
628 	u64 end = addr + range;
629 	u64 mm_start = mgr->mm_start;
630 	u64 mm_end = mm_start + mgr->mm_range;
631 
632 	return addr >= mm_start && end <= mm_end;
633 }
634 
635 static bool
drm_gpuva_in_kernel_node(struct drm_gpuva_manager * mgr,u64 addr,u64 range)636 drm_gpuva_in_kernel_node(struct drm_gpuva_manager *mgr, u64 addr, u64 range)
637 {
638 	u64 end = addr + range;
639 	u64 kstart = mgr->kernel_alloc_node.va.addr;
640 	u64 krange = mgr->kernel_alloc_node.va.range;
641 	u64 kend = kstart + krange;
642 
643 	return krange && addr < kend && kstart < end;
644 }
645 
646 static bool
drm_gpuva_range_valid(struct drm_gpuva_manager * mgr,u64 addr,u64 range)647 drm_gpuva_range_valid(struct drm_gpuva_manager *mgr,
648 		      u64 addr, u64 range)
649 {
650 	return !drm_gpuva_check_overflow(addr, range) &&
651 	       drm_gpuva_in_mm_range(mgr, addr, range) &&
652 	       !drm_gpuva_in_kernel_node(mgr, addr, range);
653 }
654 
655 /**
656  * drm_gpuva_manager_init() - initialize a &drm_gpuva_manager
657  * @mgr: pointer to the &drm_gpuva_manager to initialize
658  * @name: the name of the GPU VA space
659  * @start_offset: the start offset of the GPU VA space
660  * @range: the size of the GPU VA space
661  * @reserve_offset: the start of the kernel reserved GPU VA area
662  * @reserve_range: the size of the kernel reserved GPU VA area
663  * @ops: &drm_gpuva_fn_ops called on &drm_gpuva_sm_map / &drm_gpuva_sm_unmap
664  *
665  * The &drm_gpuva_manager must be initialized with this function before use.
666  *
667  * Note that @mgr must be cleared to 0 before calling this function. The given
668  * &name is expected to be managed by the surrounding driver structures.
669  */
670 void
drm_gpuva_manager_init(struct drm_gpuva_manager * mgr,const char * name,u64 start_offset,u64 range,u64 reserve_offset,u64 reserve_range,const struct drm_gpuva_fn_ops * ops)671 drm_gpuva_manager_init(struct drm_gpuva_manager *mgr,
672 		       const char *name,
673 		       u64 start_offset, u64 range,
674 		       u64 reserve_offset, u64 reserve_range,
675 		       const struct drm_gpuva_fn_ops *ops)
676 {
677 	mgr->rb.tree = RB_ROOT_CACHED;
678 	INIT_LIST_HEAD(&mgr->rb.list);
679 
680 	drm_gpuva_check_overflow(start_offset, range);
681 	mgr->mm_start = start_offset;
682 	mgr->mm_range = range;
683 
684 	mgr->name = name ? name : "unknown";
685 	mgr->ops = ops;
686 
687 	memset(&mgr->kernel_alloc_node, 0, sizeof(struct drm_gpuva));
688 
689 	if (reserve_range) {
690 		mgr->kernel_alloc_node.va.addr = reserve_offset;
691 		mgr->kernel_alloc_node.va.range = reserve_range;
692 
693 		if (likely(!drm_gpuva_check_overflow(reserve_offset,
694 						     reserve_range)))
695 			__drm_gpuva_insert(mgr, &mgr->kernel_alloc_node);
696 	}
697 }
698 EXPORT_SYMBOL_GPL(drm_gpuva_manager_init);
699 
700 /**
701  * drm_gpuva_manager_destroy() - cleanup a &drm_gpuva_manager
702  * @mgr: pointer to the &drm_gpuva_manager to clean up
703  *
704  * Note that it is a bug to call this function on a manager that still
705  * holds GPU VA mappings.
706  */
707 void
drm_gpuva_manager_destroy(struct drm_gpuva_manager * mgr)708 drm_gpuva_manager_destroy(struct drm_gpuva_manager *mgr)
709 {
710 	mgr->name = NULL;
711 
712 	if (mgr->kernel_alloc_node.va.range)
713 		__drm_gpuva_remove(&mgr->kernel_alloc_node);
714 
715 	WARN(!RB_EMPTY_ROOT(&mgr->rb.tree.rb_root),
716 	     "GPUVA tree is not empty, potentially leaking memory.");
717 }
718 EXPORT_SYMBOL_GPL(drm_gpuva_manager_destroy);
719 
720 static int
__drm_gpuva_insert(struct drm_gpuva_manager * mgr,struct drm_gpuva * va)721 __drm_gpuva_insert(struct drm_gpuva_manager *mgr,
722 		   struct drm_gpuva *va)
723 {
724 	struct rb_node *node;
725 	struct list_head *head;
726 
727 	if (drm_gpuva_it_iter_first(&mgr->rb.tree,
728 				    GPUVA_START(va),
729 				    GPUVA_LAST(va)))
730 		return -EEXIST;
731 
732 	va->mgr = mgr;
733 
734 	drm_gpuva_it_insert(va, &mgr->rb.tree);
735 
736 	node = rb_prev(&va->rb.node);
737 	if (node)
738 		head = &(to_drm_gpuva(node))->rb.entry;
739 	else
740 		head = &mgr->rb.list;
741 
742 	list_add(&va->rb.entry, head);
743 
744 	return 0;
745 }
746 
747 /**
748  * drm_gpuva_insert() - insert a &drm_gpuva
749  * @mgr: the &drm_gpuva_manager to insert the &drm_gpuva in
750  * @va: the &drm_gpuva to insert
751  *
752  * Insert a &drm_gpuva with a given address and range into a
753  * &drm_gpuva_manager.
754  *
755  * It is safe to use this function using the safe versions of iterating the GPU
756  * VA space, such as drm_gpuva_for_each_va_safe() and
757  * drm_gpuva_for_each_va_range_safe().
758  *
759  * Returns: 0 on success, negative error code on failure.
760  */
761 int
drm_gpuva_insert(struct drm_gpuva_manager * mgr,struct drm_gpuva * va)762 drm_gpuva_insert(struct drm_gpuva_manager *mgr,
763 		 struct drm_gpuva *va)
764 {
765 	u64 addr = va->va.addr;
766 	u64 range = va->va.range;
767 
768 	if (unlikely(!drm_gpuva_range_valid(mgr, addr, range)))
769 		return -EINVAL;
770 
771 	return __drm_gpuva_insert(mgr, va);
772 }
773 EXPORT_SYMBOL_GPL(drm_gpuva_insert);
774 
775 static void
__drm_gpuva_remove(struct drm_gpuva * va)776 __drm_gpuva_remove(struct drm_gpuva *va)
777 {
778 	drm_gpuva_it_remove(va, &va->mgr->rb.tree);
779 	list_del_init(&va->rb.entry);
780 }
781 
782 /**
783  * drm_gpuva_remove() - remove a &drm_gpuva
784  * @va: the &drm_gpuva to remove
785  *
786  * This removes the given &va from the underlaying tree.
787  *
788  * It is safe to use this function using the safe versions of iterating the GPU
789  * VA space, such as drm_gpuva_for_each_va_safe() and
790  * drm_gpuva_for_each_va_range_safe().
791  */
792 void
drm_gpuva_remove(struct drm_gpuva * va)793 drm_gpuva_remove(struct drm_gpuva *va)
794 {
795 	struct drm_gpuva_manager *mgr = va->mgr;
796 
797 	if (unlikely(va == &mgr->kernel_alloc_node)) {
798 		WARN(1, "Can't destroy kernel reserved node.\n");
799 		return;
800 	}
801 
802 	__drm_gpuva_remove(va);
803 }
804 EXPORT_SYMBOL_GPL(drm_gpuva_remove);
805 
806 /**
807  * drm_gpuva_link() - link a &drm_gpuva
808  * @va: the &drm_gpuva to link
809  *
810  * This adds the given &va to the GPU VA list of the &drm_gem_object it is
811  * associated with.
812  *
813  * This function expects the caller to protect the GEM's GPUVA list against
814  * concurrent access using the GEMs dma_resv lock.
815  */
816 void
drm_gpuva_link(struct drm_gpuva * va)817 drm_gpuva_link(struct drm_gpuva *va)
818 {
819 	struct drm_gem_object *obj = va->gem.obj;
820 
821 	if (unlikely(!obj))
822 		return;
823 
824 	drm_gem_gpuva_assert_lock_held(obj);
825 
826 	list_add_tail(&va->gem.entry, &obj->gpuva.list);
827 }
828 EXPORT_SYMBOL_GPL(drm_gpuva_link);
829 
830 /**
831  * drm_gpuva_unlink() - unlink a &drm_gpuva
832  * @va: the &drm_gpuva to unlink
833  *
834  * This removes the given &va from the GPU VA list of the &drm_gem_object it is
835  * associated with.
836  *
837  * This function expects the caller to protect the GEM's GPUVA list against
838  * concurrent access using the GEMs dma_resv lock.
839  */
840 void
drm_gpuva_unlink(struct drm_gpuva * va)841 drm_gpuva_unlink(struct drm_gpuva *va)
842 {
843 	struct drm_gem_object *obj = va->gem.obj;
844 
845 	if (unlikely(!obj))
846 		return;
847 
848 	drm_gem_gpuva_assert_lock_held(obj);
849 
850 	list_del_init(&va->gem.entry);
851 }
852 EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
853 
854 /**
855  * drm_gpuva_find_first() - find the first &drm_gpuva in the given range
856  * @mgr: the &drm_gpuva_manager to search in
857  * @addr: the &drm_gpuvas address
858  * @range: the &drm_gpuvas range
859  *
860  * Returns: the first &drm_gpuva within the given range
861  */
862 struct drm_gpuva *
drm_gpuva_find_first(struct drm_gpuva_manager * mgr,u64 addr,u64 range)863 drm_gpuva_find_first(struct drm_gpuva_manager *mgr,
864 		     u64 addr, u64 range)
865 {
866 	u64 last = addr + range - 1;
867 
868 	return drm_gpuva_it_iter_first(&mgr->rb.tree, addr, last);
869 }
870 EXPORT_SYMBOL_GPL(drm_gpuva_find_first);
871 
872 /**
873  * drm_gpuva_find() - find a &drm_gpuva
874  * @mgr: the &drm_gpuva_manager to search in
875  * @addr: the &drm_gpuvas address
876  * @range: the &drm_gpuvas range
877  *
878  * Returns: the &drm_gpuva at a given &addr and with a given &range
879  */
880 struct drm_gpuva *
drm_gpuva_find(struct drm_gpuva_manager * mgr,u64 addr,u64 range)881 drm_gpuva_find(struct drm_gpuva_manager *mgr,
882 	       u64 addr, u64 range)
883 {
884 	struct drm_gpuva *va;
885 
886 	va = drm_gpuva_find_first(mgr, addr, range);
887 	if (!va)
888 		goto out;
889 
890 	if (va->va.addr != addr ||
891 	    va->va.range != range)
892 		goto out;
893 
894 	return va;
895 
896 out:
897 	return NULL;
898 }
899 EXPORT_SYMBOL_GPL(drm_gpuva_find);
900 
901 /**
902  * drm_gpuva_find_prev() - find the &drm_gpuva before the given address
903  * @mgr: the &drm_gpuva_manager to search in
904  * @start: the given GPU VA's start address
905  *
906  * Find the adjacent &drm_gpuva before the GPU VA with given &start address.
907  *
908  * Note that if there is any free space between the GPU VA mappings no mapping
909  * is returned.
910  *
911  * Returns: a pointer to the found &drm_gpuva or NULL if none was found
912  */
913 struct drm_gpuva *
drm_gpuva_find_prev(struct drm_gpuva_manager * mgr,u64 start)914 drm_gpuva_find_prev(struct drm_gpuva_manager *mgr, u64 start)
915 {
916 	if (!drm_gpuva_range_valid(mgr, start - 1, 1))
917 		return NULL;
918 
919 	return drm_gpuva_it_iter_first(&mgr->rb.tree, start - 1, start);
920 }
921 EXPORT_SYMBOL_GPL(drm_gpuva_find_prev);
922 
923 /**
924  * drm_gpuva_find_next() - find the &drm_gpuva after the given address
925  * @mgr: the &drm_gpuva_manager to search in
926  * @end: the given GPU VA's end address
927  *
928  * Find the adjacent &drm_gpuva after the GPU VA with given &end address.
929  *
930  * Note that if there is any free space between the GPU VA mappings no mapping
931  * is returned.
932  *
933  * Returns: a pointer to the found &drm_gpuva or NULL if none was found
934  */
935 struct drm_gpuva *
drm_gpuva_find_next(struct drm_gpuva_manager * mgr,u64 end)936 drm_gpuva_find_next(struct drm_gpuva_manager *mgr, u64 end)
937 {
938 	if (!drm_gpuva_range_valid(mgr, end, 1))
939 		return NULL;
940 
941 	return drm_gpuva_it_iter_first(&mgr->rb.tree, end, end + 1);
942 }
943 EXPORT_SYMBOL_GPL(drm_gpuva_find_next);
944 
945 /**
946  * drm_gpuva_interval_empty() - indicate whether a given interval of the VA space
947  * is empty
948  * @mgr: the &drm_gpuva_manager to check the range for
949  * @addr: the start address of the range
950  * @range: the range of the interval
951  *
952  * Returns: true if the interval is empty, false otherwise
953  */
954 bool
drm_gpuva_interval_empty(struct drm_gpuva_manager * mgr,u64 addr,u64 range)955 drm_gpuva_interval_empty(struct drm_gpuva_manager *mgr, u64 addr, u64 range)
956 {
957 	return !drm_gpuva_find_first(mgr, addr, range);
958 }
959 EXPORT_SYMBOL_GPL(drm_gpuva_interval_empty);
960 
961 /**
962  * drm_gpuva_map() - helper to insert a &drm_gpuva according to a
963  * &drm_gpuva_op_map
964  * @mgr: the &drm_gpuva_manager
965  * @va: the &drm_gpuva to insert
966  * @op: the &drm_gpuva_op_map to initialize @va with
967  *
968  * Initializes the @va from the @op and inserts it into the given @mgr.
969  */
970 void
drm_gpuva_map(struct drm_gpuva_manager * mgr,struct drm_gpuva * va,struct drm_gpuva_op_map * op)971 drm_gpuva_map(struct drm_gpuva_manager *mgr,
972 	      struct drm_gpuva *va,
973 	      struct drm_gpuva_op_map *op)
974 {
975 	drm_gpuva_init_from_op(va, op);
976 	drm_gpuva_insert(mgr, va);
977 }
978 EXPORT_SYMBOL_GPL(drm_gpuva_map);
979 
980 /**
981  * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a
982  * &drm_gpuva_op_remap
983  * @prev: the &drm_gpuva to remap when keeping the start of a mapping
984  * @next: the &drm_gpuva to remap when keeping the end of a mapping
985  * @op: the &drm_gpuva_op_remap to initialize @prev and @next with
986  *
987  * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or
988  * @next.
989  */
990 void
drm_gpuva_remap(struct drm_gpuva * prev,struct drm_gpuva * next,struct drm_gpuva_op_remap * op)991 drm_gpuva_remap(struct drm_gpuva *prev,
992 		struct drm_gpuva *next,
993 		struct drm_gpuva_op_remap *op)
994 {
995 	struct drm_gpuva *curr = op->unmap->va;
996 	struct drm_gpuva_manager *mgr = curr->mgr;
997 
998 	drm_gpuva_remove(curr);
999 
1000 	if (op->prev) {
1001 		drm_gpuva_init_from_op(prev, op->prev);
1002 		drm_gpuva_insert(mgr, prev);
1003 	}
1004 
1005 	if (op->next) {
1006 		drm_gpuva_init_from_op(next, op->next);
1007 		drm_gpuva_insert(mgr, next);
1008 	}
1009 }
1010 EXPORT_SYMBOL_GPL(drm_gpuva_remap);
1011 
1012 /**
1013  * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a
1014  * &drm_gpuva_op_unmap
1015  * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove
1016  *
1017  * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap.
1018  */
1019 void
drm_gpuva_unmap(struct drm_gpuva_op_unmap * op)1020 drm_gpuva_unmap(struct drm_gpuva_op_unmap *op)
1021 {
1022 	drm_gpuva_remove(op->va);
1023 }
1024 EXPORT_SYMBOL_GPL(drm_gpuva_unmap);
1025 
1026 static int
op_map_cb(const struct drm_gpuva_fn_ops * fn,void * priv,u64 addr,u64 range,struct drm_gem_object * obj,u64 offset)1027 op_map_cb(const struct drm_gpuva_fn_ops *fn, void *priv,
1028 	  u64 addr, u64 range,
1029 	  struct drm_gem_object *obj, u64 offset)
1030 {
1031 	struct drm_gpuva_op op = {};
1032 
1033 	op.op = DRM_GPUVA_OP_MAP;
1034 	op.map.va.addr = addr;
1035 	op.map.va.range = range;
1036 	op.map.gem.obj = obj;
1037 	op.map.gem.offset = offset;
1038 
1039 	return fn->sm_step_map(&op, priv);
1040 }
1041 
1042 static int
op_remap_cb(const struct drm_gpuva_fn_ops * fn,void * priv,struct drm_gpuva_op_map * prev,struct drm_gpuva_op_map * next,struct drm_gpuva_op_unmap * unmap)1043 op_remap_cb(const struct drm_gpuva_fn_ops *fn, void *priv,
1044 	    struct drm_gpuva_op_map *prev,
1045 	    struct drm_gpuva_op_map *next,
1046 	    struct drm_gpuva_op_unmap *unmap)
1047 {
1048 	struct drm_gpuva_op op = {};
1049 	struct drm_gpuva_op_remap *r;
1050 
1051 	op.op = DRM_GPUVA_OP_REMAP;
1052 	r = &op.remap;
1053 	r->prev = prev;
1054 	r->next = next;
1055 	r->unmap = unmap;
1056 
1057 	return fn->sm_step_remap(&op, priv);
1058 }
1059 
1060 static int
op_unmap_cb(const struct drm_gpuva_fn_ops * fn,void * priv,struct drm_gpuva * va,bool merge)1061 op_unmap_cb(const struct drm_gpuva_fn_ops *fn, void *priv,
1062 	    struct drm_gpuva *va, bool merge)
1063 {
1064 	struct drm_gpuva_op op = {};
1065 
1066 	op.op = DRM_GPUVA_OP_UNMAP;
1067 	op.unmap.va = va;
1068 	op.unmap.keep = merge;
1069 
1070 	return fn->sm_step_unmap(&op, priv);
1071 }
1072 
1073 static int
__drm_gpuva_sm_map(struct drm_gpuva_manager * mgr,const struct drm_gpuva_fn_ops * ops,void * priv,u64 req_addr,u64 req_range,struct drm_gem_object * req_obj,u64 req_offset)1074 __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr,
1075 		   const struct drm_gpuva_fn_ops *ops, void *priv,
1076 		   u64 req_addr, u64 req_range,
1077 		   struct drm_gem_object *req_obj, u64 req_offset)
1078 {
1079 	struct drm_gpuva *va, *next;
1080 	u64 req_end = req_addr + req_range;
1081 	int ret;
1082 
1083 	if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range)))
1084 		return -EINVAL;
1085 
1086 	drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) {
1087 		struct drm_gem_object *obj = va->gem.obj;
1088 		u64 offset = va->gem.offset;
1089 		u64 addr = va->va.addr;
1090 		u64 range = va->va.range;
1091 		u64 end = addr + range;
1092 		bool merge = !!va->gem.obj;
1093 
1094 		if (addr == req_addr) {
1095 			merge &= obj == req_obj &&
1096 				 offset == req_offset;
1097 
1098 			if (end == req_end) {
1099 				ret = op_unmap_cb(ops, priv, va, merge);
1100 				if (ret)
1101 					return ret;
1102 				break;
1103 			}
1104 
1105 			if (end < req_end) {
1106 				ret = op_unmap_cb(ops, priv, va, merge);
1107 				if (ret)
1108 					return ret;
1109 				continue;
1110 			}
1111 
1112 			if (end > req_end) {
1113 				struct drm_gpuva_op_map n = {
1114 					.va.addr = req_end,
1115 					.va.range = range - req_range,
1116 					.gem.obj = obj,
1117 					.gem.offset = offset + req_range,
1118 				};
1119 				struct drm_gpuva_op_unmap u = {
1120 					.va = va,
1121 					.keep = merge,
1122 				};
1123 
1124 				ret = op_remap_cb(ops, priv, NULL, &n, &u);
1125 				if (ret)
1126 					return ret;
1127 				break;
1128 			}
1129 		} else if (addr < req_addr) {
1130 			u64 ls_range = req_addr - addr;
1131 			struct drm_gpuva_op_map p = {
1132 				.va.addr = addr,
1133 				.va.range = ls_range,
1134 				.gem.obj = obj,
1135 				.gem.offset = offset,
1136 			};
1137 			struct drm_gpuva_op_unmap u = { .va = va };
1138 
1139 			merge &= obj == req_obj &&
1140 				 offset + ls_range == req_offset;
1141 			u.keep = merge;
1142 
1143 			if (end == req_end) {
1144 				ret = op_remap_cb(ops, priv, &p, NULL, &u);
1145 				if (ret)
1146 					return ret;
1147 				break;
1148 			}
1149 
1150 			if (end < req_end) {
1151 				ret = op_remap_cb(ops, priv, &p, NULL, &u);
1152 				if (ret)
1153 					return ret;
1154 				continue;
1155 			}
1156 
1157 			if (end > req_end) {
1158 				struct drm_gpuva_op_map n = {
1159 					.va.addr = req_end,
1160 					.va.range = end - req_end,
1161 					.gem.obj = obj,
1162 					.gem.offset = offset + ls_range +
1163 						      req_range,
1164 				};
1165 
1166 				ret = op_remap_cb(ops, priv, &p, &n, &u);
1167 				if (ret)
1168 					return ret;
1169 				break;
1170 			}
1171 		} else if (addr > req_addr) {
1172 			merge &= obj == req_obj &&
1173 				 offset == req_offset +
1174 					   (addr - req_addr);
1175 
1176 			if (end == req_end) {
1177 				ret = op_unmap_cb(ops, priv, va, merge);
1178 				if (ret)
1179 					return ret;
1180 				break;
1181 			}
1182 
1183 			if (end < req_end) {
1184 				ret = op_unmap_cb(ops, priv, va, merge);
1185 				if (ret)
1186 					return ret;
1187 				continue;
1188 			}
1189 
1190 			if (end > req_end) {
1191 				struct drm_gpuva_op_map n = {
1192 					.va.addr = req_end,
1193 					.va.range = end - req_end,
1194 					.gem.obj = obj,
1195 					.gem.offset = offset + req_end - addr,
1196 				};
1197 				struct drm_gpuva_op_unmap u = {
1198 					.va = va,
1199 					.keep = merge,
1200 				};
1201 
1202 				ret = op_remap_cb(ops, priv, NULL, &n, &u);
1203 				if (ret)
1204 					return ret;
1205 				break;
1206 			}
1207 		}
1208 	}
1209 
1210 	return op_map_cb(ops, priv,
1211 			 req_addr, req_range,
1212 			 req_obj, req_offset);
1213 }
1214 
1215 static int
__drm_gpuva_sm_unmap(struct drm_gpuva_manager * mgr,const struct drm_gpuva_fn_ops * ops,void * priv,u64 req_addr,u64 req_range)1216 __drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr,
1217 		     const struct drm_gpuva_fn_ops *ops, void *priv,
1218 		     u64 req_addr, u64 req_range)
1219 {
1220 	struct drm_gpuva *va, *next;
1221 	u64 req_end = req_addr + req_range;
1222 	int ret;
1223 
1224 	if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range)))
1225 		return -EINVAL;
1226 
1227 	drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) {
1228 		struct drm_gpuva_op_map prev = {}, next = {};
1229 		bool prev_split = false, next_split = false;
1230 		struct drm_gem_object *obj = va->gem.obj;
1231 		u64 offset = va->gem.offset;
1232 		u64 addr = va->va.addr;
1233 		u64 range = va->va.range;
1234 		u64 end = addr + range;
1235 
1236 		if (addr < req_addr) {
1237 			prev.va.addr = addr;
1238 			prev.va.range = req_addr - addr;
1239 			prev.gem.obj = obj;
1240 			prev.gem.offset = offset;
1241 
1242 			prev_split = true;
1243 		}
1244 
1245 		if (end > req_end) {
1246 			next.va.addr = req_end;
1247 			next.va.range = end - req_end;
1248 			next.gem.obj = obj;
1249 			next.gem.offset = offset + (req_end - addr);
1250 
1251 			next_split = true;
1252 		}
1253 
1254 		if (prev_split || next_split) {
1255 			struct drm_gpuva_op_unmap unmap = { .va = va };
1256 
1257 			ret = op_remap_cb(ops, priv,
1258 					  prev_split ? &prev : NULL,
1259 					  next_split ? &next : NULL,
1260 					  &unmap);
1261 			if (ret)
1262 				return ret;
1263 		} else {
1264 			ret = op_unmap_cb(ops, priv, va, false);
1265 			if (ret)
1266 				return ret;
1267 		}
1268 	}
1269 
1270 	return 0;
1271 }
1272 
1273 /**
1274  * drm_gpuva_sm_map() - creates the &drm_gpuva_op split/merge steps
1275  * @mgr: the &drm_gpuva_manager representing the GPU VA space
1276  * @req_addr: the start address of the new mapping
1277  * @req_range: the range of the new mapping
1278  * @req_obj: the &drm_gem_object to map
1279  * @req_offset: the offset within the &drm_gem_object
1280  * @priv: pointer to a driver private data structure
1281  *
1282  * This function iterates the given range of the GPU VA space. It utilizes the
1283  * &drm_gpuva_fn_ops to call back into the driver providing the split and merge
1284  * steps.
1285  *
1286  * Drivers may use these callbacks to update the GPU VA space right away within
1287  * the callback. In case the driver decides to copy and store the operations for
1288  * later processing neither this function nor &drm_gpuva_sm_unmap is allowed to
1289  * be called before the &drm_gpuva_manager's view of the GPU VA space was
1290  * updated with the previous set of operations. To update the
1291  * &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(),
1292  * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
1293  * used.
1294  *
1295  * A sequence of callbacks can contain map, unmap and remap operations, but
1296  * the sequence of callbacks might also be empty if no operation is required,
1297  * e.g. if the requested mapping already exists in the exact same way.
1298  *
1299  * There can be an arbitrary amount of unmap operations, a maximum of two remap
1300  * operations and a single map operation. The latter one represents the original
1301  * map operation requested by the caller.
1302  *
1303  * Returns: 0 on success or a negative error code
1304  */
1305 int
drm_gpuva_sm_map(struct drm_gpuva_manager * mgr,void * priv,u64 req_addr,u64 req_range,struct drm_gem_object * req_obj,u64 req_offset)1306 drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, void *priv,
1307 		 u64 req_addr, u64 req_range,
1308 		 struct drm_gem_object *req_obj, u64 req_offset)
1309 {
1310 	const struct drm_gpuva_fn_ops *ops = mgr->ops;
1311 
1312 	if (unlikely(!(ops && ops->sm_step_map &&
1313 		       ops->sm_step_remap &&
1314 		       ops->sm_step_unmap)))
1315 		return -EINVAL;
1316 
1317 	return __drm_gpuva_sm_map(mgr, ops, priv,
1318 				  req_addr, req_range,
1319 				  req_obj, req_offset);
1320 }
1321 EXPORT_SYMBOL_GPL(drm_gpuva_sm_map);
1322 
1323 /**
1324  * drm_gpuva_sm_unmap() - creates the &drm_gpuva_ops to split on unmap
1325  * @mgr: the &drm_gpuva_manager representing the GPU VA space
1326  * @priv: pointer to a driver private data structure
1327  * @req_addr: the start address of the range to unmap
1328  * @req_range: the range of the mappings to unmap
1329  *
1330  * This function iterates the given range of the GPU VA space. It utilizes the
1331  * &drm_gpuva_fn_ops to call back into the driver providing the operations to
1332  * unmap and, if required, split existent mappings.
1333  *
1334  * Drivers may use these callbacks to update the GPU VA space right away within
1335  * the callback. In case the driver decides to copy and store the operations for
1336  * later processing neither this function nor &drm_gpuva_sm_map is allowed to be
1337  * called before the &drm_gpuva_manager's view of the GPU VA space was updated
1338  * with the previous set of operations. To update the &drm_gpuva_manager's view
1339  * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or
1340  * drm_gpuva_destroy_unlocked() should be used.
1341  *
1342  * A sequence of callbacks can contain unmap and remap operations, depending on
1343  * whether there are actual overlapping mappings to split.
1344  *
1345  * There can be an arbitrary amount of unmap operations and a maximum of two
1346  * remap operations.
1347  *
1348  * Returns: 0 on success or a negative error code
1349  */
1350 int
drm_gpuva_sm_unmap(struct drm_gpuva_manager * mgr,void * priv,u64 req_addr,u64 req_range)1351 drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, void *priv,
1352 		   u64 req_addr, u64 req_range)
1353 {
1354 	const struct drm_gpuva_fn_ops *ops = mgr->ops;
1355 
1356 	if (unlikely(!(ops && ops->sm_step_remap &&
1357 		       ops->sm_step_unmap)))
1358 		return -EINVAL;
1359 
1360 	return __drm_gpuva_sm_unmap(mgr, ops, priv,
1361 				    req_addr, req_range);
1362 }
1363 EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap);
1364 
1365 static struct drm_gpuva_op *
gpuva_op_alloc(struct drm_gpuva_manager * mgr)1366 gpuva_op_alloc(struct drm_gpuva_manager *mgr)
1367 {
1368 	const struct drm_gpuva_fn_ops *fn = mgr->ops;
1369 	struct drm_gpuva_op *op;
1370 
1371 	if (fn && fn->op_alloc)
1372 		op = fn->op_alloc();
1373 	else
1374 		op = kzalloc(sizeof(*op), GFP_KERNEL);
1375 
1376 	if (unlikely(!op))
1377 		return NULL;
1378 
1379 	return op;
1380 }
1381 
1382 static void
gpuva_op_free(struct drm_gpuva_manager * mgr,struct drm_gpuva_op * op)1383 gpuva_op_free(struct drm_gpuva_manager *mgr,
1384 	      struct drm_gpuva_op *op)
1385 {
1386 	const struct drm_gpuva_fn_ops *fn = mgr->ops;
1387 
1388 	if (fn && fn->op_free)
1389 		fn->op_free(op);
1390 	else
1391 		kfree(op);
1392 }
1393 
1394 static int
drm_gpuva_sm_step(struct drm_gpuva_op * __op,void * priv)1395 drm_gpuva_sm_step(struct drm_gpuva_op *__op,
1396 		  void *priv)
1397 {
1398 	struct {
1399 		struct drm_gpuva_manager *mgr;
1400 		struct drm_gpuva_ops *ops;
1401 	} *args = priv;
1402 	struct drm_gpuva_manager *mgr = args->mgr;
1403 	struct drm_gpuva_ops *ops = args->ops;
1404 	struct drm_gpuva_op *op;
1405 
1406 	op = gpuva_op_alloc(mgr);
1407 	if (unlikely(!op))
1408 		goto err;
1409 
1410 	memcpy(op, __op, sizeof(*op));
1411 
1412 	if (op->op == DRM_GPUVA_OP_REMAP) {
1413 		struct drm_gpuva_op_remap *__r = &__op->remap;
1414 		struct drm_gpuva_op_remap *r = &op->remap;
1415 
1416 		r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap),
1417 				   GFP_KERNEL);
1418 		if (unlikely(!r->unmap))
1419 			goto err_free_op;
1420 
1421 		if (__r->prev) {
1422 			r->prev = kmemdup(__r->prev, sizeof(*r->prev),
1423 					  GFP_KERNEL);
1424 			if (unlikely(!r->prev))
1425 				goto err_free_unmap;
1426 		}
1427 
1428 		if (__r->next) {
1429 			r->next = kmemdup(__r->next, sizeof(*r->next),
1430 					  GFP_KERNEL);
1431 			if (unlikely(!r->next))
1432 				goto err_free_prev;
1433 		}
1434 	}
1435 
1436 	list_add_tail(&op->entry, &ops->list);
1437 
1438 	return 0;
1439 
1440 err_free_unmap:
1441 	kfree(op->remap.unmap);
1442 err_free_prev:
1443 	kfree(op->remap.prev);
1444 err_free_op:
1445 	gpuva_op_free(mgr, op);
1446 err:
1447 	return -ENOMEM;
1448 }
1449 
1450 static const struct drm_gpuva_fn_ops gpuva_list_ops = {
1451 	.sm_step_map = drm_gpuva_sm_step,
1452 	.sm_step_remap = drm_gpuva_sm_step,
1453 	.sm_step_unmap = drm_gpuva_sm_step,
1454 };
1455 
1456 /**
1457  * drm_gpuva_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge
1458  * @mgr: the &drm_gpuva_manager representing the GPU VA space
1459  * @req_addr: the start address of the new mapping
1460  * @req_range: the range of the new mapping
1461  * @req_obj: the &drm_gem_object to map
1462  * @req_offset: the offset within the &drm_gem_object
1463  *
1464  * This function creates a list of operations to perform splitting and merging
1465  * of existent mapping(s) with the newly requested one.
1466  *
1467  * The list can be iterated with &drm_gpuva_for_each_op and must be processed
1468  * in the given order. It can contain map, unmap and remap operations, but it
1469  * also can be empty if no operation is required, e.g. if the requested mapping
1470  * already exists is the exact same way.
1471  *
1472  * There can be an arbitrary amount of unmap operations, a maximum of two remap
1473  * operations and a single map operation. The latter one represents the original
1474  * map operation requested by the caller.
1475  *
1476  * Note that before calling this function again with another mapping request it
1477  * is necessary to update the &drm_gpuva_manager's view of the GPU VA space. The
1478  * previously obtained operations must be either processed or abandoned. To
1479  * update the &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(),
1480  * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
1481  * used.
1482  *
1483  * After the caller finished processing the returned &drm_gpuva_ops, they must
1484  * be freed with &drm_gpuva_ops_free.
1485  *
1486  * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
1487  */
1488 struct drm_gpuva_ops *
drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager * mgr,u64 req_addr,u64 req_range,struct drm_gem_object * req_obj,u64 req_offset)1489 drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr,
1490 			    u64 req_addr, u64 req_range,
1491 			    struct drm_gem_object *req_obj, u64 req_offset)
1492 {
1493 	struct drm_gpuva_ops *ops;
1494 	struct {
1495 		struct drm_gpuva_manager *mgr;
1496 		struct drm_gpuva_ops *ops;
1497 	} args;
1498 	int ret;
1499 
1500 	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
1501 	if (unlikely(!ops))
1502 		return ERR_PTR(-ENOMEM);
1503 
1504 	INIT_LIST_HEAD(&ops->list);
1505 
1506 	args.mgr = mgr;
1507 	args.ops = ops;
1508 
1509 	ret = __drm_gpuva_sm_map(mgr, &gpuva_list_ops, &args,
1510 				 req_addr, req_range,
1511 				 req_obj, req_offset);
1512 	if (ret)
1513 		goto err_free_ops;
1514 
1515 	return ops;
1516 
1517 err_free_ops:
1518 	drm_gpuva_ops_free(mgr, ops);
1519 	return ERR_PTR(ret);
1520 }
1521 EXPORT_SYMBOL_GPL(drm_gpuva_sm_map_ops_create);
1522 
1523 /**
1524  * drm_gpuva_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on
1525  * unmap
1526  * @mgr: the &drm_gpuva_manager representing the GPU VA space
1527  * @req_addr: the start address of the range to unmap
1528  * @req_range: the range of the mappings to unmap
1529  *
1530  * This function creates a list of operations to perform unmapping and, if
1531  * required, splitting of the mappings overlapping the unmap range.
1532  *
1533  * The list can be iterated with &drm_gpuva_for_each_op and must be processed
1534  * in the given order. It can contain unmap and remap operations, depending on
1535  * whether there are actual overlapping mappings to split.
1536  *
1537  * There can be an arbitrary amount of unmap operations and a maximum of two
1538  * remap operations.
1539  *
1540  * Note that before calling this function again with another range to unmap it
1541  * is necessary to update the &drm_gpuva_manager's view of the GPU VA space. The
1542  * previously obtained operations must be processed or abandoned. To update the
1543  * &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(),
1544  * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
1545  * used.
1546  *
1547  * After the caller finished processing the returned &drm_gpuva_ops, they must
1548  * be freed with &drm_gpuva_ops_free.
1549  *
1550  * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
1551  */
1552 struct drm_gpuva_ops *
drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager * mgr,u64 req_addr,u64 req_range)1553 drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr,
1554 			      u64 req_addr, u64 req_range)
1555 {
1556 	struct drm_gpuva_ops *ops;
1557 	struct {
1558 		struct drm_gpuva_manager *mgr;
1559 		struct drm_gpuva_ops *ops;
1560 	} args;
1561 	int ret;
1562 
1563 	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
1564 	if (unlikely(!ops))
1565 		return ERR_PTR(-ENOMEM);
1566 
1567 	INIT_LIST_HEAD(&ops->list);
1568 
1569 	args.mgr = mgr;
1570 	args.ops = ops;
1571 
1572 	ret = __drm_gpuva_sm_unmap(mgr, &gpuva_list_ops, &args,
1573 				   req_addr, req_range);
1574 	if (ret)
1575 		goto err_free_ops;
1576 
1577 	return ops;
1578 
1579 err_free_ops:
1580 	drm_gpuva_ops_free(mgr, ops);
1581 	return ERR_PTR(ret);
1582 }
1583 EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap_ops_create);
1584 
1585 /**
1586  * drm_gpuva_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch
1587  * @mgr: the &drm_gpuva_manager representing the GPU VA space
1588  * @addr: the start address of the range to prefetch
1589  * @range: the range of the mappings to prefetch
1590  *
1591  * This function creates a list of operations to perform prefetching.
1592  *
1593  * The list can be iterated with &drm_gpuva_for_each_op and must be processed
1594  * in the given order. It can contain prefetch operations.
1595  *
1596  * There can be an arbitrary amount of prefetch operations.
1597  *
1598  * After the caller finished processing the returned &drm_gpuva_ops, they must
1599  * be freed with &drm_gpuva_ops_free.
1600  *
1601  * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
1602  */
1603 struct drm_gpuva_ops *
drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager * mgr,u64 addr,u64 range)1604 drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr,
1605 			      u64 addr, u64 range)
1606 {
1607 	struct drm_gpuva_ops *ops;
1608 	struct drm_gpuva_op *op;
1609 	struct drm_gpuva *va;
1610 	u64 end = addr + range;
1611 	int ret;
1612 
1613 	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
1614 	if (!ops)
1615 		return ERR_PTR(-ENOMEM);
1616 
1617 	INIT_LIST_HEAD(&ops->list);
1618 
1619 	drm_gpuva_for_each_va_range(va, mgr, addr, end) {
1620 		op = gpuva_op_alloc(mgr);
1621 		if (!op) {
1622 			ret = -ENOMEM;
1623 			goto err_free_ops;
1624 		}
1625 
1626 		op->op = DRM_GPUVA_OP_PREFETCH;
1627 		op->prefetch.va = va;
1628 		list_add_tail(&op->entry, &ops->list);
1629 	}
1630 
1631 	return ops;
1632 
1633 err_free_ops:
1634 	drm_gpuva_ops_free(mgr, ops);
1635 	return ERR_PTR(ret);
1636 }
1637 EXPORT_SYMBOL_GPL(drm_gpuva_prefetch_ops_create);
1638 
1639 /**
1640  * drm_gpuva_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
1641  * @mgr: the &drm_gpuva_manager representing the GPU VA space
1642  * @obj: the &drm_gem_object to unmap
1643  *
1644  * This function creates a list of operations to perform unmapping for every
1645  * GPUVA attached to a GEM.
1646  *
1647  * The list can be iterated with &drm_gpuva_for_each_op and consists out of an
1648  * arbitrary amount of unmap operations.
1649  *
1650  * After the caller finished processing the returned &drm_gpuva_ops, they must
1651  * be freed with &drm_gpuva_ops_free.
1652  *
1653  * It is the callers responsibility to protect the GEMs GPUVA list against
1654  * concurrent access using the GEMs dma_resv lock.
1655  *
1656  * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
1657  */
1658 struct drm_gpuva_ops *
drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager * mgr,struct drm_gem_object * obj)1659 drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr,
1660 			       struct drm_gem_object *obj)
1661 {
1662 	struct drm_gpuva_ops *ops;
1663 	struct drm_gpuva_op *op;
1664 	struct drm_gpuva *va;
1665 	int ret;
1666 
1667 	drm_gem_gpuva_assert_lock_held(obj);
1668 
1669 	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
1670 	if (!ops)
1671 		return ERR_PTR(-ENOMEM);
1672 
1673 	INIT_LIST_HEAD(&ops->list);
1674 
1675 	drm_gem_for_each_gpuva(va, obj) {
1676 		op = gpuva_op_alloc(mgr);
1677 		if (!op) {
1678 			ret = -ENOMEM;
1679 			goto err_free_ops;
1680 		}
1681 
1682 		op->op = DRM_GPUVA_OP_UNMAP;
1683 		op->unmap.va = va;
1684 		list_add_tail(&op->entry, &ops->list);
1685 	}
1686 
1687 	return ops;
1688 
1689 err_free_ops:
1690 	drm_gpuva_ops_free(mgr, ops);
1691 	return ERR_PTR(ret);
1692 }
1693 EXPORT_SYMBOL_GPL(drm_gpuva_gem_unmap_ops_create);
1694 
1695 /**
1696  * drm_gpuva_ops_free() - free the given &drm_gpuva_ops
1697  * @mgr: the &drm_gpuva_manager the ops were created for
1698  * @ops: the &drm_gpuva_ops to free
1699  *
1700  * Frees the given &drm_gpuva_ops structure including all the ops associated
1701  * with it.
1702  */
1703 void
drm_gpuva_ops_free(struct drm_gpuva_manager * mgr,struct drm_gpuva_ops * ops)1704 drm_gpuva_ops_free(struct drm_gpuva_manager *mgr,
1705 		   struct drm_gpuva_ops *ops)
1706 {
1707 	struct drm_gpuva_op *op, *next;
1708 
1709 	drm_gpuva_for_each_op_safe(op, next, ops) {
1710 		list_del(&op->entry);
1711 
1712 		if (op->op == DRM_GPUVA_OP_REMAP) {
1713 			kfree(op->remap.prev);
1714 			kfree(op->remap.next);
1715 			kfree(op->remap.unmap);
1716 		}
1717 
1718 		gpuva_op_free(mgr, op);
1719 	}
1720 
1721 	kfree(ops);
1722 }
1723 EXPORT_SYMBOL_GPL(drm_gpuva_ops_free);
1724