1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * dma-fence-util: misc functions for dma_fence objects
4 *
5 * Copyright (C) 2022 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
8 */
9
10 #include <linux/dma-fence.h>
11 #include <linux/dma-fence-array.h>
12 #include <linux/dma-fence-chain.h>
13 #include <linux/dma-fence-unwrap.h>
14 #include <linux/slab.h>
15
16 /* Internal helper to start new array iteration, don't use directly */
17 static struct dma_fence *
__dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)18 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
19 {
20 cursor->array = dma_fence_chain_contained(cursor->chain);
21 cursor->index = 0;
22 return dma_fence_array_first(cursor->array);
23 }
24
25 /**
26 * dma_fence_unwrap_first - return the first fence from fence containers
27 * @head: the entrypoint into the containers
28 * @cursor: current position inside the containers
29 *
30 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
31 * first fence.
32 */
dma_fence_unwrap_first(struct dma_fence * head,struct dma_fence_unwrap * cursor)33 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
34 struct dma_fence_unwrap *cursor)
35 {
36 cursor->chain = dma_fence_get(head);
37 return __dma_fence_unwrap_array(cursor);
38 }
39 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
40
41 /**
42 * dma_fence_unwrap_next - return the next fence from a fence containers
43 * @cursor: current position inside the containers
44 *
45 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
46 * the next fence from them.
47 */
dma_fence_unwrap_next(struct dma_fence_unwrap * cursor)48 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
49 {
50 struct dma_fence *tmp;
51
52 ++cursor->index;
53 tmp = dma_fence_array_next(cursor->array, cursor->index);
54 if (tmp)
55 return tmp;
56
57 cursor->chain = dma_fence_chain_walk(cursor->chain);
58 return __dma_fence_unwrap_array(cursor);
59 }
60 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
61
62 /* Implementation for the dma_fence_merge() marco, don't use directly */
__dma_fence_unwrap_merge(unsigned int num_fences,struct dma_fence ** fences,struct dma_fence_unwrap * iter)63 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
64 struct dma_fence **fences,
65 struct dma_fence_unwrap *iter)
66 {
67 struct dma_fence_array *result;
68 struct dma_fence *tmp, **array;
69 unsigned int i;
70 size_t count;
71
72 count = 0;
73 for (i = 0; i < num_fences; ++i) {
74 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i])
75 if (!dma_fence_is_signaled(tmp))
76 ++count;
77 }
78
79 if (count == 0)
80 return dma_fence_get_stub();
81
82 array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
83 if (!array)
84 return NULL;
85
86 /*
87 * This trashes the input fence array and uses it as position for the
88 * following merge loop. This works because the dma_fence_merge()
89 * wrapper macro is creating this temporary array on the stack together
90 * with the iterators.
91 */
92 for (i = 0; i < num_fences; ++i)
93 fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]);
94
95 count = 0;
96 do {
97 unsigned int sel;
98
99 restart:
100 tmp = NULL;
101 for (i = 0; i < num_fences; ++i) {
102 struct dma_fence *next;
103
104 while (fences[i] && dma_fence_is_signaled(fences[i]))
105 fences[i] = dma_fence_unwrap_next(&iter[i]);
106
107 next = fences[i];
108 if (!next)
109 continue;
110
111 /*
112 * We can't guarantee that inpute fences are ordered by
113 * context, but it is still quite likely when this
114 * function is used multiple times. So attempt to order
115 * the fences by context as we pass over them and merge
116 * fences with the same context.
117 */
118 if (!tmp || tmp->context > next->context) {
119 tmp = next;
120 sel = i;
121
122 } else if (tmp->context < next->context) {
123 continue;
124
125 } else if (dma_fence_is_later(tmp, next)) {
126 fences[i] = dma_fence_unwrap_next(&iter[i]);
127 goto restart;
128 } else {
129 fences[sel] = dma_fence_unwrap_next(&iter[sel]);
130 goto restart;
131 }
132 }
133
134 if (tmp) {
135 array[count++] = dma_fence_get(tmp);
136 fences[sel] = dma_fence_unwrap_next(&iter[sel]);
137 }
138 } while (tmp);
139
140 if (count == 0) {
141 tmp = dma_fence_get_stub();
142 goto return_tmp;
143 }
144
145 if (count == 1) {
146 tmp = array[0];
147 goto return_tmp;
148 }
149
150 result = dma_fence_array_create(count, array,
151 dma_fence_context_alloc(1),
152 1, false);
153 if (!result) {
154 tmp = NULL;
155 goto return_tmp;
156 }
157 return &result->base;
158
159 return_tmp:
160 kfree(array);
161 return tmp;
162 }
163 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);
164