1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6 #include "gt/intel_gpu_commands.h"
7 #include "gt/intel_gt.h"
8
9 #include "gem/i915_gem_internal.h"
10 #include "gem/selftests/igt_gem_utils.h"
11
12 #include "igt_spinner.h"
13
igt_spinner_init(struct igt_spinner * spin,struct intel_gt * gt)14 int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt)
15 {
16 int err;
17
18 memset(spin, 0, sizeof(*spin));
19 spin->gt = gt;
20
21 spin->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
22 if (IS_ERR(spin->hws)) {
23 err = PTR_ERR(spin->hws);
24 goto err;
25 }
26 i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
27
28 spin->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
29 if (IS_ERR(spin->obj)) {
30 err = PTR_ERR(spin->obj);
31 goto err_hws;
32 }
33
34 return 0;
35
36 err_hws:
37 i915_gem_object_put(spin->hws);
38 err:
39 return err;
40 }
41
igt_spinner_pin_obj(struct intel_context * ce,struct i915_gem_ww_ctx * ww,struct drm_i915_gem_object * obj,unsigned int mode,struct i915_vma ** vma)42 static void *igt_spinner_pin_obj(struct intel_context *ce,
43 struct i915_gem_ww_ctx *ww,
44 struct drm_i915_gem_object *obj,
45 unsigned int mode, struct i915_vma **vma)
46 {
47 void *vaddr;
48 int ret;
49
50 *vma = i915_vma_instance(obj, ce->vm, NULL);
51 if (IS_ERR(*vma))
52 return ERR_CAST(*vma);
53
54 ret = i915_gem_object_lock(obj, ww);
55 if (ret)
56 return ERR_PTR(ret);
57
58 vaddr = i915_gem_object_pin_map(obj, mode);
59
60 if (!ww)
61 i915_gem_object_unlock(obj);
62
63 if (IS_ERR(vaddr))
64 return vaddr;
65
66 if (ww)
67 ret = i915_vma_pin_ww(*vma, ww, 0, 0, PIN_USER);
68 else
69 ret = i915_vma_pin(*vma, 0, 0, PIN_USER);
70
71 if (ret) {
72 i915_gem_object_unpin_map(obj);
73 return ERR_PTR(ret);
74 }
75
76 return vaddr;
77 }
78
igt_spinner_pin(struct igt_spinner * spin,struct intel_context * ce,struct i915_gem_ww_ctx * ww)79 int igt_spinner_pin(struct igt_spinner *spin,
80 struct intel_context *ce,
81 struct i915_gem_ww_ctx *ww)
82 {
83 void *vaddr;
84
85 if (spin->ce && WARN_ON(spin->ce != ce))
86 return -ENODEV;
87 spin->ce = ce;
88
89 if (!spin->seqno) {
90 vaddr = igt_spinner_pin_obj(ce, ww, spin->hws, I915_MAP_WB, &spin->hws_vma);
91 if (IS_ERR(vaddr))
92 return PTR_ERR(vaddr);
93
94 spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
95 }
96
97 if (!spin->batch) {
98 unsigned int mode;
99
100 mode = i915_coherent_map_type(spin->gt->i915, spin->obj, false);
101 vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma);
102 if (IS_ERR(vaddr))
103 return PTR_ERR(vaddr);
104
105 spin->batch = vaddr;
106 }
107
108 return 0;
109 }
110
seqno_offset(u64 fence)111 static unsigned int seqno_offset(u64 fence)
112 {
113 return offset_in_page(sizeof(u32) * fence);
114 }
115
hws_address(const struct i915_vma * hws,const struct i915_request * rq)116 static u64 hws_address(const struct i915_vma *hws,
117 const struct i915_request *rq)
118 {
119 return hws->node.start + seqno_offset(rq->fence.context);
120 }
121
move_to_active(struct i915_vma * vma,struct i915_request * rq,unsigned int flags)122 static int move_to_active(struct i915_vma *vma,
123 struct i915_request *rq,
124 unsigned int flags)
125 {
126 int err;
127
128 i915_vma_lock(vma);
129 err = i915_request_await_object(rq, vma->obj,
130 flags & EXEC_OBJECT_WRITE);
131 if (err == 0)
132 err = i915_vma_move_to_active(vma, rq, flags);
133 i915_vma_unlock(vma);
134
135 return err;
136 }
137
138 struct i915_request *
igt_spinner_create_request(struct igt_spinner * spin,struct intel_context * ce,u32 arbitration_command)139 igt_spinner_create_request(struct igt_spinner *spin,
140 struct intel_context *ce,
141 u32 arbitration_command)
142 {
143 struct intel_engine_cs *engine = ce->engine;
144 struct i915_request *rq = NULL;
145 struct i915_vma *hws, *vma;
146 unsigned int flags;
147 u32 *batch;
148 int err;
149
150 GEM_BUG_ON(spin->gt != ce->vm->gt);
151
152 if (!intel_engine_can_store_dword(ce->engine))
153 return ERR_PTR(-ENODEV);
154
155 if (!spin->batch) {
156 err = igt_spinner_pin(spin, ce, NULL);
157 if (err)
158 return ERR_PTR(err);
159 }
160
161 hws = spin->hws_vma;
162 vma = spin->batch_vma;
163
164 rq = intel_context_create_request(ce);
165 if (IS_ERR(rq))
166 return ERR_CAST(rq);
167
168 err = move_to_active(vma, rq, 0);
169 if (err)
170 goto cancel_rq;
171
172 err = move_to_active(hws, rq, 0);
173 if (err)
174 goto cancel_rq;
175
176 batch = spin->batch;
177
178 if (GRAPHICS_VER(rq->engine->i915) >= 8) {
179 *batch++ = MI_STORE_DWORD_IMM_GEN4;
180 *batch++ = lower_32_bits(hws_address(hws, rq));
181 *batch++ = upper_32_bits(hws_address(hws, rq));
182 } else if (GRAPHICS_VER(rq->engine->i915) >= 6) {
183 *batch++ = MI_STORE_DWORD_IMM_GEN4;
184 *batch++ = 0;
185 *batch++ = hws_address(hws, rq);
186 } else if (GRAPHICS_VER(rq->engine->i915) >= 4) {
187 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
188 *batch++ = 0;
189 *batch++ = hws_address(hws, rq);
190 } else {
191 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
192 *batch++ = hws_address(hws, rq);
193 }
194 *batch++ = rq->fence.seqno;
195
196 *batch++ = arbitration_command;
197
198 if (GRAPHICS_VER(rq->engine->i915) >= 8)
199 *batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
200 else if (IS_HASWELL(rq->engine->i915))
201 *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW;
202 else if (GRAPHICS_VER(rq->engine->i915) >= 6)
203 *batch++ = MI_BATCH_BUFFER_START;
204 else
205 *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
206 *batch++ = lower_32_bits(vma->node.start);
207 *batch++ = upper_32_bits(vma->node.start);
208
209 *batch++ = MI_BATCH_BUFFER_END; /* not reached */
210
211 intel_gt_chipset_flush(engine->gt);
212
213 if (engine->emit_init_breadcrumb) {
214 err = engine->emit_init_breadcrumb(rq);
215 if (err)
216 goto cancel_rq;
217 }
218
219 flags = 0;
220 if (GRAPHICS_VER(rq->engine->i915) <= 5)
221 flags |= I915_DISPATCH_SECURE;
222 err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
223
224 cancel_rq:
225 if (err) {
226 i915_request_set_error_once(rq, err);
227 i915_request_add(rq);
228 }
229 return err ? ERR_PTR(err) : rq;
230 }
231
232 static u32
hws_seqno(const struct igt_spinner * spin,const struct i915_request * rq)233 hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq)
234 {
235 u32 *seqno = spin->seqno + seqno_offset(rq->fence.context);
236
237 return READ_ONCE(*seqno);
238 }
239
igt_spinner_end(struct igt_spinner * spin)240 void igt_spinner_end(struct igt_spinner *spin)
241 {
242 if (!spin->batch)
243 return;
244
245 *spin->batch = MI_BATCH_BUFFER_END;
246 intel_gt_chipset_flush(spin->gt);
247 }
248
igt_spinner_fini(struct igt_spinner * spin)249 void igt_spinner_fini(struct igt_spinner *spin)
250 {
251 igt_spinner_end(spin);
252
253 if (spin->batch) {
254 i915_vma_unpin(spin->batch_vma);
255 i915_gem_object_unpin_map(spin->obj);
256 }
257 i915_gem_object_put(spin->obj);
258
259 if (spin->seqno) {
260 i915_vma_unpin(spin->hws_vma);
261 i915_gem_object_unpin_map(spin->hws);
262 }
263 i915_gem_object_put(spin->hws);
264 }
265
igt_wait_for_spinner(struct igt_spinner * spin,struct i915_request * rq)266 bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq)
267 {
268 if (i915_request_is_ready(rq))
269 intel_engine_flush_submission(rq->engine);
270
271 return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq),
272 rq->fence.seqno),
273 100) &&
274 wait_for(i915_seqno_passed(hws_seqno(spin, rq),
275 rq->fence.seqno),
276 50));
277 }
278