1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020-2021 Intel Corporation
4  */
5 
6 #include "gt/intel_migrate.h"
7 #include "gt/intel_gpu_commands.h"
8 #include "gem/i915_gem_ttm_move.h"
9 
10 #include "i915_deps.h"
11 
12 #include "selftests/igt_reset.h"
13 #include "selftests/igt_spinner.h"
14 
igt_fill_check_buffer(struct drm_i915_gem_object * obj,bool fill)15 static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,
16 				 bool fill)
17 {
18 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
19 	unsigned int i, count = obj->base.size / sizeof(u32);
20 	enum i915_map_type map_type =
21 		i915_coherent_map_type(i915, obj, false);
22 	u32 *cur;
23 	int err = 0;
24 
25 	assert_object_held(obj);
26 	cur = i915_gem_object_pin_map(obj, map_type);
27 	if (IS_ERR(cur))
28 		return PTR_ERR(cur);
29 
30 	if (fill)
31 		for (i = 0; i < count; ++i)
32 			*cur++ = i;
33 	else
34 		for (i = 0; i < count; ++i)
35 			if (*cur++ != i) {
36 				pr_err("Object content mismatch at location %d of %d\n", i, count);
37 				err = -EINVAL;
38 				break;
39 			}
40 
41 	i915_gem_object_unpin_map(obj);
42 
43 	return err;
44 }
45 
igt_create_migrate(struct intel_gt * gt,enum intel_region_id src,enum intel_region_id dst)46 static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,
47 			      enum intel_region_id dst)
48 {
49 	struct drm_i915_private *i915 = gt->i915;
50 	struct intel_memory_region *src_mr = i915->mm.regions[src];
51 	struct intel_memory_region *dst_mr = i915->mm.regions[dst];
52 	struct drm_i915_gem_object *obj;
53 	struct i915_gem_ww_ctx ww;
54 	int err = 0;
55 
56 	GEM_BUG_ON(!src_mr);
57 	GEM_BUG_ON(!dst_mr);
58 
59 	/* Switch object backing-store on create */
60 	obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0);
61 	if (IS_ERR(obj))
62 		return PTR_ERR(obj);
63 
64 	for_i915_gem_ww(&ww, err, true) {
65 		err = i915_gem_object_lock(obj, &ww);
66 		if (err)
67 			continue;
68 
69 		err = igt_fill_check_buffer(obj, true);
70 		if (err)
71 			continue;
72 
73 		err = i915_gem_object_migrate(obj, &ww, dst);
74 		if (err)
75 			continue;
76 
77 		err = i915_gem_object_pin_pages(obj);
78 		if (err)
79 			continue;
80 
81 		if (i915_gem_object_can_migrate(obj, src))
82 			err = -EINVAL;
83 
84 		i915_gem_object_unpin_pages(obj);
85 		err = i915_gem_object_wait_migration(obj, true);
86 		if (err)
87 			continue;
88 
89 		err = igt_fill_check_buffer(obj, false);
90 	}
91 	i915_gem_object_put(obj);
92 
93 	return err;
94 }
95 
igt_smem_create_migrate(void * arg)96 static int igt_smem_create_migrate(void *arg)
97 {
98 	return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM);
99 }
100 
igt_lmem_create_migrate(void * arg)101 static int igt_lmem_create_migrate(void *arg)
102 {
103 	return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0);
104 }
105 
igt_same_create_migrate(void * arg)106 static int igt_same_create_migrate(void *arg)
107 {
108 	return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0);
109 }
110 
lmem_pages_migrate_one(struct i915_gem_ww_ctx * ww,struct drm_i915_gem_object * obj,struct i915_vma * vma,bool silent_migrate)111 static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,
112 				  struct drm_i915_gem_object *obj,
113 				  struct i915_vma *vma,
114 				  bool silent_migrate)
115 {
116 	int err;
117 
118 	err = i915_gem_object_lock(obj, ww);
119 	if (err)
120 		return err;
121 
122 	if (vma) {
123 		err = i915_vma_pin_ww(vma, ww, obj->base.size, 0,
124 				      0UL | PIN_OFFSET_FIXED |
125 				      PIN_USER);
126 		if (err) {
127 			if (err != -EINTR && err != ERESTARTSYS &&
128 			    err != -EDEADLK)
129 				pr_err("Failed to pin vma.\n");
130 			return err;
131 		}
132 
133 		i915_vma_unpin(vma);
134 	}
135 
136 	/*
137 	 * Migration will implicitly unbind (asynchronously) any bound
138 	 * vmas.
139 	 */
140 	if (i915_gem_object_is_lmem(obj)) {
141 		err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM);
142 		if (err) {
143 			if (!silent_migrate)
144 				pr_err("Object failed migration to smem\n");
145 			if (err)
146 				return err;
147 		}
148 
149 		if (i915_gem_object_is_lmem(obj)) {
150 			pr_err("object still backed by lmem\n");
151 			err = -EINVAL;
152 		}
153 
154 		if (!i915_gem_object_has_struct_page(obj)) {
155 			pr_err("object not backed by struct page\n");
156 			err = -EINVAL;
157 		}
158 
159 	} else {
160 		err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0);
161 		if (err) {
162 			if (!silent_migrate)
163 				pr_err("Object failed migration to lmem\n");
164 			if (err)
165 				return err;
166 		}
167 
168 		if (i915_gem_object_has_struct_page(obj)) {
169 			pr_err("object still backed by struct page\n");
170 			err = -EINVAL;
171 		}
172 
173 		if (!i915_gem_object_is_lmem(obj)) {
174 			pr_err("object not backed by lmem\n");
175 			err = -EINVAL;
176 		}
177 	}
178 
179 	return err;
180 }
181 
__igt_lmem_pages_migrate(struct intel_gt * gt,struct i915_address_space * vm,struct i915_deps * deps,struct igt_spinner * spin,struct dma_fence * spin_fence,bool borked_migrate)182 static int __igt_lmem_pages_migrate(struct intel_gt *gt,
183 				    struct i915_address_space *vm,
184 				    struct i915_deps *deps,
185 				    struct igt_spinner *spin,
186 				    struct dma_fence *spin_fence,
187 				    bool borked_migrate)
188 {
189 	struct drm_i915_private *i915 = gt->i915;
190 	struct drm_i915_gem_object *obj;
191 	struct i915_vma *vma = NULL;
192 	struct i915_gem_ww_ctx ww;
193 	struct i915_request *rq;
194 	int err;
195 	int i;
196 
197 	/* From LMEM to shmem and back again */
198 
199 	obj = i915_gem_object_create_lmem(i915, SZ_2M, 0);
200 	if (IS_ERR(obj))
201 		return PTR_ERR(obj);
202 
203 	if (vm) {
204 		vma = i915_vma_instance(obj, vm, NULL);
205 		if (IS_ERR(vma)) {
206 			err = PTR_ERR(vma);
207 			goto out_put;
208 		}
209 	}
210 
211 	/* Initial GPU fill, sync, CPU initialization. */
212 	for_i915_gem_ww(&ww, err, true) {
213 		err = i915_gem_object_lock(obj, &ww);
214 		if (err)
215 			continue;
216 
217 		err = ____i915_gem_object_get_pages(obj);
218 		if (err)
219 			continue;
220 
221 		err = intel_migrate_clear(&gt->migrate, &ww, deps,
222 					  obj->mm.pages->sgl, obj->cache_level,
223 					  i915_gem_object_is_lmem(obj),
224 					  0xdeadbeaf, &rq);
225 		if (rq) {
226 			err = dma_resv_reserve_fences(obj->base.resv, 1);
227 			if (!err)
228 				dma_resv_add_fence(obj->base.resv, &rq->fence,
229 						   DMA_RESV_USAGE_KERNEL);
230 			i915_request_put(rq);
231 		}
232 		if (err)
233 			continue;
234 
235 		if (!vma) {
236 			err = igt_fill_check_buffer(obj, true);
237 			if (err)
238 				continue;
239 		}
240 	}
241 	if (err)
242 		goto out_put;
243 
244 	/*
245 	 * Migrate to and from smem without explicitly syncing.
246 	 * Finalize with data in smem for fast readout.
247 	 */
248 	for (i = 1; i <= 5; ++i) {
249 		for_i915_gem_ww(&ww, err, true)
250 			err = lmem_pages_migrate_one(&ww, obj, vma,
251 						     borked_migrate);
252 		if (err)
253 			goto out_put;
254 	}
255 
256 	err = i915_gem_object_lock_interruptible(obj, NULL);
257 	if (err)
258 		goto out_put;
259 
260 	if (spin) {
261 		if (dma_fence_is_signaled(spin_fence)) {
262 			pr_err("Spinner was terminated by hangcheck.\n");
263 			err = -EBUSY;
264 			goto out_unlock;
265 		}
266 		igt_spinner_end(spin);
267 	}
268 
269 	/* Finally sync migration and check content. */
270 	err = i915_gem_object_wait_migration(obj, true);
271 	if (err)
272 		goto out_unlock;
273 
274 	if (vma) {
275 		err = i915_vma_wait_for_bind(vma);
276 		if (err)
277 			goto out_unlock;
278 	} else {
279 		err = igt_fill_check_buffer(obj, false);
280 	}
281 
282 out_unlock:
283 	i915_gem_object_unlock(obj);
284 out_put:
285 	i915_gem_object_put(obj);
286 
287 	return err;
288 }
289 
igt_lmem_pages_failsafe_migrate(void * arg)290 static int igt_lmem_pages_failsafe_migrate(void *arg)
291 {
292 	int fail_gpu, fail_alloc, ban_memcpy, ret;
293 	struct intel_gt *gt = arg;
294 
295 	for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
296 		for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {
297 			for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
298 				pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n",
299 					fail_gpu, fail_alloc, ban_memcpy);
300 				i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
301 				i915_ttm_migrate_set_failure_modes(fail_gpu,
302 								   fail_alloc);
303 				ret = __igt_lmem_pages_migrate(gt, NULL, NULL,
304 							       NULL, NULL,
305 							       ban_memcpy &&
306 							       fail_gpu);
307 
308 				if (ban_memcpy && fail_gpu) {
309 					struct intel_gt *__gt;
310 					unsigned int id;
311 
312 					if (ret != -EIO) {
313 						pr_err("expected -EIO, got (%d)\n", ret);
314 						ret = -EINVAL;
315 					} else {
316 						ret = 0;
317 					}
318 
319 					for_each_gt(__gt, gt->i915, id) {
320 						intel_wakeref_t wakeref;
321 						bool wedged;
322 
323 						mutex_lock(&__gt->reset.mutex);
324 						wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
325 						mutex_unlock(&__gt->reset.mutex);
326 
327 						if (fail_gpu && !fail_alloc) {
328 							if (!wedged) {
329 								pr_err("gt(%u) not wedged\n", id);
330 								ret = -EINVAL;
331 								continue;
332 							}
333 						} else if (wedged) {
334 							pr_err("gt(%u) incorrectly wedged\n", id);
335 							ret = -EINVAL;
336 						} else {
337 							continue;
338 						}
339 
340 						wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
341 						igt_global_reset_lock(__gt);
342 						intel_gt_reset(__gt, ALL_ENGINES, NULL);
343 						igt_global_reset_unlock(__gt);
344 						intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
345 					}
346 					if (ret)
347 						goto out_err;
348 				}
349 			}
350 		}
351 	}
352 
353 out_err:
354 	i915_ttm_migrate_set_failure_modes(false, false);
355 	i915_ttm_migrate_set_ban_memcpy(false);
356 	return ret;
357 }
358 
359 /*
360  * This subtest tests that unbinding at migration is indeed performed
361  * async. We launch a spinner and a number of migrations depending on
362  * that spinner to have terminated. Before each migration we bind a
363  * vma, which should then be async unbound by the migration operation.
364  * If we are able to schedule migrations without blocking while the
365  * spinner is still running, those unbinds are indeed async and non-
366  * blocking.
367  *
368  * Note that each async bind operation is awaiting the previous migration
369  * due to the moving fence resulting from the migration.
370  */
igt_async_migrate(struct intel_gt * gt)371 static int igt_async_migrate(struct intel_gt *gt)
372 {
373 	struct intel_engine_cs *engine;
374 	enum intel_engine_id id;
375 	struct i915_ppgtt *ppgtt;
376 	struct igt_spinner spin;
377 	int err;
378 
379 	ppgtt = i915_ppgtt_create(gt, 0);
380 	if (IS_ERR(ppgtt))
381 		return PTR_ERR(ppgtt);
382 
383 	if (igt_spinner_init(&spin, gt)) {
384 		err = -ENOMEM;
385 		goto out_spin;
386 	}
387 
388 	for_each_engine(engine, gt, id) {
389 		struct ttm_operation_ctx ctx = {
390 			.interruptible = true
391 		};
392 		struct dma_fence *spin_fence;
393 		struct intel_context *ce;
394 		struct i915_request *rq;
395 		struct i915_deps deps;
396 
397 		ce = intel_context_create(engine);
398 		if (IS_ERR(ce)) {
399 			err = PTR_ERR(ce);
400 			goto out_ce;
401 		}
402 
403 		/*
404 		 * Use MI_NOOP, making the spinner non-preemptible. If there
405 		 * is a code path where we fail async operation due to the
406 		 * running spinner, we will block and fail to end the
407 		 * spinner resulting in a deadlock. But with a non-
408 		 * preemptible spinner, hangcheck will terminate the spinner
409 		 * for us, and we will later detect that and fail the test.
410 		 */
411 		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
412 		intel_context_put(ce);
413 		if (IS_ERR(rq)) {
414 			err = PTR_ERR(rq);
415 			goto out_ce;
416 		}
417 
418 		i915_deps_init(&deps, GFP_KERNEL);
419 		err = i915_deps_add_dependency(&deps, &rq->fence, &ctx);
420 		spin_fence = dma_fence_get(&rq->fence);
421 		i915_request_add(rq);
422 		if (err)
423 			goto out_ce;
424 
425 		err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin,
426 					       spin_fence, false);
427 		i915_deps_fini(&deps);
428 		dma_fence_put(spin_fence);
429 		if (err)
430 			goto out_ce;
431 	}
432 
433 out_ce:
434 	igt_spinner_fini(&spin);
435 out_spin:
436 	i915_vm_put(&ppgtt->vm);
437 
438 	return err;
439 }
440 
441 /*
442  * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while
443  * arming the migration error check and block async migration. This
444  * will cause us to deadlock and hangcheck will terminate the spinner
445  * causing the test to fail.
446  */
447 #define ASYNC_FAIL_ALLOC 1
igt_lmem_async_migrate(void * arg)448 static int igt_lmem_async_migrate(void *arg)
449 {
450 	int fail_gpu, fail_alloc, ban_memcpy, ret;
451 	struct intel_gt *gt = arg;
452 
453 	for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
454 		for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {
455 			for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
456 				pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n",
457 					fail_gpu, fail_alloc, ban_memcpy);
458 				i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
459 				i915_ttm_migrate_set_failure_modes(fail_gpu,
460 								   fail_alloc);
461 				ret = igt_async_migrate(gt);
462 
463 				if (fail_gpu && ban_memcpy) {
464 					struct intel_gt *__gt;
465 					unsigned int id;
466 
467 					if (ret != -EIO) {
468 						pr_err("expected -EIO, got (%d)\n", ret);
469 						ret = -EINVAL;
470 					} else {
471 						ret = 0;
472 					}
473 
474 					for_each_gt(__gt, gt->i915, id) {
475 						intel_wakeref_t wakeref;
476 						bool wedged;
477 
478 						mutex_lock(&__gt->reset.mutex);
479 						wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
480 						mutex_unlock(&__gt->reset.mutex);
481 
482 						if (fail_gpu && !fail_alloc) {
483 							if (!wedged) {
484 								pr_err("gt(%u) not wedged\n", id);
485 								ret = -EINVAL;
486 								continue;
487 							}
488 						} else if (wedged) {
489 							pr_err("gt(%u) incorrectly wedged\n", id);
490 							ret = -EINVAL;
491 						} else {
492 							continue;
493 						}
494 
495 						wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
496 						igt_global_reset_lock(__gt);
497 						intel_gt_reset(__gt, ALL_ENGINES, NULL);
498 						igt_global_reset_unlock(__gt);
499 						intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
500 					}
501 				}
502 				if (ret)
503 					goto out_err;
504 			}
505 		}
506 	}
507 
508 out_err:
509 	i915_ttm_migrate_set_failure_modes(false, false);
510 	i915_ttm_migrate_set_ban_memcpy(false);
511 	return ret;
512 }
513 
i915_gem_migrate_live_selftests(struct drm_i915_private * i915)514 int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)
515 {
516 	static const struct i915_subtest tests[] = {
517 		SUBTEST(igt_smem_create_migrate),
518 		SUBTEST(igt_lmem_create_migrate),
519 		SUBTEST(igt_same_create_migrate),
520 		SUBTEST(igt_lmem_pages_failsafe_migrate),
521 		SUBTEST(igt_lmem_async_migrate),
522 	};
523 
524 	if (!HAS_LMEM(i915))
525 		return 0;
526 
527 	return intel_gt_live_subtests(tests, to_gt(i915));
528 }
529