1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include "i915_selftest.h"
7
8 #include "gt/intel_context.h"
9 #include "gt/intel_engine_regs.h"
10 #include "gt/intel_engine_user.h"
11 #include "gt/intel_gpu_commands.h"
12 #include "gt/intel_gt.h"
13 #include "gt/intel_gt_regs.h"
14 #include "gem/i915_gem_lmem.h"
15
16 #include "selftests/igt_flush_test.h"
17 #include "selftests/mock_drm.h"
18 #include "selftests/i915_random.h"
19 #include "huge_gem_object.h"
20 #include "mock_context.h"
21
22 #define OW_SIZE 16 /* in bytes */
23 #define F_SUBTILE_SIZE 64 /* in bytes */
24 #define F_TILE_WIDTH 128 /* in bytes */
25 #define F_TILE_HEIGHT 32 /* in pixels */
26 #define F_SUBTILE_WIDTH OW_SIZE /* in bytes */
27 #define F_SUBTILE_HEIGHT 4 /* in pixels */
28
linear_x_y_to_ftiled_pos(int x,int y,u32 stride,int bpp)29 static int linear_x_y_to_ftiled_pos(int x, int y, u32 stride, int bpp)
30 {
31 int tile_base;
32 int tile_x, tile_y;
33 int swizzle, subtile;
34 int pixel_size = bpp / 8;
35 int pos;
36
37 /*
38 * Subtile remapping for F tile. Note that map[a]==b implies map[b]==a
39 * so we can use the same table to tile and until.
40 */
41 static const u8 f_subtile_map[] = {
42 0, 1, 2, 3, 8, 9, 10, 11,
43 4, 5, 6, 7, 12, 13, 14, 15,
44 16, 17, 18, 19, 24, 25, 26, 27,
45 20, 21, 22, 23, 28, 29, 30, 31,
46 32, 33, 34, 35, 40, 41, 42, 43,
47 36, 37, 38, 39, 44, 45, 46, 47,
48 48, 49, 50, 51, 56, 57, 58, 59,
49 52, 53, 54, 55, 60, 61, 62, 63
50 };
51
52 x *= pixel_size;
53 /*
54 * Where does the 4k tile start (in bytes)? This is the same for Y and
55 * F so we can use the Y-tile algorithm to get to that point.
56 */
57 tile_base =
58 y / F_TILE_HEIGHT * stride * F_TILE_HEIGHT +
59 x / F_TILE_WIDTH * 4096;
60
61 /* Find pixel within tile */
62 tile_x = x % F_TILE_WIDTH;
63 tile_y = y % F_TILE_HEIGHT;
64
65 /* And figure out the subtile within the 4k tile */
66 subtile = tile_y / F_SUBTILE_HEIGHT * 8 + tile_x / F_SUBTILE_WIDTH;
67
68 /* Swizzle the subtile number according to the bspec diagram */
69 swizzle = f_subtile_map[subtile];
70
71 /* Calculate new position */
72 pos = tile_base +
73 swizzle * F_SUBTILE_SIZE +
74 tile_y % F_SUBTILE_HEIGHT * OW_SIZE +
75 tile_x % F_SUBTILE_WIDTH;
76
77 GEM_BUG_ON(!IS_ALIGNED(pos, pixel_size));
78
79 return pos / pixel_size * 4;
80 }
81
82 enum client_tiling {
83 CLIENT_TILING_LINEAR,
84 CLIENT_TILING_X,
85 CLIENT_TILING_Y,
86 CLIENT_TILING_4,
87 CLIENT_NUM_TILING_TYPES
88 };
89
90 #define WIDTH 512
91 #define HEIGHT 32
92
93 struct blit_buffer {
94 struct i915_vma *vma;
95 u32 start_val;
96 enum client_tiling tiling;
97 };
98
99 struct tiled_blits {
100 struct intel_context *ce;
101 struct blit_buffer buffers[3];
102 struct blit_buffer scratch;
103 struct i915_vma *batch;
104 u64 hole;
105 u64 align;
106 u32 width;
107 u32 height;
108 };
109
supports_x_tiling(const struct drm_i915_private * i915)110 static bool supports_x_tiling(const struct drm_i915_private *i915)
111 {
112 int gen = GRAPHICS_VER(i915);
113
114 if (gen < 12)
115 return true;
116
117 if (!HAS_LMEM(i915) || IS_DG1(i915))
118 return false;
119
120 return true;
121 }
122
fast_blit_ok(const struct blit_buffer * buf)123 static bool fast_blit_ok(const struct blit_buffer *buf)
124 {
125 int gen = GRAPHICS_VER(buf->vma->vm->i915);
126
127 if (gen < 9)
128 return false;
129
130 if (gen < 12)
131 return true;
132
133 /* filter out platforms with unsupported X-tile support in fastblit */
134 if (buf->tiling == CLIENT_TILING_X && !supports_x_tiling(buf->vma->vm->i915))
135 return false;
136
137 return true;
138 }
139
prepare_blit(const struct tiled_blits * t,struct blit_buffer * dst,struct blit_buffer * src,struct drm_i915_gem_object * batch)140 static int prepare_blit(const struct tiled_blits *t,
141 struct blit_buffer *dst,
142 struct blit_buffer *src,
143 struct drm_i915_gem_object *batch)
144 {
145 const int ver = GRAPHICS_VER(to_i915(batch->base.dev));
146 bool use_64b_reloc = ver >= 8;
147 u32 src_pitch, dst_pitch;
148 u32 cmd, *cs;
149
150 cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
151 if (IS_ERR(cs))
152 return PTR_ERR(cs);
153
154 if (fast_blit_ok(dst) && fast_blit_ok(src)) {
155 struct intel_gt *gt = t->ce->engine->gt;
156 u32 src_tiles = 0, dst_tiles = 0;
157 u32 src_4t = 0, dst_4t = 0;
158
159 /* Need to program BLIT_CCTL if it is not done previously
160 * before using XY_FAST_COPY_BLT
161 */
162 *cs++ = MI_LOAD_REGISTER_IMM(1);
163 *cs++ = i915_mmio_reg_offset(BLIT_CCTL(t->ce->engine->mmio_base));
164 *cs++ = (BLIT_CCTL_SRC_MOCS(gt->mocs.uc_index) |
165 BLIT_CCTL_DST_MOCS(gt->mocs.uc_index));
166
167 src_pitch = t->width; /* in dwords */
168 if (src->tiling == CLIENT_TILING_4) {
169 src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(YMAJOR);
170 src_4t = XY_FAST_COPY_BLT_D1_SRC_TILE4;
171 } else if (src->tiling == CLIENT_TILING_Y) {
172 src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(YMAJOR);
173 } else if (src->tiling == CLIENT_TILING_X) {
174 src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(TILE_X);
175 } else {
176 src_pitch *= 4; /* in bytes */
177 }
178
179 dst_pitch = t->width; /* in dwords */
180 if (dst->tiling == CLIENT_TILING_4) {
181 dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(YMAJOR);
182 dst_4t = XY_FAST_COPY_BLT_D1_DST_TILE4;
183 } else if (dst->tiling == CLIENT_TILING_Y) {
184 dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(YMAJOR);
185 } else if (dst->tiling == CLIENT_TILING_X) {
186 dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(TILE_X);
187 } else {
188 dst_pitch *= 4; /* in bytes */
189 }
190
191 *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2) |
192 src_tiles | dst_tiles;
193 *cs++ = src_4t | dst_4t | BLT_DEPTH_32 | dst_pitch;
194 *cs++ = 0;
195 *cs++ = t->height << 16 | t->width;
196 *cs++ = lower_32_bits(dst->vma->node.start);
197 *cs++ = upper_32_bits(dst->vma->node.start);
198 *cs++ = 0;
199 *cs++ = src_pitch;
200 *cs++ = lower_32_bits(src->vma->node.start);
201 *cs++ = upper_32_bits(src->vma->node.start);
202 } else {
203 if (ver >= 6) {
204 *cs++ = MI_LOAD_REGISTER_IMM(1);
205 *cs++ = i915_mmio_reg_offset(BCS_SWCTRL);
206 cmd = (BCS_SRC_Y | BCS_DST_Y) << 16;
207 if (src->tiling == CLIENT_TILING_Y)
208 cmd |= BCS_SRC_Y;
209 if (dst->tiling == CLIENT_TILING_Y)
210 cmd |= BCS_DST_Y;
211 *cs++ = cmd;
212
213 cmd = MI_FLUSH_DW;
214 if (ver >= 8)
215 cmd++;
216 *cs++ = cmd;
217 *cs++ = 0;
218 *cs++ = 0;
219 *cs++ = 0;
220 }
221
222 cmd = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (8 - 2);
223 if (ver >= 8)
224 cmd += 2;
225
226 src_pitch = t->width * 4;
227 if (src->tiling) {
228 cmd |= XY_SRC_COPY_BLT_SRC_TILED;
229 src_pitch /= 4;
230 }
231
232 dst_pitch = t->width * 4;
233 if (dst->tiling) {
234 cmd |= XY_SRC_COPY_BLT_DST_TILED;
235 dst_pitch /= 4;
236 }
237
238 *cs++ = cmd;
239 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | dst_pitch;
240 *cs++ = 0;
241 *cs++ = t->height << 16 | t->width;
242 *cs++ = lower_32_bits(dst->vma->node.start);
243 if (use_64b_reloc)
244 *cs++ = upper_32_bits(dst->vma->node.start);
245 *cs++ = 0;
246 *cs++ = src_pitch;
247 *cs++ = lower_32_bits(src->vma->node.start);
248 if (use_64b_reloc)
249 *cs++ = upper_32_bits(src->vma->node.start);
250 }
251
252 *cs++ = MI_BATCH_BUFFER_END;
253
254 i915_gem_object_flush_map(batch);
255 i915_gem_object_unpin_map(batch);
256
257 return 0;
258 }
259
tiled_blits_destroy_buffers(struct tiled_blits * t)260 static void tiled_blits_destroy_buffers(struct tiled_blits *t)
261 {
262 int i;
263
264 for (i = 0; i < ARRAY_SIZE(t->buffers); i++)
265 i915_vma_put(t->buffers[i].vma);
266
267 i915_vma_put(t->scratch.vma);
268 i915_vma_put(t->batch);
269 }
270
271 static struct i915_vma *
__create_vma(struct tiled_blits * t,size_t size,bool lmem)272 __create_vma(struct tiled_blits *t, size_t size, bool lmem)
273 {
274 struct drm_i915_private *i915 = t->ce->vm->i915;
275 struct drm_i915_gem_object *obj;
276 struct i915_vma *vma;
277
278 if (lmem)
279 obj = i915_gem_object_create_lmem(i915, size, 0);
280 else
281 obj = i915_gem_object_create_shmem(i915, size);
282 if (IS_ERR(obj))
283 return ERR_CAST(obj);
284
285 vma = i915_vma_instance(obj, t->ce->vm, NULL);
286 if (IS_ERR(vma))
287 i915_gem_object_put(obj);
288
289 return vma;
290 }
291
create_vma(struct tiled_blits * t,bool lmem)292 static struct i915_vma *create_vma(struct tiled_blits *t, bool lmem)
293 {
294 return __create_vma(t, PAGE_ALIGN(t->width * t->height * 4), lmem);
295 }
296
tiled_blits_create_buffers(struct tiled_blits * t,int width,int height,struct rnd_state * prng)297 static int tiled_blits_create_buffers(struct tiled_blits *t,
298 int width, int height,
299 struct rnd_state *prng)
300 {
301 struct drm_i915_private *i915 = t->ce->engine->i915;
302 int i;
303
304 t->width = width;
305 t->height = height;
306
307 t->batch = __create_vma(t, PAGE_SIZE, false);
308 if (IS_ERR(t->batch))
309 return PTR_ERR(t->batch);
310
311 t->scratch.vma = create_vma(t, false);
312 if (IS_ERR(t->scratch.vma)) {
313 i915_vma_put(t->batch);
314 return PTR_ERR(t->scratch.vma);
315 }
316
317 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) {
318 struct i915_vma *vma;
319
320 vma = create_vma(t, HAS_LMEM(i915) && i % 2);
321 if (IS_ERR(vma)) {
322 tiled_blits_destroy_buffers(t);
323 return PTR_ERR(vma);
324 }
325
326 t->buffers[i].vma = vma;
327 t->buffers[i].tiling =
328 i915_prandom_u32_max_state(CLIENT_NUM_TILING_TYPES, prng);
329
330 /* Platforms support either TileY or Tile4, not both */
331 if (HAS_4TILE(i915) && t->buffers[i].tiling == CLIENT_TILING_Y)
332 t->buffers[i].tiling = CLIENT_TILING_4;
333 else if (!HAS_4TILE(i915) && t->buffers[i].tiling == CLIENT_TILING_4)
334 t->buffers[i].tiling = CLIENT_TILING_Y;
335 }
336
337 return 0;
338 }
339
fill_scratch(struct tiled_blits * t,u32 * vaddr,u32 val)340 static void fill_scratch(struct tiled_blits *t, u32 *vaddr, u32 val)
341 {
342 int i;
343
344 t->scratch.start_val = val;
345 for (i = 0; i < t->width * t->height; i++)
346 vaddr[i] = val++;
347
348 i915_gem_object_flush_map(t->scratch.vma->obj);
349 }
350
swizzle_bit(unsigned int bit,u64 offset)351 static u64 swizzle_bit(unsigned int bit, u64 offset)
352 {
353 return (offset & BIT_ULL(bit)) >> (bit - 6);
354 }
355
tiled_offset(const struct intel_gt * gt,u64 v,unsigned int stride,enum client_tiling tiling,int x_pos,int y_pos)356 static u64 tiled_offset(const struct intel_gt *gt,
357 u64 v,
358 unsigned int stride,
359 enum client_tiling tiling,
360 int x_pos, int y_pos)
361 {
362 unsigned int swizzle;
363 u64 x, y;
364
365 if (tiling == CLIENT_TILING_LINEAR)
366 return v;
367
368 y = div64_u64_rem(v, stride, &x);
369
370 if (tiling == CLIENT_TILING_4) {
371 v = linear_x_y_to_ftiled_pos(x_pos, y_pos, stride, 32);
372
373 /* no swizzling for f-tiling */
374 swizzle = I915_BIT_6_SWIZZLE_NONE;
375 } else if (tiling == CLIENT_TILING_X) {
376 v = div64_u64_rem(y, 8, &y) * stride * 8;
377 v += y * 512;
378 v += div64_u64_rem(x, 512, &x) << 12;
379 v += x;
380
381 swizzle = gt->ggtt->bit_6_swizzle_x;
382 } else {
383 const unsigned int ytile_span = 16;
384 const unsigned int ytile_height = 512;
385
386 v = div64_u64_rem(y, 32, &y) * stride * 32;
387 v += y * ytile_span;
388 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
389 v += x;
390
391 swizzle = gt->ggtt->bit_6_swizzle_y;
392 }
393
394 switch (swizzle) {
395 case I915_BIT_6_SWIZZLE_9:
396 v ^= swizzle_bit(9, v);
397 break;
398 case I915_BIT_6_SWIZZLE_9_10:
399 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
400 break;
401 case I915_BIT_6_SWIZZLE_9_11:
402 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
403 break;
404 case I915_BIT_6_SWIZZLE_9_10_11:
405 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
406 break;
407 }
408
409 return v;
410 }
411
repr_tiling(enum client_tiling tiling)412 static const char *repr_tiling(enum client_tiling tiling)
413 {
414 switch (tiling) {
415 case CLIENT_TILING_LINEAR: return "linear";
416 case CLIENT_TILING_X: return "X";
417 case CLIENT_TILING_Y: return "Y";
418 case CLIENT_TILING_4: return "F";
419 default: return "unknown";
420 }
421 }
422
verify_buffer(const struct tiled_blits * t,struct blit_buffer * buf,struct rnd_state * prng)423 static int verify_buffer(const struct tiled_blits *t,
424 struct blit_buffer *buf,
425 struct rnd_state *prng)
426 {
427 const u32 *vaddr;
428 int ret = 0;
429 int x, y, p;
430
431 x = i915_prandom_u32_max_state(t->width, prng);
432 y = i915_prandom_u32_max_state(t->height, prng);
433 p = y * t->width + x;
434
435 vaddr = i915_gem_object_pin_map_unlocked(buf->vma->obj, I915_MAP_WC);
436 if (IS_ERR(vaddr))
437 return PTR_ERR(vaddr);
438
439 if (vaddr[0] != buf->start_val) {
440 ret = -EINVAL;
441 } else {
442 u64 v = tiled_offset(buf->vma->vm->gt,
443 p * 4, t->width * 4,
444 buf->tiling, x, y);
445
446 if (vaddr[v / sizeof(*vaddr)] != buf->start_val + p)
447 ret = -EINVAL;
448 }
449 if (ret) {
450 pr_err("Invalid %s tiling detected at (%d, %d), start_val %x\n",
451 repr_tiling(buf->tiling),
452 x, y, buf->start_val);
453 igt_hexdump(vaddr, 4096);
454 }
455
456 i915_gem_object_unpin_map(buf->vma->obj);
457 return ret;
458 }
459
move_to_active(struct i915_vma * vma,struct i915_request * rq,unsigned int flags)460 static int move_to_active(struct i915_vma *vma,
461 struct i915_request *rq,
462 unsigned int flags)
463 {
464 int err;
465
466 i915_vma_lock(vma);
467 err = i915_request_await_object(rq, vma->obj, false);
468 if (err == 0)
469 err = i915_vma_move_to_active(vma, rq, flags);
470 i915_vma_unlock(vma);
471
472 return err;
473 }
474
pin_buffer(struct i915_vma * vma,u64 addr)475 static int pin_buffer(struct i915_vma *vma, u64 addr)
476 {
477 int err;
478
479 if (drm_mm_node_allocated(&vma->node) && vma->node.start != addr) {
480 err = i915_vma_unbind_unlocked(vma);
481 if (err)
482 return err;
483 }
484
485 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED | addr);
486 if (err)
487 return err;
488
489 return 0;
490 }
491
492 static int
tiled_blit(struct tiled_blits * t,struct blit_buffer * dst,u64 dst_addr,struct blit_buffer * src,u64 src_addr)493 tiled_blit(struct tiled_blits *t,
494 struct blit_buffer *dst, u64 dst_addr,
495 struct blit_buffer *src, u64 src_addr)
496 {
497 struct i915_request *rq;
498 int err;
499
500 err = pin_buffer(src->vma, src_addr);
501 if (err) {
502 pr_err("Cannot pin src @ %llx\n", src_addr);
503 return err;
504 }
505
506 err = pin_buffer(dst->vma, dst_addr);
507 if (err) {
508 pr_err("Cannot pin dst @ %llx\n", dst_addr);
509 goto err_src;
510 }
511
512 err = i915_vma_pin(t->batch, 0, 0, PIN_USER | PIN_HIGH);
513 if (err) {
514 pr_err("cannot pin batch\n");
515 goto err_dst;
516 }
517
518 err = prepare_blit(t, dst, src, t->batch->obj);
519 if (err)
520 goto err_bb;
521
522 rq = intel_context_create_request(t->ce);
523 if (IS_ERR(rq)) {
524 err = PTR_ERR(rq);
525 goto err_bb;
526 }
527
528 err = move_to_active(t->batch, rq, 0);
529 if (!err)
530 err = move_to_active(src->vma, rq, 0);
531 if (!err)
532 err = move_to_active(dst->vma, rq, 0);
533 if (!err)
534 err = rq->engine->emit_bb_start(rq,
535 t->batch->node.start,
536 t->batch->node.size,
537 0);
538 i915_request_get(rq);
539 i915_request_add(rq);
540 if (i915_request_wait(rq, 0, HZ / 2) < 0)
541 err = -ETIME;
542 i915_request_put(rq);
543
544 dst->start_val = src->start_val;
545 err_bb:
546 i915_vma_unpin(t->batch);
547 err_dst:
548 i915_vma_unpin(dst->vma);
549 err_src:
550 i915_vma_unpin(src->vma);
551 return err;
552 }
553
554 static struct tiled_blits *
tiled_blits_create(struct intel_engine_cs * engine,struct rnd_state * prng)555 tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng)
556 {
557 struct drm_mm_node hole;
558 struct tiled_blits *t;
559 u64 hole_size;
560 int err;
561
562 t = kzalloc(sizeof(*t), GFP_KERNEL);
563 if (!t)
564 return ERR_PTR(-ENOMEM);
565
566 t->ce = intel_context_create(engine);
567 if (IS_ERR(t->ce)) {
568 err = PTR_ERR(t->ce);
569 goto err_free;
570 }
571
572 t->align = i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_LOCAL);
573 t->align = max(t->align,
574 i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_SYSTEM));
575
576 hole_size = 2 * round_up(WIDTH * HEIGHT * 4, t->align);
577 hole_size *= 2; /* room to maneuver */
578 hole_size += 2 * t->align; /* padding on either side */
579
580 mutex_lock(&t->ce->vm->mutex);
581 memset(&hole, 0, sizeof(hole));
582 err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole,
583 hole_size, t->align,
584 I915_COLOR_UNEVICTABLE,
585 0, U64_MAX,
586 DRM_MM_INSERT_BEST);
587 if (!err)
588 drm_mm_remove_node(&hole);
589 mutex_unlock(&t->ce->vm->mutex);
590 if (err) {
591 err = -ENODEV;
592 goto err_put;
593 }
594
595 t->hole = hole.start + t->align;
596 pr_info("Using hole at %llx\n", t->hole);
597
598 err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng);
599 if (err)
600 goto err_put;
601
602 return t;
603
604 err_put:
605 intel_context_put(t->ce);
606 err_free:
607 kfree(t);
608 return ERR_PTR(err);
609 }
610
tiled_blits_destroy(struct tiled_blits * t)611 static void tiled_blits_destroy(struct tiled_blits *t)
612 {
613 tiled_blits_destroy_buffers(t);
614
615 intel_context_put(t->ce);
616 kfree(t);
617 }
618
tiled_blits_prepare(struct tiled_blits * t,struct rnd_state * prng)619 static int tiled_blits_prepare(struct tiled_blits *t,
620 struct rnd_state *prng)
621 {
622 u64 offset = round_up(t->width * t->height * 4, t->align);
623 u32 *map;
624 int err;
625 int i;
626
627 map = i915_gem_object_pin_map_unlocked(t->scratch.vma->obj, I915_MAP_WC);
628 if (IS_ERR(map))
629 return PTR_ERR(map);
630
631 /* Use scratch to fill objects */
632 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) {
633 fill_scratch(t, map, prandom_u32_state(prng));
634 GEM_BUG_ON(verify_buffer(t, &t->scratch, prng));
635
636 err = tiled_blit(t,
637 &t->buffers[i], t->hole + offset,
638 &t->scratch, t->hole);
639 if (err == 0)
640 err = verify_buffer(t, &t->buffers[i], prng);
641 if (err) {
642 pr_err("Failed to create buffer %d\n", i);
643 break;
644 }
645 }
646
647 i915_gem_object_unpin_map(t->scratch.vma->obj);
648 return err;
649 }
650
tiled_blits_bounce(struct tiled_blits * t,struct rnd_state * prng)651 static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng)
652 {
653 u64 offset = round_up(t->width * t->height * 4, 2 * t->align);
654 int err;
655
656 /* We want to check position invariant tiling across GTT eviction */
657
658 err = tiled_blit(t,
659 &t->buffers[1], t->hole + offset / 2,
660 &t->buffers[0], t->hole + 2 * offset);
661 if (err)
662 return err;
663
664 /* Simulating GTT eviction of the same buffer / layout */
665 t->buffers[2].tiling = t->buffers[0].tiling;
666
667 /* Reposition so that we overlap the old addresses, and slightly off */
668 err = tiled_blit(t,
669 &t->buffers[2], t->hole + t->align,
670 &t->buffers[1], t->hole + 3 * offset / 2);
671 if (err)
672 return err;
673
674 err = verify_buffer(t, &t->buffers[2], prng);
675 if (err)
676 return err;
677
678 return 0;
679 }
680
__igt_client_tiled_blits(struct intel_engine_cs * engine,struct rnd_state * prng)681 static int __igt_client_tiled_blits(struct intel_engine_cs *engine,
682 struct rnd_state *prng)
683 {
684 struct tiled_blits *t;
685 int err;
686
687 t = tiled_blits_create(engine, prng);
688 if (IS_ERR(t))
689 return PTR_ERR(t);
690
691 err = tiled_blits_prepare(t, prng);
692 if (err)
693 goto out;
694
695 err = tiled_blits_bounce(t, prng);
696 if (err)
697 goto out;
698
699 out:
700 tiled_blits_destroy(t);
701 return err;
702 }
703
has_bit17_swizzle(int sw)704 static bool has_bit17_swizzle(int sw)
705 {
706 return (sw == I915_BIT_6_SWIZZLE_9_10_17 ||
707 sw == I915_BIT_6_SWIZZLE_9_17);
708 }
709
bad_swizzling(struct drm_i915_private * i915)710 static bool bad_swizzling(struct drm_i915_private *i915)
711 {
712 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
713
714 if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
715 return true;
716
717 if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) ||
718 has_bit17_swizzle(ggtt->bit_6_swizzle_y))
719 return true;
720
721 return false;
722 }
723
igt_client_tiled_blits(void * arg)724 static int igt_client_tiled_blits(void *arg)
725 {
726 struct drm_i915_private *i915 = arg;
727 I915_RND_STATE(prng);
728 int inst = 0;
729
730 /* Test requires explicit BLT tiling controls */
731 if (GRAPHICS_VER(i915) < 4)
732 return 0;
733
734 if (bad_swizzling(i915)) /* Requires sane (sub-page) swizzling */
735 return 0;
736
737 do {
738 struct intel_engine_cs *engine;
739 int err;
740
741 engine = intel_engine_lookup_user(i915,
742 I915_ENGINE_CLASS_COPY,
743 inst++);
744 if (!engine)
745 return 0;
746
747 err = __igt_client_tiled_blits(engine, &prng);
748 if (err == -ENODEV)
749 err = 0;
750 if (err)
751 return err;
752 } while (1);
753 }
754
i915_gem_client_blt_live_selftests(struct drm_i915_private * i915)755 int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915)
756 {
757 static const struct i915_subtest tests[] = {
758 SUBTEST(igt_client_tiled_blits),
759 };
760
761 if (intel_gt_is_wedged(to_gt(i915)))
762 return 0;
763
764 return i915_live_subtests(tests, i915);
765 }
766