1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2020 Intel Corporation
4 */
5
6 #include "i915_drv.h"
7 #include "intel_context.h"
8 #include "intel_gpu_commands.h"
9 #include "intel_gt.h"
10 #include "intel_gtt.h"
11 #include "intel_migrate.h"
12 #include "intel_ring.h"
13
14 struct insert_pte_data {
15 u64 offset;
16 };
17
18 #define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
19
20 #define GET_CCS_BYTES(i915, size) (HAS_FLAT_CCS(i915) ? \
21 DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0)
engine_supports_migration(struct intel_engine_cs * engine)22 static bool engine_supports_migration(struct intel_engine_cs *engine)
23 {
24 if (!engine)
25 return false;
26
27 /*
28 * We need the ability to prevent aribtration (MI_ARB_ON_OFF),
29 * the ability to write PTE using inline data (MI_STORE_DATA)
30 * and of course the ability to do the block transfer (blits).
31 */
32 GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS);
33
34 return true;
35 }
36
xehpsdv_toggle_pdes(struct i915_address_space * vm,struct i915_page_table * pt,void * data)37 static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
38 struct i915_page_table *pt,
39 void *data)
40 {
41 struct insert_pte_data *d = data;
42
43 /*
44 * Insert a dummy PTE into every PT that will map to LMEM to ensure
45 * we have a correctly setup PDE structure for later use.
46 */
47 vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM);
48 GEM_BUG_ON(!pt->is_compact);
49 d->offset += SZ_2M;
50 }
51
xehpsdv_insert_pte(struct i915_address_space * vm,struct i915_page_table * pt,void * data)52 static void xehpsdv_insert_pte(struct i915_address_space *vm,
53 struct i915_page_table *pt,
54 void *data)
55 {
56 struct insert_pte_data *d = data;
57
58 /*
59 * We are playing tricks here, since the actual pt, from the hw
60 * pov, is only 256bytes with 32 entries, or 4096bytes with 512
61 * entries, but we are still guaranteed that the physical
62 * alignment is 64K underneath for the pt, and we are careful
63 * not to access the space in the void.
64 */
65 vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM);
66 d->offset += SZ_64K;
67 }
68
insert_pte(struct i915_address_space * vm,struct i915_page_table * pt,void * data)69 static void insert_pte(struct i915_address_space *vm,
70 struct i915_page_table *pt,
71 void *data)
72 {
73 struct insert_pte_data *d = data;
74
75 vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE,
76 i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0);
77 d->offset += PAGE_SIZE;
78 }
79
migrate_vm(struct intel_gt * gt)80 static struct i915_address_space *migrate_vm(struct intel_gt *gt)
81 {
82 struct i915_vm_pt_stash stash = {};
83 struct i915_ppgtt *vm;
84 int err;
85 int i;
86
87 /*
88 * We construct a very special VM for use by all migration contexts,
89 * it is kept pinned so that it can be used at any time. As we need
90 * to pre-allocate the page directories for the migration VM, this
91 * limits us to only using a small number of prepared vma.
92 *
93 * To be able to pipeline and reschedule migration operations while
94 * avoiding unnecessary contention on the vm itself, the PTE updates
95 * are inline with the blits. All the blits use the same fixed
96 * addresses, with the backing store redirection being updated on the
97 * fly. Only 2 implicit vma are used for all migration operations.
98 *
99 * We lay the ppGTT out as:
100 *
101 * [0, CHUNK_SZ) -> first object
102 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second object
103 * [2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE
104 *
105 * By exposing the dma addresses of the page directories themselves
106 * within the ppGTT, we are then able to rewrite the PTE prior to use.
107 * But the PTE update and subsequent migration operation must be atomic,
108 * i.e. within the same non-preemptible window so that we do not switch
109 * to another migration context that overwrites the PTE.
110 *
111 * This changes quite a bit on platforms with HAS_64K_PAGES support,
112 * where we instead have three windows, each CHUNK_SIZE in size. The
113 * first is reserved for mapping system-memory, and that just uses the
114 * 512 entry layout using 4K GTT pages. The other two windows just map
115 * lmem pages and must use the new compact 32 entry layout using 64K GTT
116 * pages, which ensures we can address any lmem object that the user
117 * throws at us. We then also use the xehpsdv_toggle_pdes as a way of
118 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
119 * compact layout for each of these page-tables, that fall within the
120 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
121 *
122 * We lay the ppGTT out as:
123 *
124 * [0, CHUNK_SZ) -> first window/object, maps smem
125 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src
126 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst
127 *
128 * For the PTE window it's also quite different, since each PTE must
129 * point to some 64K page, one for each PT(since it's in lmem), and yet
130 * each is only <= 4096bytes, but since the unused space within that PTE
131 * range is never touched, this should be fine.
132 *
133 * So basically each PT now needs 64K of virtual memory, instead of 4K,
134 * which looks like:
135 *
136 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE
137 */
138
139 vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY);
140 if (IS_ERR(vm))
141 return ERR_CAST(vm);
142
143 if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
144 err = -ENODEV;
145 goto err_vm;
146 }
147
148 if (HAS_64K_PAGES(gt->i915))
149 stash.pt_sz = I915_GTT_PAGE_SIZE_64K;
150
151 /*
152 * Each engine instance is assigned its own chunk in the VM, so
153 * that we can run multiple instances concurrently
154 */
155 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
156 struct intel_engine_cs *engine;
157 u64 base = (u64)i << 32;
158 struct insert_pte_data d = {};
159 struct i915_gem_ww_ctx ww;
160 u64 sz;
161
162 engine = gt->engine_class[COPY_ENGINE_CLASS][i];
163 if (!engine_supports_migration(engine))
164 continue;
165
166 /*
167 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
168 * 4x2 page directories for source/destination.
169 */
170 if (HAS_64K_PAGES(gt->i915))
171 sz = 3 * CHUNK_SZ;
172 else
173 sz = 2 * CHUNK_SZ;
174 d.offset = base + sz;
175
176 /*
177 * We need another page directory setup so that we can write
178 * the 8x512 PTE in each chunk.
179 */
180 if (HAS_64K_PAGES(gt->i915))
181 sz += (sz / SZ_2M) * SZ_64K;
182 else
183 sz += (sz >> 12) * sizeof(u64);
184
185 err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
186 if (err)
187 goto err_vm;
188
189 for_i915_gem_ww(&ww, err, true) {
190 err = i915_vm_lock_objects(&vm->vm, &ww);
191 if (err)
192 continue;
193 err = i915_vm_map_pt_stash(&vm->vm, &stash);
194 if (err)
195 continue;
196
197 vm->vm.allocate_va_range(&vm->vm, &stash, base, sz);
198 }
199 i915_vm_free_pt_stash(&vm->vm, &stash);
200 if (err)
201 goto err_vm;
202
203 /* Now allow the GPU to rewrite the PTE via its own ppGTT */
204 if (HAS_64K_PAGES(gt->i915)) {
205 vm->vm.foreach(&vm->vm, base, d.offset - base,
206 xehpsdv_insert_pte, &d);
207 d.offset = base + CHUNK_SZ;
208 vm->vm.foreach(&vm->vm,
209 d.offset,
210 2 * CHUNK_SZ,
211 xehpsdv_toggle_pdes, &d);
212 } else {
213 vm->vm.foreach(&vm->vm, base, d.offset - base,
214 insert_pte, &d);
215 }
216 }
217
218 return &vm->vm;
219
220 err_vm:
221 i915_vm_put(&vm->vm);
222 return ERR_PTR(err);
223 }
224
first_copy_engine(struct intel_gt * gt)225 static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
226 {
227 struct intel_engine_cs *engine;
228 int i;
229
230 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
231 engine = gt->engine_class[COPY_ENGINE_CLASS][i];
232 if (engine_supports_migration(engine))
233 return engine;
234 }
235
236 return NULL;
237 }
238
pinned_context(struct intel_gt * gt)239 static struct intel_context *pinned_context(struct intel_gt *gt)
240 {
241 static struct lock_class_key key;
242 struct intel_engine_cs *engine;
243 struct i915_address_space *vm;
244 struct intel_context *ce;
245
246 engine = first_copy_engine(gt);
247 if (!engine)
248 return ERR_PTR(-ENODEV);
249
250 vm = migrate_vm(gt);
251 if (IS_ERR(vm))
252 return ERR_CAST(vm);
253
254 ce = intel_engine_create_pinned_context(engine, vm, SZ_512K,
255 I915_GEM_HWS_MIGRATE,
256 &key, "migrate");
257 i915_vm_put(vm);
258 return ce;
259 }
260
intel_migrate_init(struct intel_migrate * m,struct intel_gt * gt)261 int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
262 {
263 struct intel_context *ce;
264
265 memset(m, 0, sizeof(*m));
266
267 ce = pinned_context(gt);
268 if (IS_ERR(ce))
269 return PTR_ERR(ce);
270
271 m->context = ce;
272 return 0;
273 }
274
random_index(unsigned int max)275 static int random_index(unsigned int max)
276 {
277 return upper_32_bits(mul_u32_u32(get_random_u32(), max));
278 }
279
__migrate_engines(struct intel_gt * gt)280 static struct intel_context *__migrate_engines(struct intel_gt *gt)
281 {
282 struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
283 struct intel_engine_cs *engine;
284 unsigned int count, i;
285
286 count = 0;
287 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
288 engine = gt->engine_class[COPY_ENGINE_CLASS][i];
289 if (engine_supports_migration(engine))
290 engines[count++] = engine;
291 }
292
293 return intel_context_create(engines[random_index(count)]);
294 }
295
intel_migrate_create_context(struct intel_migrate * m)296 struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
297 {
298 struct intel_context *ce;
299
300 /*
301 * We randomly distribute contexts across the engines upon constrction,
302 * as they all share the same pinned vm, and so in order to allow
303 * multiple blits to run in parallel, we must construct each blit
304 * to use a different range of the vm for its GTT. This has to be
305 * known at construction, so we can not use the late greedy load
306 * balancing of the virtual-engine.
307 */
308 ce = __migrate_engines(m->context->engine->gt);
309 if (IS_ERR(ce))
310 return ce;
311
312 ce->ring = NULL;
313 ce->ring_size = SZ_256K;
314
315 i915_vm_put(ce->vm);
316 ce->vm = i915_vm_get(m->context->vm);
317
318 return ce;
319 }
320
sg_sgt(struct scatterlist * sg)321 static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
322 {
323 dma_addr_t addr = sg_dma_address(sg);
324
325 return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
326 }
327
emit_no_arbitration(struct i915_request * rq)328 static int emit_no_arbitration(struct i915_request *rq)
329 {
330 u32 *cs;
331
332 cs = intel_ring_begin(rq, 2);
333 if (IS_ERR(cs))
334 return PTR_ERR(cs);
335
336 /* Explicitly disable preemption for this request. */
337 *cs++ = MI_ARB_ON_OFF;
338 *cs++ = MI_NOOP;
339 intel_ring_advance(rq, cs);
340
341 return 0;
342 }
343
max_pte_pkt_size(struct i915_request * rq,int pkt)344 static int max_pte_pkt_size(struct i915_request *rq, int pkt)
345 {
346 struct intel_ring *ring = rq->ring;
347
348 pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5);
349 pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5);
350
351 return pkt;
352 }
353
emit_pte(struct i915_request * rq,struct sgt_dma * it,enum i915_cache_level cache_level,bool is_lmem,u64 offset,int length)354 static int emit_pte(struct i915_request *rq,
355 struct sgt_dma *it,
356 enum i915_cache_level cache_level,
357 bool is_lmem,
358 u64 offset,
359 int length)
360 {
361 bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915);
362 const u64 encode = rq->context->vm->pte_encode(0, cache_level,
363 is_lmem ? PTE_LM : 0);
364 struct intel_ring *ring = rq->ring;
365 int pkt, dword_length;
366 u32 total = 0;
367 u32 page_size;
368 u32 *hdr, *cs;
369
370 GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8);
371
372 page_size = I915_GTT_PAGE_SIZE;
373 dword_length = 0x400;
374
375 /* Compute the page directory offset for the target address range */
376 if (has_64K_pages) {
377 GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M));
378
379 offset /= SZ_2M;
380 offset *= SZ_64K;
381 offset += 3 * CHUNK_SZ;
382
383 if (is_lmem) {
384 page_size = I915_GTT_PAGE_SIZE_64K;
385 dword_length = 0x40;
386 }
387 } else {
388 offset >>= 12;
389 offset *= sizeof(u64);
390 offset += 2 * CHUNK_SZ;
391 }
392
393 offset += (u64)rq->engine->instance << 32;
394
395 cs = intel_ring_begin(rq, 6);
396 if (IS_ERR(cs))
397 return PTR_ERR(cs);
398
399 /* Pack as many PTE updates as possible into a single MI command */
400 pkt = max_pte_pkt_size(rq, dword_length);
401
402 hdr = cs;
403 *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */
404 *cs++ = lower_32_bits(offset);
405 *cs++ = upper_32_bits(offset);
406
407 do {
408 if (cs - hdr >= pkt) {
409 int dword_rem;
410
411 *hdr += cs - hdr - 2;
412 *cs++ = MI_NOOP;
413
414 ring->emit = (void *)cs - ring->vaddr;
415 intel_ring_advance(rq, cs);
416 intel_ring_update_space(ring);
417
418 cs = intel_ring_begin(rq, 6);
419 if (IS_ERR(cs))
420 return PTR_ERR(cs);
421
422 dword_rem = dword_length;
423 if (has_64K_pages) {
424 if (IS_ALIGNED(total, SZ_2M)) {
425 offset = round_up(offset, SZ_64K);
426 } else {
427 dword_rem = SZ_2M - (total & (SZ_2M - 1));
428 dword_rem /= page_size;
429 dword_rem *= 2;
430 }
431 }
432
433 pkt = max_pte_pkt_size(rq, dword_rem);
434
435 hdr = cs;
436 *cs++ = MI_STORE_DATA_IMM | REG_BIT(21);
437 *cs++ = lower_32_bits(offset);
438 *cs++ = upper_32_bits(offset);
439 }
440
441 GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size));
442
443 *cs++ = lower_32_bits(encode | it->dma);
444 *cs++ = upper_32_bits(encode | it->dma);
445
446 offset += 8;
447 total += page_size;
448
449 it->dma += page_size;
450 if (it->dma >= it->max) {
451 it->sg = __sg_next(it->sg);
452 if (!it->sg || sg_dma_len(it->sg) == 0)
453 break;
454
455 it->dma = sg_dma_address(it->sg);
456 it->max = it->dma + sg_dma_len(it->sg);
457 }
458 } while (total < length);
459
460 *hdr += cs - hdr - 2;
461 *cs++ = MI_NOOP;
462
463 ring->emit = (void *)cs - ring->vaddr;
464 intel_ring_advance(rq, cs);
465 intel_ring_update_space(ring);
466
467 return total;
468 }
469
wa_1209644611_applies(int ver,u32 size)470 static bool wa_1209644611_applies(int ver, u32 size)
471 {
472 u32 height = size >> PAGE_SHIFT;
473
474 if (ver != 11)
475 return false;
476
477 return height % 4 == 3 && height <= 8;
478 }
479
480 /**
481 * DOC: Flat-CCS - Memory compression for Local memory
482 *
483 * On Xe-HP and later devices, we use dedicated compression control state (CCS)
484 * stored in local memory for each surface, to support the 3D and media
485 * compression formats.
486 *
487 * The memory required for the CCS of the entire local memory is 1/256 of the
488 * local memory size. So before the kernel boot, the required memory is reserved
489 * for the CCS data and a secure register will be programmed with the CCS base
490 * address.
491 *
492 * Flat CCS data needs to be cleared when a lmem object is allocated.
493 * And CCS data can be copied in and out of CCS region through
494 * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly.
495 *
496 * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
497 * its preference list, on memory pressure, i915 needs to migrate the lmem
498 * content into smem. If the lmem object is Flat-CCS compressed by userspace,
499 * then i915 needs to decompress it. But I915 lack the required information
500 * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
501 *
502 * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
503 * be temporarily evicted to smem, along with the auxiliary CCS state, where
504 * it can be potentially swapped-out at a later point, if required.
505 * If userspace later touches the evicted pages, then we always move
506 * the backing memory back to lmem, which includes restoring the saved CCS state,
507 * and potentially performing any required swap-in.
508 *
509 * For the migration of the lmem objects with smem in placement list, such as
510 * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
511 */
512
i915_flush_dw(u32 * cmd,u32 flags)513 static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
514 {
515 *cmd++ = MI_FLUSH_DW | flags;
516 *cmd++ = 0;
517 *cmd++ = 0;
518
519 return cmd;
520 }
521
emit_copy_ccs(struct i915_request * rq,u32 dst_offset,u8 dst_access,u32 src_offset,u8 src_access,int size)522 static int emit_copy_ccs(struct i915_request *rq,
523 u32 dst_offset, u8 dst_access,
524 u32 src_offset, u8 src_access, int size)
525 {
526 struct drm_i915_private *i915 = rq->engine->i915;
527 int mocs = rq->engine->gt->mocs.uc_index << 1;
528 u32 num_ccs_blks;
529 u32 *cs;
530
531 cs = intel_ring_begin(rq, 12);
532 if (IS_ERR(cs))
533 return PTR_ERR(cs);
534
535 num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
536 NUM_CCS_BYTES_PER_BLOCK);
537 GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER);
538 cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
539
540 /*
541 * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS
542 * data in and out of the CCS region.
543 *
544 * We can copy at most 1024 blocks of 256 bytes using one
545 * XY_CTRL_SURF_COPY_BLT instruction.
546 *
547 * In case we need to copy more than 1024 blocks, we need to add
548 * another instruction to the same batch buffer.
549 *
550 * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS.
551 *
552 * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM.
553 */
554 *cs++ = XY_CTRL_SURF_COPY_BLT |
555 src_access << SRC_ACCESS_TYPE_SHIFT |
556 dst_access << DST_ACCESS_TYPE_SHIFT |
557 ((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT;
558 *cs++ = src_offset;
559 *cs++ = rq->engine->instance |
560 FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
561 *cs++ = dst_offset;
562 *cs++ = rq->engine->instance |
563 FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
564
565 cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
566 *cs++ = MI_NOOP;
567
568 intel_ring_advance(rq, cs);
569
570 return 0;
571 }
572
emit_copy(struct i915_request * rq,u32 dst_offset,u32 src_offset,int size)573 static int emit_copy(struct i915_request *rq,
574 u32 dst_offset, u32 src_offset, int size)
575 {
576 const int ver = GRAPHICS_VER(rq->engine->i915);
577 u32 instance = rq->engine->instance;
578 u32 *cs;
579
580 cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);
581 if (IS_ERR(cs))
582 return PTR_ERR(cs);
583
584 if (ver >= 9 && !wa_1209644611_applies(ver, size)) {
585 *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
586 *cs++ = BLT_DEPTH_32 | PAGE_SIZE;
587 *cs++ = 0;
588 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
589 *cs++ = dst_offset;
590 *cs++ = instance;
591 *cs++ = 0;
592 *cs++ = PAGE_SIZE;
593 *cs++ = src_offset;
594 *cs++ = instance;
595 } else if (ver >= 8) {
596 *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
597 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
598 *cs++ = 0;
599 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
600 *cs++ = dst_offset;
601 *cs++ = instance;
602 *cs++ = 0;
603 *cs++ = PAGE_SIZE;
604 *cs++ = src_offset;
605 *cs++ = instance;
606 } else {
607 GEM_BUG_ON(instance);
608 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
609 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
610 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
611 *cs++ = dst_offset;
612 *cs++ = PAGE_SIZE;
613 *cs++ = src_offset;
614 }
615
616 intel_ring_advance(rq, cs);
617 return 0;
618 }
619
scatter_list_length(struct scatterlist * sg)620 static u64 scatter_list_length(struct scatterlist *sg)
621 {
622 u64 len = 0;
623
624 while (sg && sg_dma_len(sg)) {
625 len += sg_dma_len(sg);
626 sg = sg_next(sg);
627 }
628
629 return len;
630 }
631
632 static int
calculate_chunk_sz(struct drm_i915_private * i915,bool src_is_lmem,u64 bytes_to_cpy,u64 ccs_bytes_to_cpy)633 calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem,
634 u64 bytes_to_cpy, u64 ccs_bytes_to_cpy)
635 {
636 if (ccs_bytes_to_cpy && !src_is_lmem)
637 /*
638 * When CHUNK_SZ is passed all the pages upto CHUNK_SZ
639 * will be taken for the blt. in Flat-ccs supported
640 * platform Smem obj will have more pages than required
641 * for main meory hence limit it to the required size
642 * for main memory
643 */
644 return min_t(u64, bytes_to_cpy, CHUNK_SZ);
645 else
646 return CHUNK_SZ;
647 }
648
get_ccs_sg_sgt(struct sgt_dma * it,u64 bytes_to_cpy)649 static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy)
650 {
651 u64 len;
652
653 do {
654 GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg));
655 len = it->max - it->dma;
656 if (len > bytes_to_cpy) {
657 it->dma += bytes_to_cpy;
658 break;
659 }
660
661 bytes_to_cpy -= len;
662
663 it->sg = __sg_next(it->sg);
664 it->dma = sg_dma_address(it->sg);
665 it->max = it->dma + sg_dma_len(it->sg);
666 } while (bytes_to_cpy);
667 }
668
669 int
intel_context_migrate_copy(struct intel_context * ce,const struct i915_deps * deps,struct scatterlist * src,enum i915_cache_level src_cache_level,bool src_is_lmem,struct scatterlist * dst,enum i915_cache_level dst_cache_level,bool dst_is_lmem,struct i915_request ** out)670 intel_context_migrate_copy(struct intel_context *ce,
671 const struct i915_deps *deps,
672 struct scatterlist *src,
673 enum i915_cache_level src_cache_level,
674 bool src_is_lmem,
675 struct scatterlist *dst,
676 enum i915_cache_level dst_cache_level,
677 bool dst_is_lmem,
678 struct i915_request **out)
679 {
680 struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs;
681 struct drm_i915_private *i915 = ce->engine->i915;
682 u64 ccs_bytes_to_cpy = 0, bytes_to_cpy;
683 enum i915_cache_level ccs_cache_level;
684 u32 src_offset, dst_offset;
685 u8 src_access, dst_access;
686 struct i915_request *rq;
687 u64 src_sz, dst_sz;
688 bool ccs_is_src, overwrite_ccs;
689 int err;
690
691 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
692 GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem));
693 *out = NULL;
694
695 GEM_BUG_ON(ce->ring->size < SZ_64K);
696
697 src_sz = scatter_list_length(src);
698 bytes_to_cpy = src_sz;
699
700 if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) {
701 src_access = !src_is_lmem && dst_is_lmem;
702 dst_access = !src_access;
703
704 dst_sz = scatter_list_length(dst);
705 if (src_is_lmem) {
706 it_ccs = it_dst;
707 ccs_cache_level = dst_cache_level;
708 ccs_is_src = false;
709 } else if (dst_is_lmem) {
710 bytes_to_cpy = dst_sz;
711 it_ccs = it_src;
712 ccs_cache_level = src_cache_level;
713 ccs_is_src = true;
714 }
715
716 /*
717 * When there is a eviction of ccs needed smem will have the
718 * extra pages for the ccs data
719 *
720 * TO-DO: Want to move the size mismatch check to a WARN_ON,
721 * but still we have some requests of smem->lmem with same size.
722 * Need to fix it.
723 */
724 ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
725 if (ccs_bytes_to_cpy)
726 get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);
727 }
728
729 overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem;
730
731 src_offset = 0;
732 dst_offset = CHUNK_SZ;
733 if (HAS_64K_PAGES(ce->engine->i915)) {
734 src_offset = 0;
735 dst_offset = 0;
736 if (src_is_lmem)
737 src_offset = CHUNK_SZ;
738 if (dst_is_lmem)
739 dst_offset = 2 * CHUNK_SZ;
740 }
741
742 do {
743 int len;
744
745 rq = i915_request_create(ce);
746 if (IS_ERR(rq)) {
747 err = PTR_ERR(rq);
748 goto out_ce;
749 }
750
751 if (deps) {
752 err = i915_request_await_deps(rq, deps);
753 if (err)
754 goto out_rq;
755
756 if (rq->engine->emit_init_breadcrumb) {
757 err = rq->engine->emit_init_breadcrumb(rq);
758 if (err)
759 goto out_rq;
760 }
761
762 deps = NULL;
763 }
764
765 /* The PTE updates + copy must not be interrupted. */
766 err = emit_no_arbitration(rq);
767 if (err)
768 goto out_rq;
769
770 src_sz = calculate_chunk_sz(i915, src_is_lmem,
771 bytes_to_cpy, ccs_bytes_to_cpy);
772
773 len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem,
774 src_offset, src_sz);
775 if (!len) {
776 err = -EINVAL;
777 goto out_rq;
778 }
779 if (len < 0) {
780 err = len;
781 goto out_rq;
782 }
783
784 err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem,
785 dst_offset, len);
786 if (err < 0)
787 goto out_rq;
788 if (err < len) {
789 err = -EINVAL;
790 goto out_rq;
791 }
792
793 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
794 if (err)
795 goto out_rq;
796
797 err = emit_copy(rq, dst_offset, src_offset, len);
798 if (err)
799 goto out_rq;
800
801 bytes_to_cpy -= len;
802
803 if (ccs_bytes_to_cpy) {
804 int ccs_sz;
805
806 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
807 if (err)
808 goto out_rq;
809
810 ccs_sz = GET_CCS_BYTES(i915, len);
811 err = emit_pte(rq, &it_ccs, ccs_cache_level, false,
812 ccs_is_src ? src_offset : dst_offset,
813 ccs_sz);
814 if (err < 0)
815 goto out_rq;
816 if (err < ccs_sz) {
817 err = -EINVAL;
818 goto out_rq;
819 }
820
821 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
822 if (err)
823 goto out_rq;
824
825 err = emit_copy_ccs(rq, dst_offset, dst_access,
826 src_offset, src_access, len);
827 if (err)
828 goto out_rq;
829
830 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
831 if (err)
832 goto out_rq;
833 ccs_bytes_to_cpy -= ccs_sz;
834 } else if (overwrite_ccs) {
835 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
836 if (err)
837 goto out_rq;
838
839 /*
840 * While we can't always restore/manage the CCS state,
841 * we still need to ensure we don't leak the CCS state
842 * from the previous user, so make sure we overwrite it
843 * with something.
844 */
845 err = emit_copy_ccs(rq, dst_offset, INDIRECT_ACCESS,
846 dst_offset, DIRECT_ACCESS, len);
847 if (err)
848 goto out_rq;
849
850 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
851 if (err)
852 goto out_rq;
853 }
854
855 /* Arbitration is re-enabled between requests. */
856 out_rq:
857 if (*out)
858 i915_request_put(*out);
859 *out = i915_request_get(rq);
860 i915_request_add(rq);
861
862 if (err)
863 break;
864
865 if (!bytes_to_cpy && !ccs_bytes_to_cpy) {
866 if (src_is_lmem)
867 WARN_ON(it_src.sg && sg_dma_len(it_src.sg));
868 else
869 WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg));
870 break;
871 }
872
873 if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) ||
874 !it_dst.sg || !sg_dma_len(it_dst.sg) ||
875 (ccs_bytes_to_cpy && (!it_ccs.sg ||
876 !sg_dma_len(it_ccs.sg))))) {
877 err = -EINVAL;
878 break;
879 }
880
881 cond_resched();
882 } while (1);
883
884 out_ce:
885 return err;
886 }
887
emit_clear(struct i915_request * rq,u32 offset,int size,u32 value,bool is_lmem)888 static int emit_clear(struct i915_request *rq, u32 offset, int size,
889 u32 value, bool is_lmem)
890 {
891 struct drm_i915_private *i915 = rq->engine->i915;
892 int mocs = rq->engine->gt->mocs.uc_index << 1;
893 const int ver = GRAPHICS_VER(i915);
894 int ring_sz;
895 u32 *cs;
896
897 GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
898
899 if (HAS_FLAT_CCS(i915) && ver >= 12)
900 ring_sz = XY_FAST_COLOR_BLT_DW;
901 else if (ver >= 8)
902 ring_sz = 8;
903 else
904 ring_sz = 6;
905
906 cs = intel_ring_begin(rq, ring_sz);
907 if (IS_ERR(cs))
908 return PTR_ERR(cs);
909
910 if (HAS_FLAT_CCS(i915) && ver >= 12) {
911 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
912 (XY_FAST_COLOR_BLT_DW - 2);
913 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
914 (PAGE_SIZE - 1);
915 *cs++ = 0;
916 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
917 *cs++ = offset;
918 *cs++ = rq->engine->instance;
919 *cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
920 /* BG7 */
921 *cs++ = value;
922 *cs++ = 0;
923 *cs++ = 0;
924 *cs++ = 0;
925 /* BG11 */
926 *cs++ = 0;
927 *cs++ = 0;
928 /* BG13 */
929 *cs++ = 0;
930 *cs++ = 0;
931 *cs++ = 0;
932 } else if (ver >= 8) {
933 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);
934 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
935 *cs++ = 0;
936 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
937 *cs++ = offset;
938 *cs++ = rq->engine->instance;
939 *cs++ = value;
940 *cs++ = MI_NOOP;
941 } else {
942 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
943 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
944 *cs++ = 0;
945 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
946 *cs++ = offset;
947 *cs++ = value;
948 }
949
950 intel_ring_advance(rq, cs);
951 return 0;
952 }
953
954 int
intel_context_migrate_clear(struct intel_context * ce,const struct i915_deps * deps,struct scatterlist * sg,enum i915_cache_level cache_level,bool is_lmem,u32 value,struct i915_request ** out)955 intel_context_migrate_clear(struct intel_context *ce,
956 const struct i915_deps *deps,
957 struct scatterlist *sg,
958 enum i915_cache_level cache_level,
959 bool is_lmem,
960 u32 value,
961 struct i915_request **out)
962 {
963 struct drm_i915_private *i915 = ce->engine->i915;
964 struct sgt_dma it = sg_sgt(sg);
965 struct i915_request *rq;
966 u32 offset;
967 int err;
968
969 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
970 *out = NULL;
971
972 GEM_BUG_ON(ce->ring->size < SZ_64K);
973
974 offset = 0;
975 if (HAS_64K_PAGES(i915) && is_lmem)
976 offset = CHUNK_SZ;
977
978 do {
979 int len;
980
981 rq = i915_request_create(ce);
982 if (IS_ERR(rq)) {
983 err = PTR_ERR(rq);
984 goto out_ce;
985 }
986
987 if (deps) {
988 err = i915_request_await_deps(rq, deps);
989 if (err)
990 goto out_rq;
991
992 if (rq->engine->emit_init_breadcrumb) {
993 err = rq->engine->emit_init_breadcrumb(rq);
994 if (err)
995 goto out_rq;
996 }
997
998 deps = NULL;
999 }
1000
1001 /* The PTE updates + clear must not be interrupted. */
1002 err = emit_no_arbitration(rq);
1003 if (err)
1004 goto out_rq;
1005
1006 len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ);
1007 if (len <= 0) {
1008 err = len;
1009 goto out_rq;
1010 }
1011
1012 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1013 if (err)
1014 goto out_rq;
1015
1016 err = emit_clear(rq, offset, len, value, is_lmem);
1017 if (err)
1018 goto out_rq;
1019
1020 if (HAS_FLAT_CCS(i915) && is_lmem && !value) {
1021 /*
1022 * copy the content of memory into corresponding
1023 * ccs surface
1024 */
1025 err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset,
1026 DIRECT_ACCESS, len);
1027 if (err)
1028 goto out_rq;
1029 }
1030
1031 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1032
1033 /* Arbitration is re-enabled between requests. */
1034 out_rq:
1035 if (*out)
1036 i915_request_put(*out);
1037 *out = i915_request_get(rq);
1038 i915_request_add(rq);
1039 if (err || !it.sg || !sg_dma_len(it.sg))
1040 break;
1041
1042 cond_resched();
1043 } while (1);
1044
1045 out_ce:
1046 return err;
1047 }
1048
intel_migrate_copy(struct intel_migrate * m,struct i915_gem_ww_ctx * ww,const struct i915_deps * deps,struct scatterlist * src,enum i915_cache_level src_cache_level,bool src_is_lmem,struct scatterlist * dst,enum i915_cache_level dst_cache_level,bool dst_is_lmem,struct i915_request ** out)1049 int intel_migrate_copy(struct intel_migrate *m,
1050 struct i915_gem_ww_ctx *ww,
1051 const struct i915_deps *deps,
1052 struct scatterlist *src,
1053 enum i915_cache_level src_cache_level,
1054 bool src_is_lmem,
1055 struct scatterlist *dst,
1056 enum i915_cache_level dst_cache_level,
1057 bool dst_is_lmem,
1058 struct i915_request **out)
1059 {
1060 struct intel_context *ce;
1061 int err;
1062
1063 *out = NULL;
1064 if (!m->context)
1065 return -ENODEV;
1066
1067 ce = intel_migrate_create_context(m);
1068 if (IS_ERR(ce))
1069 ce = intel_context_get(m->context);
1070 GEM_BUG_ON(IS_ERR(ce));
1071
1072 err = intel_context_pin_ww(ce, ww);
1073 if (err)
1074 goto out;
1075
1076 err = intel_context_migrate_copy(ce, deps,
1077 src, src_cache_level, src_is_lmem,
1078 dst, dst_cache_level, dst_is_lmem,
1079 out);
1080
1081 intel_context_unpin(ce);
1082 out:
1083 intel_context_put(ce);
1084 return err;
1085 }
1086
1087 int
intel_migrate_clear(struct intel_migrate * m,struct i915_gem_ww_ctx * ww,const struct i915_deps * deps,struct scatterlist * sg,enum i915_cache_level cache_level,bool is_lmem,u32 value,struct i915_request ** out)1088 intel_migrate_clear(struct intel_migrate *m,
1089 struct i915_gem_ww_ctx *ww,
1090 const struct i915_deps *deps,
1091 struct scatterlist *sg,
1092 enum i915_cache_level cache_level,
1093 bool is_lmem,
1094 u32 value,
1095 struct i915_request **out)
1096 {
1097 struct intel_context *ce;
1098 int err;
1099
1100 *out = NULL;
1101 if (!m->context)
1102 return -ENODEV;
1103
1104 ce = intel_migrate_create_context(m);
1105 if (IS_ERR(ce))
1106 ce = intel_context_get(m->context);
1107 GEM_BUG_ON(IS_ERR(ce));
1108
1109 err = intel_context_pin_ww(ce, ww);
1110 if (err)
1111 goto out;
1112
1113 err = intel_context_migrate_clear(ce, deps, sg, cache_level,
1114 is_lmem, value, out);
1115
1116 intel_context_unpin(ce);
1117 out:
1118 intel_context_put(ce);
1119 return err;
1120 }
1121
intel_migrate_fini(struct intel_migrate * m)1122 void intel_migrate_fini(struct intel_migrate *m)
1123 {
1124 struct intel_context *ce;
1125
1126 ce = fetch_and_zero(&m->context);
1127 if (!ce)
1128 return;
1129
1130 intel_engine_destroy_pinned_context(ce);
1131 }
1132
1133 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1134 #include "selftest_migrate.c"
1135 #endif
1136