1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "intel_context.h"
8 #include "intel_gpu_commands.h"
9 #include "intel_gt.h"
10 #include "intel_gtt.h"
11 #include "intel_migrate.h"
12 #include "intel_ring.h"
13 
14 struct insert_pte_data {
15 	u64 offset;
16 };
17 
18 #define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
19 
20 #define GET_CCS_BYTES(i915, size)	(HAS_FLAT_CCS(i915) ? \
21 					 DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0)
engine_supports_migration(struct intel_engine_cs * engine)22 static bool engine_supports_migration(struct intel_engine_cs *engine)
23 {
24 	if (!engine)
25 		return false;
26 
27 	/*
28 	 * We need the ability to prevent aribtration (MI_ARB_ON_OFF),
29 	 * the ability to write PTE using inline data (MI_STORE_DATA)
30 	 * and of course the ability to do the block transfer (blits).
31 	 */
32 	GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS);
33 
34 	return true;
35 }
36 
xehpsdv_toggle_pdes(struct i915_address_space * vm,struct i915_page_table * pt,void * data)37 static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
38 				struct i915_page_table *pt,
39 				void *data)
40 {
41 	struct insert_pte_data *d = data;
42 
43 	/*
44 	 * Insert a dummy PTE into every PT that will map to LMEM to ensure
45 	 * we have a correctly setup PDE structure for later use.
46 	 */
47 	vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM);
48 	GEM_BUG_ON(!pt->is_compact);
49 	d->offset += SZ_2M;
50 }
51 
xehpsdv_insert_pte(struct i915_address_space * vm,struct i915_page_table * pt,void * data)52 static void xehpsdv_insert_pte(struct i915_address_space *vm,
53 			       struct i915_page_table *pt,
54 			       void *data)
55 {
56 	struct insert_pte_data *d = data;
57 
58 	/*
59 	 * We are playing tricks here, since the actual pt, from the hw
60 	 * pov, is only 256bytes with 32 entries, or 4096bytes with 512
61 	 * entries, but we are still guaranteed that the physical
62 	 * alignment is 64K underneath for the pt, and we are careful
63 	 * not to access the space in the void.
64 	 */
65 	vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM);
66 	d->offset += SZ_64K;
67 }
68 
insert_pte(struct i915_address_space * vm,struct i915_page_table * pt,void * data)69 static void insert_pte(struct i915_address_space *vm,
70 		       struct i915_page_table *pt,
71 		       void *data)
72 {
73 	struct insert_pte_data *d = data;
74 
75 	vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE,
76 			i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0);
77 	d->offset += PAGE_SIZE;
78 }
79 
migrate_vm(struct intel_gt * gt)80 static struct i915_address_space *migrate_vm(struct intel_gt *gt)
81 {
82 	struct i915_vm_pt_stash stash = {};
83 	struct i915_ppgtt *vm;
84 	int err;
85 	int i;
86 
87 	/*
88 	 * We construct a very special VM for use by all migration contexts,
89 	 * it is kept pinned so that it can be used at any time. As we need
90 	 * to pre-allocate the page directories for the migration VM, this
91 	 * limits us to only using a small number of prepared vma.
92 	 *
93 	 * To be able to pipeline and reschedule migration operations while
94 	 * avoiding unnecessary contention on the vm itself, the PTE updates
95 	 * are inline with the blits. All the blits use the same fixed
96 	 * addresses, with the backing store redirection being updated on the
97 	 * fly. Only 2 implicit vma are used for all migration operations.
98 	 *
99 	 * We lay the ppGTT out as:
100 	 *
101 	 *	[0, CHUNK_SZ) -> first object
102 	 *	[CHUNK_SZ, 2 * CHUNK_SZ) -> second object
103 	 *	[2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE
104 	 *
105 	 * By exposing the dma addresses of the page directories themselves
106 	 * within the ppGTT, we are then able to rewrite the PTE prior to use.
107 	 * But the PTE update and subsequent migration operation must be atomic,
108 	 * i.e. within the same non-preemptible window so that we do not switch
109 	 * to another migration context that overwrites the PTE.
110 	 *
111 	 * This changes quite a bit on platforms with HAS_64K_PAGES support,
112 	 * where we instead have three windows, each CHUNK_SIZE in size. The
113 	 * first is reserved for mapping system-memory, and that just uses the
114 	 * 512 entry layout using 4K GTT pages. The other two windows just map
115 	 * lmem pages and must use the new compact 32 entry layout using 64K GTT
116 	 * pages, which ensures we can address any lmem object that the user
117 	 * throws at us. We then also use the xehpsdv_toggle_pdes as a way of
118 	 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
119 	 * compact layout for each of these page-tables, that fall within the
120 	 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
121 	 *
122 	 * We lay the ppGTT out as:
123 	 *
124 	 * [0, CHUNK_SZ) -> first window/object, maps smem
125 	 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src
126 	 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst
127 	 *
128 	 * For the PTE window it's also quite different, since each PTE must
129 	 * point to some 64K page, one for each PT(since it's in lmem), and yet
130 	 * each is only <= 4096bytes, but since the unused space within that PTE
131 	 * range is never touched, this should be fine.
132 	 *
133 	 * So basically each PT now needs 64K of virtual memory, instead of 4K,
134 	 * which looks like:
135 	 *
136 	 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE
137 	 */
138 
139 	vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY);
140 	if (IS_ERR(vm))
141 		return ERR_CAST(vm);
142 
143 	if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
144 		err = -ENODEV;
145 		goto err_vm;
146 	}
147 
148 	if (HAS_64K_PAGES(gt->i915))
149 		stash.pt_sz = I915_GTT_PAGE_SIZE_64K;
150 
151 	/*
152 	 * Each engine instance is assigned its own chunk in the VM, so
153 	 * that we can run multiple instances concurrently
154 	 */
155 	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
156 		struct intel_engine_cs *engine;
157 		u64 base = (u64)i << 32;
158 		struct insert_pte_data d = {};
159 		struct i915_gem_ww_ctx ww;
160 		u64 sz;
161 
162 		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
163 		if (!engine_supports_migration(engine))
164 			continue;
165 
166 		/*
167 		 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
168 		 * 4x2 page directories for source/destination.
169 		 */
170 		if (HAS_64K_PAGES(gt->i915))
171 			sz = 3 * CHUNK_SZ;
172 		else
173 			sz = 2 * CHUNK_SZ;
174 		d.offset = base + sz;
175 
176 		/*
177 		 * We need another page directory setup so that we can write
178 		 * the 8x512 PTE in each chunk.
179 		 */
180 		if (HAS_64K_PAGES(gt->i915))
181 			sz += (sz / SZ_2M) * SZ_64K;
182 		else
183 			sz += (sz >> 12) * sizeof(u64);
184 
185 		err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
186 		if (err)
187 			goto err_vm;
188 
189 		for_i915_gem_ww(&ww, err, true) {
190 			err = i915_vm_lock_objects(&vm->vm, &ww);
191 			if (err)
192 				continue;
193 			err = i915_vm_map_pt_stash(&vm->vm, &stash);
194 			if (err)
195 				continue;
196 
197 			vm->vm.allocate_va_range(&vm->vm, &stash, base, sz);
198 		}
199 		i915_vm_free_pt_stash(&vm->vm, &stash);
200 		if (err)
201 			goto err_vm;
202 
203 		/* Now allow the GPU to rewrite the PTE via its own ppGTT */
204 		if (HAS_64K_PAGES(gt->i915)) {
205 			vm->vm.foreach(&vm->vm, base, d.offset - base,
206 				       xehpsdv_insert_pte, &d);
207 			d.offset = base + CHUNK_SZ;
208 			vm->vm.foreach(&vm->vm,
209 				       d.offset,
210 				       2 * CHUNK_SZ,
211 				       xehpsdv_toggle_pdes, &d);
212 		} else {
213 			vm->vm.foreach(&vm->vm, base, d.offset - base,
214 				       insert_pte, &d);
215 		}
216 	}
217 
218 	return &vm->vm;
219 
220 err_vm:
221 	i915_vm_put(&vm->vm);
222 	return ERR_PTR(err);
223 }
224 
first_copy_engine(struct intel_gt * gt)225 static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
226 {
227 	struct intel_engine_cs *engine;
228 	int i;
229 
230 	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
231 		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
232 		if (engine_supports_migration(engine))
233 			return engine;
234 	}
235 
236 	return NULL;
237 }
238 
pinned_context(struct intel_gt * gt)239 static struct intel_context *pinned_context(struct intel_gt *gt)
240 {
241 	static struct lock_class_key key;
242 	struct intel_engine_cs *engine;
243 	struct i915_address_space *vm;
244 	struct intel_context *ce;
245 
246 	engine = first_copy_engine(gt);
247 	if (!engine)
248 		return ERR_PTR(-ENODEV);
249 
250 	vm = migrate_vm(gt);
251 	if (IS_ERR(vm))
252 		return ERR_CAST(vm);
253 
254 	ce = intel_engine_create_pinned_context(engine, vm, SZ_512K,
255 						I915_GEM_HWS_MIGRATE,
256 						&key, "migrate");
257 	i915_vm_put(vm);
258 	return ce;
259 }
260 
intel_migrate_init(struct intel_migrate * m,struct intel_gt * gt)261 int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
262 {
263 	struct intel_context *ce;
264 
265 	memset(m, 0, sizeof(*m));
266 
267 	ce = pinned_context(gt);
268 	if (IS_ERR(ce))
269 		return PTR_ERR(ce);
270 
271 	m->context = ce;
272 	return 0;
273 }
274 
random_index(unsigned int max)275 static int random_index(unsigned int max)
276 {
277 	return upper_32_bits(mul_u32_u32(get_random_u32(), max));
278 }
279 
__migrate_engines(struct intel_gt * gt)280 static struct intel_context *__migrate_engines(struct intel_gt *gt)
281 {
282 	struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
283 	struct intel_engine_cs *engine;
284 	unsigned int count, i;
285 
286 	count = 0;
287 	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
288 		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
289 		if (engine_supports_migration(engine))
290 			engines[count++] = engine;
291 	}
292 
293 	return intel_context_create(engines[random_index(count)]);
294 }
295 
intel_migrate_create_context(struct intel_migrate * m)296 struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
297 {
298 	struct intel_context *ce;
299 
300 	/*
301 	 * We randomly distribute contexts across the engines upon constrction,
302 	 * as they all share the same pinned vm, and so in order to allow
303 	 * multiple blits to run in parallel, we must construct each blit
304 	 * to use a different range of the vm for its GTT. This has to be
305 	 * known at construction, so we can not use the late greedy load
306 	 * balancing of the virtual-engine.
307 	 */
308 	ce = __migrate_engines(m->context->engine->gt);
309 	if (IS_ERR(ce))
310 		return ce;
311 
312 	ce->ring = NULL;
313 	ce->ring_size = SZ_256K;
314 
315 	i915_vm_put(ce->vm);
316 	ce->vm = i915_vm_get(m->context->vm);
317 
318 	return ce;
319 }
320 
sg_sgt(struct scatterlist * sg)321 static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
322 {
323 	dma_addr_t addr = sg_dma_address(sg);
324 
325 	return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
326 }
327 
emit_no_arbitration(struct i915_request * rq)328 static int emit_no_arbitration(struct i915_request *rq)
329 {
330 	u32 *cs;
331 
332 	cs = intel_ring_begin(rq, 2);
333 	if (IS_ERR(cs))
334 		return PTR_ERR(cs);
335 
336 	/* Explicitly disable preemption for this request. */
337 	*cs++ = MI_ARB_ON_OFF;
338 	*cs++ = MI_NOOP;
339 	intel_ring_advance(rq, cs);
340 
341 	return 0;
342 }
343 
max_pte_pkt_size(struct i915_request * rq,int pkt)344 static int max_pte_pkt_size(struct i915_request *rq, int pkt)
345 {
346 	struct intel_ring *ring = rq->ring;
347 
348 	pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5);
349 	pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5);
350 
351 	return pkt;
352 }
353 
emit_pte(struct i915_request * rq,struct sgt_dma * it,enum i915_cache_level cache_level,bool is_lmem,u64 offset,int length)354 static int emit_pte(struct i915_request *rq,
355 		    struct sgt_dma *it,
356 		    enum i915_cache_level cache_level,
357 		    bool is_lmem,
358 		    u64 offset,
359 		    int length)
360 {
361 	bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915);
362 	const u64 encode = rq->context->vm->pte_encode(0, cache_level,
363 						       is_lmem ? PTE_LM : 0);
364 	struct intel_ring *ring = rq->ring;
365 	int pkt, dword_length;
366 	u32 total = 0;
367 	u32 page_size;
368 	u32 *hdr, *cs;
369 
370 	GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8);
371 
372 	page_size = I915_GTT_PAGE_SIZE;
373 	dword_length = 0x400;
374 
375 	/* Compute the page directory offset for the target address range */
376 	if (has_64K_pages) {
377 		GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M));
378 
379 		offset /= SZ_2M;
380 		offset *= SZ_64K;
381 		offset += 3 * CHUNK_SZ;
382 
383 		if (is_lmem) {
384 			page_size = I915_GTT_PAGE_SIZE_64K;
385 			dword_length = 0x40;
386 		}
387 	} else {
388 		offset >>= 12;
389 		offset *= sizeof(u64);
390 		offset += 2 * CHUNK_SZ;
391 	}
392 
393 	offset += (u64)rq->engine->instance << 32;
394 
395 	cs = intel_ring_begin(rq, 6);
396 	if (IS_ERR(cs))
397 		return PTR_ERR(cs);
398 
399 	/* Pack as many PTE updates as possible into a single MI command */
400 	pkt = max_pte_pkt_size(rq, dword_length);
401 
402 	hdr = cs;
403 	*cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */
404 	*cs++ = lower_32_bits(offset);
405 	*cs++ = upper_32_bits(offset);
406 
407 	do {
408 		if (cs - hdr >= pkt) {
409 			int dword_rem;
410 
411 			*hdr += cs - hdr - 2;
412 			*cs++ = MI_NOOP;
413 
414 			ring->emit = (void *)cs - ring->vaddr;
415 			intel_ring_advance(rq, cs);
416 			intel_ring_update_space(ring);
417 
418 			cs = intel_ring_begin(rq, 6);
419 			if (IS_ERR(cs))
420 				return PTR_ERR(cs);
421 
422 			dword_rem = dword_length;
423 			if (has_64K_pages) {
424 				if (IS_ALIGNED(total, SZ_2M)) {
425 					offset = round_up(offset, SZ_64K);
426 				} else {
427 					dword_rem = SZ_2M - (total & (SZ_2M - 1));
428 					dword_rem /= page_size;
429 					dword_rem *= 2;
430 				}
431 			}
432 
433 			pkt = max_pte_pkt_size(rq, dword_rem);
434 
435 			hdr = cs;
436 			*cs++ = MI_STORE_DATA_IMM | REG_BIT(21);
437 			*cs++ = lower_32_bits(offset);
438 			*cs++ = upper_32_bits(offset);
439 		}
440 
441 		GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size));
442 
443 		*cs++ = lower_32_bits(encode | it->dma);
444 		*cs++ = upper_32_bits(encode | it->dma);
445 
446 		offset += 8;
447 		total += page_size;
448 
449 		it->dma += page_size;
450 		if (it->dma >= it->max) {
451 			it->sg = __sg_next(it->sg);
452 			if (!it->sg || sg_dma_len(it->sg) == 0)
453 				break;
454 
455 			it->dma = sg_dma_address(it->sg);
456 			it->max = it->dma + sg_dma_len(it->sg);
457 		}
458 	} while (total < length);
459 
460 	*hdr += cs - hdr - 2;
461 	*cs++ = MI_NOOP;
462 
463 	ring->emit = (void *)cs - ring->vaddr;
464 	intel_ring_advance(rq, cs);
465 	intel_ring_update_space(ring);
466 
467 	return total;
468 }
469 
wa_1209644611_applies(int ver,u32 size)470 static bool wa_1209644611_applies(int ver, u32 size)
471 {
472 	u32 height = size >> PAGE_SHIFT;
473 
474 	if (ver != 11)
475 		return false;
476 
477 	return height % 4 == 3 && height <= 8;
478 }
479 
480 /**
481  * DOC: Flat-CCS - Memory compression for Local memory
482  *
483  * On Xe-HP and later devices, we use dedicated compression control state (CCS)
484  * stored in local memory for each surface, to support the 3D and media
485  * compression formats.
486  *
487  * The memory required for the CCS of the entire local memory is 1/256 of the
488  * local memory size. So before the kernel boot, the required memory is reserved
489  * for the CCS data and a secure register will be programmed with the CCS base
490  * address.
491  *
492  * Flat CCS data needs to be cleared when a lmem object is allocated.
493  * And CCS data can be copied in and out of CCS region through
494  * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly.
495  *
496  * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
497  * its preference list, on memory pressure, i915 needs to migrate the lmem
498  * content into smem. If the lmem object is Flat-CCS compressed by userspace,
499  * then i915 needs to decompress it. But I915 lack the required information
500  * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
501  *
502  * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
503  * be temporarily evicted to smem, along with the auxiliary CCS state, where
504  * it can be potentially swapped-out at a later point, if required.
505  * If userspace later touches the evicted pages, then we always move
506  * the backing memory back to lmem, which includes restoring the saved CCS state,
507  * and potentially performing any required swap-in.
508  *
509  * For the migration of the lmem objects with smem in placement list, such as
510  * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
511  */
512 
i915_flush_dw(u32 * cmd,u32 flags)513 static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
514 {
515 	*cmd++ = MI_FLUSH_DW | flags;
516 	*cmd++ = 0;
517 	*cmd++ = 0;
518 
519 	return cmd;
520 }
521 
emit_copy_ccs(struct i915_request * rq,u32 dst_offset,u8 dst_access,u32 src_offset,u8 src_access,int size)522 static int emit_copy_ccs(struct i915_request *rq,
523 			 u32 dst_offset, u8 dst_access,
524 			 u32 src_offset, u8 src_access, int size)
525 {
526 	struct drm_i915_private *i915 = rq->engine->i915;
527 	int mocs = rq->engine->gt->mocs.uc_index << 1;
528 	u32 num_ccs_blks;
529 	u32 *cs;
530 
531 	cs = intel_ring_begin(rq, 12);
532 	if (IS_ERR(cs))
533 		return PTR_ERR(cs);
534 
535 	num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
536 				    NUM_CCS_BYTES_PER_BLOCK);
537 	GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER);
538 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
539 
540 	/*
541 	 * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS
542 	 * data in and out of the CCS region.
543 	 *
544 	 * We can copy at most 1024 blocks of 256 bytes using one
545 	 * XY_CTRL_SURF_COPY_BLT instruction.
546 	 *
547 	 * In case we need to copy more than 1024 blocks, we need to add
548 	 * another instruction to the same batch buffer.
549 	 *
550 	 * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS.
551 	 *
552 	 * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM.
553 	 */
554 	*cs++ = XY_CTRL_SURF_COPY_BLT |
555 		src_access << SRC_ACCESS_TYPE_SHIFT |
556 		dst_access << DST_ACCESS_TYPE_SHIFT |
557 		((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT;
558 	*cs++ = src_offset;
559 	*cs++ = rq->engine->instance |
560 		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
561 	*cs++ = dst_offset;
562 	*cs++ = rq->engine->instance |
563 		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
564 
565 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
566 	*cs++ = MI_NOOP;
567 
568 	intel_ring_advance(rq, cs);
569 
570 	return 0;
571 }
572 
emit_copy(struct i915_request * rq,u32 dst_offset,u32 src_offset,int size)573 static int emit_copy(struct i915_request *rq,
574 		     u32 dst_offset, u32 src_offset, int size)
575 {
576 	const int ver = GRAPHICS_VER(rq->engine->i915);
577 	u32 instance = rq->engine->instance;
578 	u32 *cs;
579 
580 	cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);
581 	if (IS_ERR(cs))
582 		return PTR_ERR(cs);
583 
584 	if (ver >= 9 && !wa_1209644611_applies(ver, size)) {
585 		*cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
586 		*cs++ = BLT_DEPTH_32 | PAGE_SIZE;
587 		*cs++ = 0;
588 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
589 		*cs++ = dst_offset;
590 		*cs++ = instance;
591 		*cs++ = 0;
592 		*cs++ = PAGE_SIZE;
593 		*cs++ = src_offset;
594 		*cs++ = instance;
595 	} else if (ver >= 8) {
596 		*cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
597 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
598 		*cs++ = 0;
599 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
600 		*cs++ = dst_offset;
601 		*cs++ = instance;
602 		*cs++ = 0;
603 		*cs++ = PAGE_SIZE;
604 		*cs++ = src_offset;
605 		*cs++ = instance;
606 	} else {
607 		GEM_BUG_ON(instance);
608 		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
609 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
610 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
611 		*cs++ = dst_offset;
612 		*cs++ = PAGE_SIZE;
613 		*cs++ = src_offset;
614 	}
615 
616 	intel_ring_advance(rq, cs);
617 	return 0;
618 }
619 
scatter_list_length(struct scatterlist * sg)620 static u64 scatter_list_length(struct scatterlist *sg)
621 {
622 	u64 len = 0;
623 
624 	while (sg && sg_dma_len(sg)) {
625 		len += sg_dma_len(sg);
626 		sg = sg_next(sg);
627 	}
628 
629 	return len;
630 }
631 
632 static int
calculate_chunk_sz(struct drm_i915_private * i915,bool src_is_lmem,u64 bytes_to_cpy,u64 ccs_bytes_to_cpy)633 calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem,
634 		   u64 bytes_to_cpy, u64 ccs_bytes_to_cpy)
635 {
636 	if (ccs_bytes_to_cpy && !src_is_lmem)
637 		/*
638 		 * When CHUNK_SZ is passed all the pages upto CHUNK_SZ
639 		 * will be taken for the blt. in Flat-ccs supported
640 		 * platform Smem obj will have more pages than required
641 		 * for main meory hence limit it to the required size
642 		 * for main memory
643 		 */
644 		return min_t(u64, bytes_to_cpy, CHUNK_SZ);
645 	else
646 		return CHUNK_SZ;
647 }
648 
get_ccs_sg_sgt(struct sgt_dma * it,u64 bytes_to_cpy)649 static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy)
650 {
651 	u64 len;
652 
653 	do {
654 		GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg));
655 		len = it->max - it->dma;
656 		if (len > bytes_to_cpy) {
657 			it->dma += bytes_to_cpy;
658 			break;
659 		}
660 
661 		bytes_to_cpy -= len;
662 
663 		it->sg = __sg_next(it->sg);
664 		it->dma = sg_dma_address(it->sg);
665 		it->max = it->dma + sg_dma_len(it->sg);
666 	} while (bytes_to_cpy);
667 }
668 
669 int
intel_context_migrate_copy(struct intel_context * ce,const struct i915_deps * deps,struct scatterlist * src,enum i915_cache_level src_cache_level,bool src_is_lmem,struct scatterlist * dst,enum i915_cache_level dst_cache_level,bool dst_is_lmem,struct i915_request ** out)670 intel_context_migrate_copy(struct intel_context *ce,
671 			   const struct i915_deps *deps,
672 			   struct scatterlist *src,
673 			   enum i915_cache_level src_cache_level,
674 			   bool src_is_lmem,
675 			   struct scatterlist *dst,
676 			   enum i915_cache_level dst_cache_level,
677 			   bool dst_is_lmem,
678 			   struct i915_request **out)
679 {
680 	struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs;
681 	struct drm_i915_private *i915 = ce->engine->i915;
682 	u64 ccs_bytes_to_cpy = 0, bytes_to_cpy;
683 	enum i915_cache_level ccs_cache_level;
684 	u32 src_offset, dst_offset;
685 	u8 src_access, dst_access;
686 	struct i915_request *rq;
687 	u64 src_sz, dst_sz;
688 	bool ccs_is_src, overwrite_ccs;
689 	int err;
690 
691 	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
692 	GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem));
693 	*out = NULL;
694 
695 	GEM_BUG_ON(ce->ring->size < SZ_64K);
696 
697 	src_sz = scatter_list_length(src);
698 	bytes_to_cpy = src_sz;
699 
700 	if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) {
701 		src_access = !src_is_lmem && dst_is_lmem;
702 		dst_access = !src_access;
703 
704 		dst_sz = scatter_list_length(dst);
705 		if (src_is_lmem) {
706 			it_ccs = it_dst;
707 			ccs_cache_level = dst_cache_level;
708 			ccs_is_src = false;
709 		} else if (dst_is_lmem) {
710 			bytes_to_cpy = dst_sz;
711 			it_ccs = it_src;
712 			ccs_cache_level = src_cache_level;
713 			ccs_is_src = true;
714 		}
715 
716 		/*
717 		 * When there is a eviction of ccs needed smem will have the
718 		 * extra pages for the ccs data
719 		 *
720 		 * TO-DO: Want to move the size mismatch check to a WARN_ON,
721 		 * but still we have some requests of smem->lmem with same size.
722 		 * Need to fix it.
723 		 */
724 		ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
725 		if (ccs_bytes_to_cpy)
726 			get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);
727 	}
728 
729 	overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem;
730 
731 	src_offset = 0;
732 	dst_offset = CHUNK_SZ;
733 	if (HAS_64K_PAGES(ce->engine->i915)) {
734 		src_offset = 0;
735 		dst_offset = 0;
736 		if (src_is_lmem)
737 			src_offset = CHUNK_SZ;
738 		if (dst_is_lmem)
739 			dst_offset = 2 * CHUNK_SZ;
740 	}
741 
742 	do {
743 		int len;
744 
745 		rq = i915_request_create(ce);
746 		if (IS_ERR(rq)) {
747 			err = PTR_ERR(rq);
748 			goto out_ce;
749 		}
750 
751 		if (deps) {
752 			err = i915_request_await_deps(rq, deps);
753 			if (err)
754 				goto out_rq;
755 
756 			if (rq->engine->emit_init_breadcrumb) {
757 				err = rq->engine->emit_init_breadcrumb(rq);
758 				if (err)
759 					goto out_rq;
760 			}
761 
762 			deps = NULL;
763 		}
764 
765 		/* The PTE updates + copy must not be interrupted. */
766 		err = emit_no_arbitration(rq);
767 		if (err)
768 			goto out_rq;
769 
770 		src_sz = calculate_chunk_sz(i915, src_is_lmem,
771 					    bytes_to_cpy, ccs_bytes_to_cpy);
772 
773 		len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem,
774 			       src_offset, src_sz);
775 		if (!len) {
776 			err = -EINVAL;
777 			goto out_rq;
778 		}
779 		if (len < 0) {
780 			err = len;
781 			goto out_rq;
782 		}
783 
784 		err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem,
785 			       dst_offset, len);
786 		if (err < 0)
787 			goto out_rq;
788 		if (err < len) {
789 			err = -EINVAL;
790 			goto out_rq;
791 		}
792 
793 		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
794 		if (err)
795 			goto out_rq;
796 
797 		err = emit_copy(rq, dst_offset,	src_offset, len);
798 		if (err)
799 			goto out_rq;
800 
801 		bytes_to_cpy -= len;
802 
803 		if (ccs_bytes_to_cpy) {
804 			int ccs_sz;
805 
806 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
807 			if (err)
808 				goto out_rq;
809 
810 			ccs_sz = GET_CCS_BYTES(i915, len);
811 			err = emit_pte(rq, &it_ccs, ccs_cache_level, false,
812 				       ccs_is_src ? src_offset : dst_offset,
813 				       ccs_sz);
814 			if (err < 0)
815 				goto out_rq;
816 			if (err < ccs_sz) {
817 				err = -EINVAL;
818 				goto out_rq;
819 			}
820 
821 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
822 			if (err)
823 				goto out_rq;
824 
825 			err = emit_copy_ccs(rq, dst_offset, dst_access,
826 					    src_offset, src_access, len);
827 			if (err)
828 				goto out_rq;
829 
830 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
831 			if (err)
832 				goto out_rq;
833 			ccs_bytes_to_cpy -= ccs_sz;
834 		} else if (overwrite_ccs) {
835 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
836 			if (err)
837 				goto out_rq;
838 
839 			/*
840 			 * While we can't always restore/manage the CCS state,
841 			 * we still need to ensure we don't leak the CCS state
842 			 * from the previous user, so make sure we overwrite it
843 			 * with something.
844 			 */
845 			err = emit_copy_ccs(rq, dst_offset, INDIRECT_ACCESS,
846 					    dst_offset, DIRECT_ACCESS, len);
847 			if (err)
848 				goto out_rq;
849 
850 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
851 			if (err)
852 				goto out_rq;
853 		}
854 
855 		/* Arbitration is re-enabled between requests. */
856 out_rq:
857 		if (*out)
858 			i915_request_put(*out);
859 		*out = i915_request_get(rq);
860 		i915_request_add(rq);
861 
862 		if (err)
863 			break;
864 
865 		if (!bytes_to_cpy && !ccs_bytes_to_cpy) {
866 			if (src_is_lmem)
867 				WARN_ON(it_src.sg && sg_dma_len(it_src.sg));
868 			else
869 				WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg));
870 			break;
871 		}
872 
873 		if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) ||
874 			    !it_dst.sg || !sg_dma_len(it_dst.sg) ||
875 			    (ccs_bytes_to_cpy && (!it_ccs.sg ||
876 						  !sg_dma_len(it_ccs.sg))))) {
877 			err = -EINVAL;
878 			break;
879 		}
880 
881 		cond_resched();
882 	} while (1);
883 
884 out_ce:
885 	return err;
886 }
887 
emit_clear(struct i915_request * rq,u32 offset,int size,u32 value,bool is_lmem)888 static int emit_clear(struct i915_request *rq, u32 offset, int size,
889 		      u32 value, bool is_lmem)
890 {
891 	struct drm_i915_private *i915 = rq->engine->i915;
892 	int mocs = rq->engine->gt->mocs.uc_index << 1;
893 	const int ver = GRAPHICS_VER(i915);
894 	int ring_sz;
895 	u32 *cs;
896 
897 	GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
898 
899 	if (HAS_FLAT_CCS(i915) && ver >= 12)
900 		ring_sz = XY_FAST_COLOR_BLT_DW;
901 	else if (ver >= 8)
902 		ring_sz = 8;
903 	else
904 		ring_sz = 6;
905 
906 	cs = intel_ring_begin(rq, ring_sz);
907 	if (IS_ERR(cs))
908 		return PTR_ERR(cs);
909 
910 	if (HAS_FLAT_CCS(i915) && ver >= 12) {
911 		*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
912 			(XY_FAST_COLOR_BLT_DW - 2);
913 		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
914 			(PAGE_SIZE - 1);
915 		*cs++ = 0;
916 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
917 		*cs++ = offset;
918 		*cs++ = rq->engine->instance;
919 		*cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
920 		/* BG7 */
921 		*cs++ = value;
922 		*cs++ = 0;
923 		*cs++ = 0;
924 		*cs++ = 0;
925 		/* BG11 */
926 		*cs++ = 0;
927 		*cs++ = 0;
928 		/* BG13 */
929 		*cs++ = 0;
930 		*cs++ = 0;
931 		*cs++ = 0;
932 	} else if (ver >= 8) {
933 		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);
934 		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
935 		*cs++ = 0;
936 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
937 		*cs++ = offset;
938 		*cs++ = rq->engine->instance;
939 		*cs++ = value;
940 		*cs++ = MI_NOOP;
941 	} else {
942 		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
943 		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
944 		*cs++ = 0;
945 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
946 		*cs++ = offset;
947 		*cs++ = value;
948 	}
949 
950 	intel_ring_advance(rq, cs);
951 	return 0;
952 }
953 
954 int
intel_context_migrate_clear(struct intel_context * ce,const struct i915_deps * deps,struct scatterlist * sg,enum i915_cache_level cache_level,bool is_lmem,u32 value,struct i915_request ** out)955 intel_context_migrate_clear(struct intel_context *ce,
956 			    const struct i915_deps *deps,
957 			    struct scatterlist *sg,
958 			    enum i915_cache_level cache_level,
959 			    bool is_lmem,
960 			    u32 value,
961 			    struct i915_request **out)
962 {
963 	struct drm_i915_private *i915 = ce->engine->i915;
964 	struct sgt_dma it = sg_sgt(sg);
965 	struct i915_request *rq;
966 	u32 offset;
967 	int err;
968 
969 	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
970 	*out = NULL;
971 
972 	GEM_BUG_ON(ce->ring->size < SZ_64K);
973 
974 	offset = 0;
975 	if (HAS_64K_PAGES(i915) && is_lmem)
976 		offset = CHUNK_SZ;
977 
978 	do {
979 		int len;
980 
981 		rq = i915_request_create(ce);
982 		if (IS_ERR(rq)) {
983 			err = PTR_ERR(rq);
984 			goto out_ce;
985 		}
986 
987 		if (deps) {
988 			err = i915_request_await_deps(rq, deps);
989 			if (err)
990 				goto out_rq;
991 
992 			if (rq->engine->emit_init_breadcrumb) {
993 				err = rq->engine->emit_init_breadcrumb(rq);
994 				if (err)
995 					goto out_rq;
996 			}
997 
998 			deps = NULL;
999 		}
1000 
1001 		/* The PTE updates + clear must not be interrupted. */
1002 		err = emit_no_arbitration(rq);
1003 		if (err)
1004 			goto out_rq;
1005 
1006 		len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ);
1007 		if (len <= 0) {
1008 			err = len;
1009 			goto out_rq;
1010 		}
1011 
1012 		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1013 		if (err)
1014 			goto out_rq;
1015 
1016 		err = emit_clear(rq, offset, len, value, is_lmem);
1017 		if (err)
1018 			goto out_rq;
1019 
1020 		if (HAS_FLAT_CCS(i915) && is_lmem && !value) {
1021 			/*
1022 			 * copy the content of memory into corresponding
1023 			 * ccs surface
1024 			 */
1025 			err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset,
1026 					    DIRECT_ACCESS, len);
1027 			if (err)
1028 				goto out_rq;
1029 		}
1030 
1031 		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1032 
1033 		/* Arbitration is re-enabled between requests. */
1034 out_rq:
1035 		if (*out)
1036 			i915_request_put(*out);
1037 		*out = i915_request_get(rq);
1038 		i915_request_add(rq);
1039 		if (err || !it.sg || !sg_dma_len(it.sg))
1040 			break;
1041 
1042 		cond_resched();
1043 	} while (1);
1044 
1045 out_ce:
1046 	return err;
1047 }
1048 
intel_migrate_copy(struct intel_migrate * m,struct i915_gem_ww_ctx * ww,const struct i915_deps * deps,struct scatterlist * src,enum i915_cache_level src_cache_level,bool src_is_lmem,struct scatterlist * dst,enum i915_cache_level dst_cache_level,bool dst_is_lmem,struct i915_request ** out)1049 int intel_migrate_copy(struct intel_migrate *m,
1050 		       struct i915_gem_ww_ctx *ww,
1051 		       const struct i915_deps *deps,
1052 		       struct scatterlist *src,
1053 		       enum i915_cache_level src_cache_level,
1054 		       bool src_is_lmem,
1055 		       struct scatterlist *dst,
1056 		       enum i915_cache_level dst_cache_level,
1057 		       bool dst_is_lmem,
1058 		       struct i915_request **out)
1059 {
1060 	struct intel_context *ce;
1061 	int err;
1062 
1063 	*out = NULL;
1064 	if (!m->context)
1065 		return -ENODEV;
1066 
1067 	ce = intel_migrate_create_context(m);
1068 	if (IS_ERR(ce))
1069 		ce = intel_context_get(m->context);
1070 	GEM_BUG_ON(IS_ERR(ce));
1071 
1072 	err = intel_context_pin_ww(ce, ww);
1073 	if (err)
1074 		goto out;
1075 
1076 	err = intel_context_migrate_copy(ce, deps,
1077 					 src, src_cache_level, src_is_lmem,
1078 					 dst, dst_cache_level, dst_is_lmem,
1079 					 out);
1080 
1081 	intel_context_unpin(ce);
1082 out:
1083 	intel_context_put(ce);
1084 	return err;
1085 }
1086 
1087 int
intel_migrate_clear(struct intel_migrate * m,struct i915_gem_ww_ctx * ww,const struct i915_deps * deps,struct scatterlist * sg,enum i915_cache_level cache_level,bool is_lmem,u32 value,struct i915_request ** out)1088 intel_migrate_clear(struct intel_migrate *m,
1089 		    struct i915_gem_ww_ctx *ww,
1090 		    const struct i915_deps *deps,
1091 		    struct scatterlist *sg,
1092 		    enum i915_cache_level cache_level,
1093 		    bool is_lmem,
1094 		    u32 value,
1095 		    struct i915_request **out)
1096 {
1097 	struct intel_context *ce;
1098 	int err;
1099 
1100 	*out = NULL;
1101 	if (!m->context)
1102 		return -ENODEV;
1103 
1104 	ce = intel_migrate_create_context(m);
1105 	if (IS_ERR(ce))
1106 		ce = intel_context_get(m->context);
1107 	GEM_BUG_ON(IS_ERR(ce));
1108 
1109 	err = intel_context_pin_ww(ce, ww);
1110 	if (err)
1111 		goto out;
1112 
1113 	err = intel_context_migrate_clear(ce, deps, sg, cache_level,
1114 					  is_lmem, value, out);
1115 
1116 	intel_context_unpin(ce);
1117 out:
1118 	intel_context_put(ce);
1119 	return err;
1120 }
1121 
intel_migrate_fini(struct intel_migrate * m)1122 void intel_migrate_fini(struct intel_migrate *m)
1123 {
1124 	struct intel_context *ce;
1125 
1126 	ce = fetch_and_zero(&m->context);
1127 	if (!ce)
1128 		return;
1129 
1130 	intel_engine_destroy_pinned_context(ce);
1131 }
1132 
1133 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1134 #include "selftest_migrate.c"
1135 #endif
1136