1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include <asm/set_memory.h>
7 #include <asm/smp.h>
8 #include <linux/types.h>
9 #include <linux/stop_machine.h>
10 
11 #include <drm/drm_managed.h>
12 #include <drm/i915_drm.h>
13 #include <drm/intel-gtt.h>
14 
15 #include "display/intel_display.h"
16 #include "gem/i915_gem_lmem.h"
17 
18 #include "intel_ggtt_gmch.h"
19 #include "intel_gt.h"
20 #include "intel_gt_regs.h"
21 #include "intel_pci_config.h"
22 #include "i915_drv.h"
23 #include "i915_pci.h"
24 #include "i915_scatterlist.h"
25 #include "i915_utils.h"
26 #include "i915_vgpu.h"
27 
28 #include "intel_gtt.h"
29 #include "gen8_ppgtt.h"
30 
i915_ggtt_color_adjust(const struct drm_mm_node * node,unsigned long color,u64 * start,u64 * end)31 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
32 				   unsigned long color,
33 				   u64 *start,
34 				   u64 *end)
35 {
36 	if (i915_node_color_differs(node, color))
37 		*start += I915_GTT_PAGE_SIZE;
38 
39 	/*
40 	 * Also leave a space between the unallocated reserved node after the
41 	 * GTT and any objects within the GTT, i.e. we use the color adjustment
42 	 * to insert a guard page to prevent prefetches crossing over the
43 	 * GTT boundary.
44 	 */
45 	node = list_next_entry(node, node_list);
46 	if (node->color != color)
47 		*end -= I915_GTT_PAGE_SIZE;
48 }
49 
ggtt_init_hw(struct i915_ggtt * ggtt)50 static int ggtt_init_hw(struct i915_ggtt *ggtt)
51 {
52 	struct drm_i915_private *i915 = ggtt->vm.i915;
53 
54 	i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
55 
56 	ggtt->vm.is_ggtt = true;
57 
58 	/* Only VLV supports read-only GGTT mappings */
59 	ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
60 
61 	if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
62 		ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
63 
64 	if (ggtt->mappable_end) {
65 		if (!io_mapping_init_wc(&ggtt->iomap,
66 					ggtt->gmadr.start,
67 					ggtt->mappable_end)) {
68 			ggtt->vm.cleanup(&ggtt->vm);
69 			return -EIO;
70 		}
71 
72 		ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
73 					      ggtt->mappable_end);
74 	}
75 
76 	intel_ggtt_init_fences(ggtt);
77 
78 	return 0;
79 }
80 
81 /**
82  * i915_ggtt_init_hw - Initialize GGTT hardware
83  * @i915: i915 device
84  */
i915_ggtt_init_hw(struct drm_i915_private * i915)85 int i915_ggtt_init_hw(struct drm_i915_private *i915)
86 {
87 	int ret;
88 
89 	/*
90 	 * Note that we use page colouring to enforce a guard page at the
91 	 * end of the address space. This is required as the CS may prefetch
92 	 * beyond the end of the batch buffer, across the page boundary,
93 	 * and beyond the end of the GTT if we do not provide a guard.
94 	 */
95 	ret = ggtt_init_hw(to_gt(i915)->ggtt);
96 	if (ret)
97 		return ret;
98 
99 	return 0;
100 }
101 
102 /**
103  * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
104  * @vm: The VM to suspend the mappings for
105  *
106  * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
107  * DPT page table.
108  */
i915_ggtt_suspend_vm(struct i915_address_space * vm)109 void i915_ggtt_suspend_vm(struct i915_address_space *vm)
110 {
111 	struct i915_vma *vma, *vn;
112 	int save_skip_rewrite;
113 
114 	drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
115 
116 retry:
117 	i915_gem_drain_freed_objects(vm->i915);
118 
119 	mutex_lock(&vm->mutex);
120 
121 	/*
122 	 * Skip rewriting PTE on VMA unbind.
123 	 * FIXME: Use an argument to i915_vma_unbind() instead?
124 	 */
125 	save_skip_rewrite = vm->skip_pte_rewrite;
126 	vm->skip_pte_rewrite = true;
127 
128 	list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
129 		struct drm_i915_gem_object *obj = vma->obj;
130 
131 		GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
132 
133 		if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
134 			continue;
135 
136 		/* unlikely to race when GPU is idle, so no worry about slowpath.. */
137 		if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {
138 			/*
139 			 * No dead objects should appear here, GPU should be
140 			 * completely idle, and userspace suspended
141 			 */
142 			i915_gem_object_get(obj);
143 
144 			mutex_unlock(&vm->mutex);
145 
146 			i915_gem_object_lock(obj, NULL);
147 			GEM_WARN_ON(i915_vma_unbind(vma));
148 			i915_gem_object_unlock(obj);
149 			i915_gem_object_put(obj);
150 
151 			vm->skip_pte_rewrite = save_skip_rewrite;
152 			goto retry;
153 		}
154 
155 		if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
156 			i915_vma_wait_for_bind(vma);
157 
158 			__i915_vma_evict(vma, false);
159 			drm_mm_remove_node(&vma->node);
160 		}
161 
162 		i915_gem_object_unlock(obj);
163 	}
164 
165 	vm->clear_range(vm, 0, vm->total);
166 
167 	vm->skip_pte_rewrite = save_skip_rewrite;
168 
169 	mutex_unlock(&vm->mutex);
170 }
171 
i915_ggtt_suspend(struct i915_ggtt * ggtt)172 void i915_ggtt_suspend(struct i915_ggtt *ggtt)
173 {
174 	struct intel_gt *gt;
175 
176 	i915_ggtt_suspend_vm(&ggtt->vm);
177 	ggtt->invalidate(ggtt);
178 
179 	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
180 		intel_gt_check_and_clear_faults(gt);
181 }
182 
gen6_ggtt_invalidate(struct i915_ggtt * ggtt)183 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
184 {
185 	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
186 
187 	spin_lock_irq(&uncore->lock);
188 	intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
189 	intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
190 	spin_unlock_irq(&uncore->lock);
191 }
192 
needs_wc_ggtt_mapping(struct drm_i915_private * i915)193 static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915)
194 {
195 	/*
196 	 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
197 	 * will be dropped. For WC mappings in general we have 64 byte burst
198 	 * writes when the WC buffer is flushed, so we can't use it, but have to
199 	 * resort to an uncached mapping. The WC issue is easily caught by the
200 	 * readback check when writing GTT PTE entries.
201 	 */
202 	if (!IS_GEN9_LP(i915) && GRAPHICS_VER(i915) < 11)
203 		return true;
204 
205 	return false;
206 }
207 
gen8_ggtt_invalidate(struct i915_ggtt * ggtt)208 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
209 {
210 	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
211 
212 	/*
213 	 * Note that as an uncached mmio write, this will flush the
214 	 * WCB of the writes into the GGTT before it triggers the invalidate.
215 	 *
216 	 * Only perform this when GGTT is mapped as WC, see ggtt_probe_common().
217 	 */
218 	if (needs_wc_ggtt_mapping(ggtt->vm.i915))
219 		intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6,
220 				      GFX_FLSH_CNTL_EN);
221 }
222 
guc_ggtt_invalidate(struct i915_ggtt * ggtt)223 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
224 {
225 	struct drm_i915_private *i915 = ggtt->vm.i915;
226 
227 	gen8_ggtt_invalidate(ggtt);
228 
229 	if (GRAPHICS_VER(i915) >= 12) {
230 		struct intel_gt *gt;
231 
232 		list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
233 			intel_uncore_write_fw(gt->uncore,
234 					      GEN12_GUC_TLB_INV_CR,
235 					      GEN12_GUC_TLB_INV_CR_INVALIDATE);
236 	} else {
237 		intel_uncore_write_fw(ggtt->vm.gt->uncore,
238 				      GEN8_GTCR, GEN8_GTCR_INVALIDATE);
239 	}
240 }
241 
mtl_ggtt_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)242 static u64 mtl_ggtt_pte_encode(dma_addr_t addr,
243 			       unsigned int pat_index,
244 			       u32 flags)
245 {
246 	gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
247 
248 	WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK);
249 
250 	if (flags & PTE_LM)
251 		pte |= GEN12_GGTT_PTE_LM;
252 
253 	if (pat_index & BIT(0))
254 		pte |= MTL_GGTT_PTE_PAT0;
255 
256 	if (pat_index & BIT(1))
257 		pte |= MTL_GGTT_PTE_PAT1;
258 
259 	return pte;
260 }
261 
gen8_ggtt_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)262 u64 gen8_ggtt_pte_encode(dma_addr_t addr,
263 			 unsigned int pat_index,
264 			 u32 flags)
265 {
266 	gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
267 
268 	if (flags & PTE_LM)
269 		pte |= GEN12_GGTT_PTE_LM;
270 
271 	return pte;
272 }
273 
gen8_set_pte(void __iomem * addr,gen8_pte_t pte)274 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
275 {
276 	writeq(pte, addr);
277 }
278 
gen8_ggtt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 flags)279 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
280 				  dma_addr_t addr,
281 				  u64 offset,
282 				  unsigned int pat_index,
283 				  u32 flags)
284 {
285 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
286 	gen8_pte_t __iomem *pte =
287 		(gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
288 
289 	gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags));
290 
291 	ggtt->invalidate(ggtt);
292 }
293 
gen8_ggtt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)294 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
295 				     struct i915_vma_resource *vma_res,
296 				     unsigned int pat_index,
297 				     u32 flags)
298 {
299 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
300 	const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);
301 	gen8_pte_t __iomem *gte;
302 	gen8_pte_t __iomem *end;
303 	struct sgt_iter iter;
304 	dma_addr_t addr;
305 
306 	/*
307 	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
308 	 * not to allow the user to override access to a read only page.
309 	 */
310 
311 	gte = (gen8_pte_t __iomem *)ggtt->gsm;
312 	gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
313 	end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
314 	while (gte < end)
315 		gen8_set_pte(gte++, vm->scratch[0]->encode);
316 	end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
317 
318 	for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
319 		gen8_set_pte(gte++, pte_encode | addr);
320 	GEM_BUG_ON(gte > end);
321 
322 	/* Fill the allocated but "unused" space beyond the end of the buffer */
323 	while (gte < end)
324 		gen8_set_pte(gte++, vm->scratch[0]->encode);
325 
326 	/*
327 	 * We want to flush the TLBs only after we're certain all the PTE
328 	 * updates have finished.
329 	 */
330 	ggtt->invalidate(ggtt);
331 }
332 
gen8_ggtt_clear_range(struct i915_address_space * vm,u64 start,u64 length)333 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
334 				  u64 start, u64 length)
335 {
336 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
337 	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
338 	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
339 	const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
340 	gen8_pte_t __iomem *gtt_base =
341 		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
342 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
343 	int i;
344 
345 	if (WARN(num_entries > max_entries,
346 		 "First entry = %d; Num entries = %d (max=%d)\n",
347 		 first_entry, num_entries, max_entries))
348 		num_entries = max_entries;
349 
350 	for (i = 0; i < num_entries; i++)
351 		gen8_set_pte(&gtt_base[i], scratch_pte);
352 }
353 
gen6_ggtt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 flags)354 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
355 				  dma_addr_t addr,
356 				  u64 offset,
357 				  unsigned int pat_index,
358 				  u32 flags)
359 {
360 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
361 	gen6_pte_t __iomem *pte =
362 		(gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
363 
364 	iowrite32(vm->pte_encode(addr, pat_index, flags), pte);
365 
366 	ggtt->invalidate(ggtt);
367 }
368 
369 /*
370  * Binds an object into the global gtt with the specified cache level.
371  * The object will be accessible to the GPU via commands whose operands
372  * reference offsets within the global GTT as well as accessible by the GPU
373  * through the GMADR mapped BAR (i915->mm.gtt->gtt).
374  */
gen6_ggtt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)375 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
376 				     struct i915_vma_resource *vma_res,
377 				     unsigned int pat_index,
378 				     u32 flags)
379 {
380 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
381 	gen6_pte_t __iomem *gte;
382 	gen6_pte_t __iomem *end;
383 	struct sgt_iter iter;
384 	dma_addr_t addr;
385 
386 	gte = (gen6_pte_t __iomem *)ggtt->gsm;
387 	gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
388 
389 	end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
390 	while (gte < end)
391 		iowrite32(vm->scratch[0]->encode, gte++);
392 	end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
393 	for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
394 		iowrite32(vm->pte_encode(addr, pat_index, flags), gte++);
395 	GEM_BUG_ON(gte > end);
396 
397 	/* Fill the allocated but "unused" space beyond the end of the buffer */
398 	while (gte < end)
399 		iowrite32(vm->scratch[0]->encode, gte++);
400 
401 	/*
402 	 * We want to flush the TLBs only after we're certain all the PTE
403 	 * updates have finished.
404 	 */
405 	ggtt->invalidate(ggtt);
406 }
407 
nop_clear_range(struct i915_address_space * vm,u64 start,u64 length)408 static void nop_clear_range(struct i915_address_space *vm,
409 			    u64 start, u64 length)
410 {
411 }
412 
bxt_vtd_ggtt_wa(struct i915_address_space * vm)413 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
414 {
415 	/*
416 	 * Make sure the internal GAM fifo has been cleared of all GTT
417 	 * writes before exiting stop_machine(). This guarantees that
418 	 * any aperture accesses waiting to start in another process
419 	 * cannot back up behind the GTT writes causing a hang.
420 	 * The register can be any arbitrary GAM register.
421 	 */
422 	intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
423 }
424 
425 struct insert_page {
426 	struct i915_address_space *vm;
427 	dma_addr_t addr;
428 	u64 offset;
429 	unsigned int pat_index;
430 };
431 
bxt_vtd_ggtt_insert_page__cb(void * _arg)432 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
433 {
434 	struct insert_page *arg = _arg;
435 
436 	gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset,
437 			      arg->pat_index, 0);
438 	bxt_vtd_ggtt_wa(arg->vm);
439 
440 	return 0;
441 }
442 
bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 unused)443 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
444 					  dma_addr_t addr,
445 					  u64 offset,
446 					  unsigned int pat_index,
447 					  u32 unused)
448 {
449 	struct insert_page arg = { vm, addr, offset, pat_index };
450 
451 	stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
452 }
453 
454 struct insert_entries {
455 	struct i915_address_space *vm;
456 	struct i915_vma_resource *vma_res;
457 	unsigned int pat_index;
458 	u32 flags;
459 };
460 
bxt_vtd_ggtt_insert_entries__cb(void * _arg)461 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
462 {
463 	struct insert_entries *arg = _arg;
464 
465 	gen8_ggtt_insert_entries(arg->vm, arg->vma_res,
466 				 arg->pat_index, arg->flags);
467 	bxt_vtd_ggtt_wa(arg->vm);
468 
469 	return 0;
470 }
471 
bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)472 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
473 					     struct i915_vma_resource *vma_res,
474 					     unsigned int pat_index,
475 					     u32 flags)
476 {
477 	struct insert_entries arg = { vm, vma_res, pat_index, flags };
478 
479 	stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
480 }
481 
gen6_ggtt_clear_range(struct i915_address_space * vm,u64 start,u64 length)482 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
483 				  u64 start, u64 length)
484 {
485 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
486 	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
487 	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
488 	gen6_pte_t scratch_pte, __iomem *gtt_base =
489 		(gen6_pte_t __iomem *)ggtt->gsm + first_entry;
490 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
491 	int i;
492 
493 	if (WARN(num_entries > max_entries,
494 		 "First entry = %d; Num entries = %d (max=%d)\n",
495 		 first_entry, num_entries, max_entries))
496 		num_entries = max_entries;
497 
498 	scratch_pte = vm->scratch[0]->encode;
499 	for (i = 0; i < num_entries; i++)
500 		iowrite32(scratch_pte, &gtt_base[i]);
501 }
502 
intel_ggtt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)503 void intel_ggtt_bind_vma(struct i915_address_space *vm,
504 			 struct i915_vm_pt_stash *stash,
505 			 struct i915_vma_resource *vma_res,
506 			 unsigned int pat_index,
507 			 u32 flags)
508 {
509 	u32 pte_flags;
510 
511 	if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))
512 		return;
513 
514 	vma_res->bound_flags |= flags;
515 
516 	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
517 	pte_flags = 0;
518 	if (vma_res->bi.readonly)
519 		pte_flags |= PTE_READ_ONLY;
520 	if (vma_res->bi.lmem)
521 		pte_flags |= PTE_LM;
522 
523 	vm->insert_entries(vm, vma_res, pat_index, pte_flags);
524 	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
525 }
526 
intel_ggtt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)527 void intel_ggtt_unbind_vma(struct i915_address_space *vm,
528 			   struct i915_vma_resource *vma_res)
529 {
530 	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
531 }
532 
533 /*
534  * Reserve the top of the GuC address space for firmware images. Addresses
535  * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC,
536  * which makes for a suitable range to hold GuC/HuC firmware images if the
537  * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT
538  * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk
539  * of the same size anyway, which is far more than needed, to keep the logic
540  * in uc_fw_ggtt_offset() simple.
541  */
542 #define GUC_TOP_RESERVE_SIZE (SZ_4G - GUC_GGTT_TOP)
543 
ggtt_reserve_guc_top(struct i915_ggtt * ggtt)544 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
545 {
546 	u64 offset;
547 	int ret;
548 
549 	if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
550 		return 0;
551 
552 	GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE);
553 	offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE;
554 
555 	ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw,
556 				   GUC_TOP_RESERVE_SIZE, offset,
557 				   I915_COLOR_UNEVICTABLE, PIN_NOEVICT);
558 	if (ret)
559 		drm_dbg(&ggtt->vm.i915->drm,
560 			"Failed to reserve top of GGTT for GuC\n");
561 
562 	return ret;
563 }
564 
ggtt_release_guc_top(struct i915_ggtt * ggtt)565 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
566 {
567 	if (drm_mm_node_allocated(&ggtt->uc_fw))
568 		drm_mm_remove_node(&ggtt->uc_fw);
569 }
570 
cleanup_init_ggtt(struct i915_ggtt * ggtt)571 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
572 {
573 	ggtt_release_guc_top(ggtt);
574 	if (drm_mm_node_allocated(&ggtt->error_capture))
575 		drm_mm_remove_node(&ggtt->error_capture);
576 	mutex_destroy(&ggtt->error_mutex);
577 }
578 
init_ggtt(struct i915_ggtt * ggtt)579 static int init_ggtt(struct i915_ggtt *ggtt)
580 {
581 	/*
582 	 * Let GEM Manage all of the aperture.
583 	 *
584 	 * However, leave one page at the end still bound to the scratch page.
585 	 * There are a number of places where the hardware apparently prefetches
586 	 * past the end of the object, and we've seen multiple hangs with the
587 	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
588 	 * aperture.  One page should be enough to keep any prefetching inside
589 	 * of the aperture.
590 	 */
591 	unsigned long hole_start, hole_end;
592 	struct drm_mm_node *entry;
593 	int ret;
594 
595 	/*
596 	 * GuC requires all resources that we're sharing with it to be placed in
597 	 * non-WOPCM memory. If GuC is not present or not in use we still need a
598 	 * small bias as ring wraparound at offset 0 sometimes hangs. No idea
599 	 * why.
600 	 */
601 	ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
602 			       intel_wopcm_guc_size(&ggtt->vm.gt->wopcm));
603 
604 	ret = intel_vgt_balloon(ggtt);
605 	if (ret)
606 		return ret;
607 
608 	mutex_init(&ggtt->error_mutex);
609 	if (ggtt->mappable_end) {
610 		/*
611 		 * Reserve a mappable slot for our lockless error capture.
612 		 *
613 		 * We strongly prefer taking address 0x0 in order to protect
614 		 * other critical buffers against accidental overwrites,
615 		 * as writing to address 0 is a very common mistake.
616 		 *
617 		 * Since 0 may already be in use by the system (e.g. the BIOS
618 		 * framebuffer), we let the reservation fail quietly and hope
619 		 * 0 remains reserved always.
620 		 *
621 		 * If we fail to reserve 0, and then fail to find any space
622 		 * for an error-capture, remain silent. We can afford not
623 		 * to reserve an error_capture node as we have fallback
624 		 * paths, and we trust that 0 will remain reserved. However,
625 		 * the only likely reason for failure to insert is a driver
626 		 * bug, which we expect to cause other failures...
627 		 *
628 		 * Since CPU can perform speculative reads on error capture
629 		 * (write-combining allows it) add scratch page after error
630 		 * capture to avoid DMAR errors.
631 		 */
632 		ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE;
633 		ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
634 		if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
635 			drm_mm_insert_node_in_range(&ggtt->vm.mm,
636 						    &ggtt->error_capture,
637 						    ggtt->error_capture.size, 0,
638 						    ggtt->error_capture.color,
639 						    0, ggtt->mappable_end,
640 						    DRM_MM_INSERT_LOW);
641 	}
642 	if (drm_mm_node_allocated(&ggtt->error_capture)) {
643 		u64 start = ggtt->error_capture.start;
644 		u64 size = ggtt->error_capture.size;
645 
646 		ggtt->vm.scratch_range(&ggtt->vm, start, size);
647 		drm_dbg(&ggtt->vm.i915->drm,
648 			"Reserved GGTT:[%llx, %llx] for use by error capture\n",
649 			start, start + size);
650 	}
651 
652 	/*
653 	 * The upper portion of the GuC address space has a sizeable hole
654 	 * (several MB) that is inaccessible by GuC. Reserve this range within
655 	 * GGTT as it can comfortably hold GuC/HuC firmware images.
656 	 */
657 	ret = ggtt_reserve_guc_top(ggtt);
658 	if (ret)
659 		goto err;
660 
661 	/* Clear any non-preallocated blocks */
662 	drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
663 		drm_dbg(&ggtt->vm.i915->drm,
664 			"clearing unused GTT space: [%lx, %lx]\n",
665 			hole_start, hole_end);
666 		ggtt->vm.clear_range(&ggtt->vm, hole_start,
667 				     hole_end - hole_start);
668 	}
669 
670 	/* And finally clear the reserved guard page */
671 	ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
672 
673 	return 0;
674 
675 err:
676 	cleanup_init_ggtt(ggtt);
677 	return ret;
678 }
679 
aliasing_gtt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)680 static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
681 				  struct i915_vm_pt_stash *stash,
682 				  struct i915_vma_resource *vma_res,
683 				  unsigned int pat_index,
684 				  u32 flags)
685 {
686 	u32 pte_flags;
687 
688 	/* Currently applicable only to VLV */
689 	pte_flags = 0;
690 	if (vma_res->bi.readonly)
691 		pte_flags |= PTE_READ_ONLY;
692 
693 	if (flags & I915_VMA_LOCAL_BIND)
694 		ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
695 			       stash, vma_res, pat_index, flags);
696 
697 	if (flags & I915_VMA_GLOBAL_BIND)
698 		vm->insert_entries(vm, vma_res, pat_index, pte_flags);
699 
700 	vma_res->bound_flags |= flags;
701 }
702 
aliasing_gtt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)703 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
704 				    struct i915_vma_resource *vma_res)
705 {
706 	if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)
707 		vm->clear_range(vm, vma_res->start, vma_res->vma_size);
708 
709 	if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)
710 		ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
711 }
712 
init_aliasing_ppgtt(struct i915_ggtt * ggtt)713 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
714 {
715 	struct i915_vm_pt_stash stash = {};
716 	struct i915_ppgtt *ppgtt;
717 	int err;
718 
719 	ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
720 	if (IS_ERR(ppgtt))
721 		return PTR_ERR(ppgtt);
722 
723 	if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
724 		err = -ENODEV;
725 		goto err_ppgtt;
726 	}
727 
728 	err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
729 	if (err)
730 		goto err_ppgtt;
731 
732 	i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
733 	err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
734 	i915_gem_object_unlock(ppgtt->vm.scratch[0]);
735 	if (err)
736 		goto err_stash;
737 
738 	/*
739 	 * Note we only pre-allocate as far as the end of the global
740 	 * GTT. On 48b / 4-level page-tables, the difference is very,
741 	 * very significant! We have to preallocate as GVT/vgpu does
742 	 * not like the page directory disappearing.
743 	 */
744 	ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
745 
746 	ggtt->alias = ppgtt;
747 	ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
748 
749 	GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);
750 	ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
751 
752 	GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);
753 	ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
754 
755 	i915_vm_free_pt_stash(&ppgtt->vm, &stash);
756 	return 0;
757 
758 err_stash:
759 	i915_vm_free_pt_stash(&ppgtt->vm, &stash);
760 err_ppgtt:
761 	i915_vm_put(&ppgtt->vm);
762 	return err;
763 }
764 
fini_aliasing_ppgtt(struct i915_ggtt * ggtt)765 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
766 {
767 	struct i915_ppgtt *ppgtt;
768 
769 	ppgtt = fetch_and_zero(&ggtt->alias);
770 	if (!ppgtt)
771 		return;
772 
773 	i915_vm_put(&ppgtt->vm);
774 
775 	ggtt->vm.vma_ops.bind_vma   = intel_ggtt_bind_vma;
776 	ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
777 }
778 
i915_init_ggtt(struct drm_i915_private * i915)779 int i915_init_ggtt(struct drm_i915_private *i915)
780 {
781 	int ret;
782 
783 	ret = init_ggtt(to_gt(i915)->ggtt);
784 	if (ret)
785 		return ret;
786 
787 	if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
788 		ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
789 		if (ret)
790 			cleanup_init_ggtt(to_gt(i915)->ggtt);
791 	}
792 
793 	return 0;
794 }
795 
ggtt_cleanup_hw(struct i915_ggtt * ggtt)796 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
797 {
798 	struct i915_vma *vma, *vn;
799 
800 	flush_workqueue(ggtt->vm.i915->wq);
801 	i915_gem_drain_freed_objects(ggtt->vm.i915);
802 
803 	mutex_lock(&ggtt->vm.mutex);
804 
805 	ggtt->vm.skip_pte_rewrite = true;
806 
807 	list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
808 		struct drm_i915_gem_object *obj = vma->obj;
809 		bool trylock;
810 
811 		trylock = i915_gem_object_trylock(obj, NULL);
812 		WARN_ON(!trylock);
813 
814 		WARN_ON(__i915_vma_unbind(vma));
815 		if (trylock)
816 			i915_gem_object_unlock(obj);
817 	}
818 
819 	if (drm_mm_node_allocated(&ggtt->error_capture))
820 		drm_mm_remove_node(&ggtt->error_capture);
821 	mutex_destroy(&ggtt->error_mutex);
822 
823 	ggtt_release_guc_top(ggtt);
824 	intel_vgt_deballoon(ggtt);
825 
826 	ggtt->vm.cleanup(&ggtt->vm);
827 
828 	mutex_unlock(&ggtt->vm.mutex);
829 	i915_address_space_fini(&ggtt->vm);
830 
831 	arch_phys_wc_del(ggtt->mtrr);
832 
833 	if (ggtt->iomap.size)
834 		io_mapping_fini(&ggtt->iomap);
835 }
836 
837 /**
838  * i915_ggtt_driver_release - Clean up GGTT hardware initialization
839  * @i915: i915 device
840  */
i915_ggtt_driver_release(struct drm_i915_private * i915)841 void i915_ggtt_driver_release(struct drm_i915_private *i915)
842 {
843 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
844 
845 	fini_aliasing_ppgtt(ggtt);
846 
847 	intel_ggtt_fini_fences(ggtt);
848 	ggtt_cleanup_hw(ggtt);
849 }
850 
851 /**
852  * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
853  * all free objects have been drained.
854  * @i915: i915 device
855  */
i915_ggtt_driver_late_release(struct drm_i915_private * i915)856 void i915_ggtt_driver_late_release(struct drm_i915_private *i915)
857 {
858 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
859 
860 	GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);
861 	dma_resv_fini(&ggtt->vm._resv);
862 }
863 
gen6_get_total_gtt_size(u16 snb_gmch_ctl)864 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
865 {
866 	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
867 	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
868 	return snb_gmch_ctl << 20;
869 }
870 
gen8_get_total_gtt_size(u16 bdw_gmch_ctl)871 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
872 {
873 	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
874 	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
875 	if (bdw_gmch_ctl)
876 		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
877 
878 #ifdef CONFIG_X86_32
879 	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
880 	if (bdw_gmch_ctl > 4)
881 		bdw_gmch_ctl = 4;
882 #endif
883 
884 	return bdw_gmch_ctl << 20;
885 }
886 
chv_get_total_gtt_size(u16 gmch_ctrl)887 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
888 {
889 	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
890 	gmch_ctrl &= SNB_GMCH_GGMS_MASK;
891 
892 	if (gmch_ctrl)
893 		return 1 << (20 + gmch_ctrl);
894 
895 	return 0;
896 }
897 
gen6_gttmmadr_size(struct drm_i915_private * i915)898 static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915)
899 {
900 	/*
901 	 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset
902 	 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset
903 	 */
904 	GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
905 	return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M;
906 }
907 
gen6_gttadr_offset(struct drm_i915_private * i915)908 static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915)
909 {
910 	return gen6_gttmmadr_size(i915) / 2;
911 }
912 
ggtt_probe_common(struct i915_ggtt * ggtt,u64 size)913 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
914 {
915 	struct drm_i915_private *i915 = ggtt->vm.i915;
916 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
917 	phys_addr_t phys_addr;
918 	u32 pte_flags;
919 	int ret;
920 
921 	GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915));
922 	phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915);
923 
924 	if (needs_wc_ggtt_mapping(i915))
925 		ggtt->gsm = ioremap_wc(phys_addr, size);
926 	else
927 		ggtt->gsm = ioremap(phys_addr, size);
928 
929 	if (!ggtt->gsm) {
930 		drm_err(&i915->drm, "Failed to map the ggtt page table\n");
931 		return -ENOMEM;
932 	}
933 
934 	kref_init(&ggtt->vm.resv_ref);
935 	ret = setup_scratch_page(&ggtt->vm);
936 	if (ret) {
937 		drm_err(&i915->drm, "Scratch setup failed\n");
938 		/* iounmap will also get called at remove, but meh */
939 		iounmap(ggtt->gsm);
940 		return ret;
941 	}
942 
943 	pte_flags = 0;
944 	if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
945 		pte_flags |= PTE_LM;
946 
947 	ggtt->vm.scratch[0]->encode =
948 		ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),
949 				    i915_gem_get_pat_index(i915,
950 							   I915_CACHE_NONE),
951 				    pte_flags);
952 
953 	return 0;
954 }
955 
gen6_gmch_remove(struct i915_address_space * vm)956 static void gen6_gmch_remove(struct i915_address_space *vm)
957 {
958 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
959 
960 	iounmap(ggtt->gsm);
961 	free_scratch(vm);
962 }
963 
pci_resource(struct pci_dev * pdev,int bar)964 static struct resource pci_resource(struct pci_dev *pdev, int bar)
965 {
966 	return DEFINE_RES_MEM(pci_resource_start(pdev, bar),
967 			      pci_resource_len(pdev, bar));
968 }
969 
gen8_gmch_probe(struct i915_ggtt * ggtt)970 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
971 {
972 	struct drm_i915_private *i915 = ggtt->vm.i915;
973 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
974 	unsigned int size;
975 	u16 snb_gmch_ctl;
976 
977 	if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) {
978 		if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
979 			return -ENXIO;
980 
981 		ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
982 		ggtt->mappable_end = resource_size(&ggtt->gmadr);
983 	}
984 
985 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
986 	if (IS_CHERRYVIEW(i915))
987 		size = chv_get_total_gtt_size(snb_gmch_ctl);
988 	else
989 		size = gen8_get_total_gtt_size(snb_gmch_ctl);
990 
991 	ggtt->vm.alloc_pt_dma = alloc_pt_dma;
992 	ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
993 	ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY;
994 
995 	ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
996 	ggtt->vm.cleanup = gen6_gmch_remove;
997 	ggtt->vm.insert_page = gen8_ggtt_insert_page;
998 	ggtt->vm.clear_range = nop_clear_range;
999 	ggtt->vm.scratch_range = gen8_ggtt_clear_range;
1000 
1001 	ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
1002 
1003 	/*
1004 	 * Serialize GTT updates with aperture access on BXT if VT-d is on,
1005 	 * and always on CHV.
1006 	 */
1007 	if (intel_vm_no_concurrent_access_wa(i915)) {
1008 		ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
1009 		ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
1010 
1011 		/*
1012 		 * Calling stop_machine() version of GGTT update function
1013 		 * at error capture/reset path will raise lockdep warning.
1014 		 * Allow calling gen8_ggtt_insert_* directly at reset path
1015 		 * which is safe from parallel GGTT updates.
1016 		 */
1017 		ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
1018 		ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;
1019 
1020 		ggtt->vm.bind_async_flags =
1021 			I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
1022 	}
1023 
1024 	if (intel_uc_wants_guc(&ggtt->vm.gt->uc))
1025 		ggtt->invalidate = guc_ggtt_invalidate;
1026 	else
1027 		ggtt->invalidate = gen8_ggtt_invalidate;
1028 
1029 	ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1030 	ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1031 
1032 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1033 		ggtt->vm.pte_encode = mtl_ggtt_pte_encode;
1034 	else
1035 		ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
1036 
1037 	return ggtt_probe_common(ggtt, size);
1038 }
1039 
1040 /*
1041  * For pre-gen8 platforms pat_index is the same as enum i915_cache_level,
1042  * so the switch-case statements in these PTE encode functions are still valid.
1043  * See translation table LEGACY_CACHELEVEL.
1044  */
snb_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)1045 static u64 snb_pte_encode(dma_addr_t addr,
1046 			  unsigned int pat_index,
1047 			  u32 flags)
1048 {
1049 	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1050 
1051 	switch (pat_index) {
1052 	case I915_CACHE_L3_LLC:
1053 	case I915_CACHE_LLC:
1054 		pte |= GEN6_PTE_CACHE_LLC;
1055 		break;
1056 	case I915_CACHE_NONE:
1057 		pte |= GEN6_PTE_UNCACHED;
1058 		break;
1059 	default:
1060 		MISSING_CASE(pat_index);
1061 	}
1062 
1063 	return pte;
1064 }
1065 
ivb_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)1066 static u64 ivb_pte_encode(dma_addr_t addr,
1067 			  unsigned int pat_index,
1068 			  u32 flags)
1069 {
1070 	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1071 
1072 	switch (pat_index) {
1073 	case I915_CACHE_L3_LLC:
1074 		pte |= GEN7_PTE_CACHE_L3_LLC;
1075 		break;
1076 	case I915_CACHE_LLC:
1077 		pte |= GEN6_PTE_CACHE_LLC;
1078 		break;
1079 	case I915_CACHE_NONE:
1080 		pte |= GEN6_PTE_UNCACHED;
1081 		break;
1082 	default:
1083 		MISSING_CASE(pat_index);
1084 	}
1085 
1086 	return pte;
1087 }
1088 
byt_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)1089 static u64 byt_pte_encode(dma_addr_t addr,
1090 			  unsigned int pat_index,
1091 			  u32 flags)
1092 {
1093 	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1094 
1095 	if (!(flags & PTE_READ_ONLY))
1096 		pte |= BYT_PTE_WRITEABLE;
1097 
1098 	if (pat_index != I915_CACHE_NONE)
1099 		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
1100 
1101 	return pte;
1102 }
1103 
hsw_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)1104 static u64 hsw_pte_encode(dma_addr_t addr,
1105 			  unsigned int pat_index,
1106 			  u32 flags)
1107 {
1108 	gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1109 
1110 	if (pat_index != I915_CACHE_NONE)
1111 		pte |= HSW_WB_LLC_AGE3;
1112 
1113 	return pte;
1114 }
1115 
iris_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)1116 static u64 iris_pte_encode(dma_addr_t addr,
1117 			   unsigned int pat_index,
1118 			   u32 flags)
1119 {
1120 	gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1121 
1122 	switch (pat_index) {
1123 	case I915_CACHE_NONE:
1124 		break;
1125 	case I915_CACHE_WT:
1126 		pte |= HSW_WT_ELLC_LLC_AGE3;
1127 		break;
1128 	default:
1129 		pte |= HSW_WB_ELLC_LLC_AGE3;
1130 		break;
1131 	}
1132 
1133 	return pte;
1134 }
1135 
gen6_gmch_probe(struct i915_ggtt * ggtt)1136 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
1137 {
1138 	struct drm_i915_private *i915 = ggtt->vm.i915;
1139 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1140 	unsigned int size;
1141 	u16 snb_gmch_ctl;
1142 
1143 	if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
1144 		return -ENXIO;
1145 
1146 	ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
1147 	ggtt->mappable_end = resource_size(&ggtt->gmadr);
1148 
1149 	/*
1150 	 * 64/512MB is the current min/max we actually know of, but this is
1151 	 * just a coarse sanity check.
1152 	 */
1153 	if (ggtt->mappable_end < (64 << 20) ||
1154 	    ggtt->mappable_end > (512 << 20)) {
1155 		drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",
1156 			&ggtt->mappable_end);
1157 		return -ENXIO;
1158 	}
1159 
1160 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1161 
1162 	size = gen6_get_total_gtt_size(snb_gmch_ctl);
1163 	ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
1164 
1165 	ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1166 	ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1167 
1168 	ggtt->vm.clear_range = nop_clear_range;
1169 	if (!HAS_FULL_PPGTT(i915))
1170 		ggtt->vm.clear_range = gen6_ggtt_clear_range;
1171 	ggtt->vm.scratch_range = gen6_ggtt_clear_range;
1172 	ggtt->vm.insert_page = gen6_ggtt_insert_page;
1173 	ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
1174 	ggtt->vm.cleanup = gen6_gmch_remove;
1175 
1176 	ggtt->invalidate = gen6_ggtt_invalidate;
1177 
1178 	if (HAS_EDRAM(i915))
1179 		ggtt->vm.pte_encode = iris_pte_encode;
1180 	else if (IS_HASWELL(i915))
1181 		ggtt->vm.pte_encode = hsw_pte_encode;
1182 	else if (IS_VALLEYVIEW(i915))
1183 		ggtt->vm.pte_encode = byt_pte_encode;
1184 	else if (GRAPHICS_VER(i915) >= 7)
1185 		ggtt->vm.pte_encode = ivb_pte_encode;
1186 	else
1187 		ggtt->vm.pte_encode = snb_pte_encode;
1188 
1189 	ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1190 	ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1191 
1192 	return ggtt_probe_common(ggtt, size);
1193 }
1194 
ggtt_probe_hw(struct i915_ggtt * ggtt,struct intel_gt * gt)1195 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1196 {
1197 	struct drm_i915_private *i915 = gt->i915;
1198 	int ret;
1199 
1200 	ggtt->vm.gt = gt;
1201 	ggtt->vm.i915 = i915;
1202 	ggtt->vm.dma = i915->drm.dev;
1203 	dma_resv_init(&ggtt->vm._resv);
1204 
1205 	if (GRAPHICS_VER(i915) >= 8)
1206 		ret = gen8_gmch_probe(ggtt);
1207 	else if (GRAPHICS_VER(i915) >= 6)
1208 		ret = gen6_gmch_probe(ggtt);
1209 	else
1210 		ret = intel_ggtt_gmch_probe(ggtt);
1211 
1212 	if (ret) {
1213 		dma_resv_fini(&ggtt->vm._resv);
1214 		return ret;
1215 	}
1216 
1217 	if ((ggtt->vm.total - 1) >> 32) {
1218 		drm_err(&i915->drm,
1219 			"We never expected a Global GTT with more than 32bits"
1220 			" of address space! Found %lldM!\n",
1221 			ggtt->vm.total >> 20);
1222 		ggtt->vm.total = 1ULL << 32;
1223 		ggtt->mappable_end =
1224 			min_t(u64, ggtt->mappable_end, ggtt->vm.total);
1225 	}
1226 
1227 	if (ggtt->mappable_end > ggtt->vm.total) {
1228 		drm_err(&i915->drm,
1229 			"mappable aperture extends past end of GGTT,"
1230 			" aperture=%pa, total=%llx\n",
1231 			&ggtt->mappable_end, ggtt->vm.total);
1232 		ggtt->mappable_end = ggtt->vm.total;
1233 	}
1234 
1235 	/* GMADR is the PCI mmio aperture into the global GTT. */
1236 	drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
1237 	drm_dbg(&i915->drm, "GMADR size = %lluM\n",
1238 		(u64)ggtt->mappable_end >> 20);
1239 	drm_dbg(&i915->drm, "DSM size = %lluM\n",
1240 		(u64)resource_size(&intel_graphics_stolen_res) >> 20);
1241 
1242 	return 0;
1243 }
1244 
1245 /**
1246  * i915_ggtt_probe_hw - Probe GGTT hardware location
1247  * @i915: i915 device
1248  */
i915_ggtt_probe_hw(struct drm_i915_private * i915)1249 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
1250 {
1251 	struct intel_gt *gt;
1252 	int ret, i;
1253 
1254 	for_each_gt(gt, i915, i) {
1255 		ret = intel_gt_assign_ggtt(gt);
1256 		if (ret)
1257 			return ret;
1258 	}
1259 
1260 	ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
1261 	if (ret)
1262 		return ret;
1263 
1264 	if (i915_vtd_active(i915))
1265 		drm_info(&i915->drm, "VT-d active for gfx access\n");
1266 
1267 	return 0;
1268 }
1269 
i915_ggtt_create(struct drm_i915_private * i915)1270 struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915)
1271 {
1272 	struct i915_ggtt *ggtt;
1273 
1274 	ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL);
1275 	if (!ggtt)
1276 		return ERR_PTR(-ENOMEM);
1277 
1278 	INIT_LIST_HEAD(&ggtt->gt_list);
1279 
1280 	return ggtt;
1281 }
1282 
i915_ggtt_enable_hw(struct drm_i915_private * i915)1283 int i915_ggtt_enable_hw(struct drm_i915_private *i915)
1284 {
1285 	if (GRAPHICS_VER(i915) < 6)
1286 		return intel_ggtt_gmch_enable_hw(i915);
1287 
1288 	return 0;
1289 }
1290 
1291 /**
1292  * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
1293  * @vm: The VM to restore the mappings for
1294  *
1295  * Restore the memory mappings for all objects mapped to HW via the GGTT or a
1296  * DPT page table.
1297  *
1298  * Returns %true if restoring the mapping for any object that was in a write
1299  * domain before suspend.
1300  */
i915_ggtt_resume_vm(struct i915_address_space * vm)1301 bool i915_ggtt_resume_vm(struct i915_address_space *vm)
1302 {
1303 	struct i915_vma *vma;
1304 	bool write_domain_objs = false;
1305 
1306 	drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
1307 
1308 	/* First fill our portion of the GTT with scratch pages */
1309 	vm->clear_range(vm, 0, vm->total);
1310 
1311 	/* clflush objects bound into the GGTT and rebind them. */
1312 	list_for_each_entry(vma, &vm->bound_list, vm_link) {
1313 		struct drm_i915_gem_object *obj = vma->obj;
1314 		unsigned int was_bound =
1315 			atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
1316 
1317 		GEM_BUG_ON(!was_bound);
1318 
1319 		/*
1320 		 * Clear the bound flags of the vma resource to allow
1321 		 * ptes to be repopulated.
1322 		 */
1323 		vma->resource->bound_flags = 0;
1324 		vma->ops->bind_vma(vm, NULL, vma->resource,
1325 				   obj ? obj->pat_index :
1326 					 i915_gem_get_pat_index(vm->i915,
1327 								I915_CACHE_NONE),
1328 				   was_bound);
1329 
1330 		if (obj) { /* only used during resume => exclusive access */
1331 			write_domain_objs |= fetch_and_zero(&obj->write_domain);
1332 			obj->read_domains |= I915_GEM_DOMAIN_GTT;
1333 		}
1334 	}
1335 
1336 	return write_domain_objs;
1337 }
1338 
i915_ggtt_resume(struct i915_ggtt * ggtt)1339 void i915_ggtt_resume(struct i915_ggtt *ggtt)
1340 {
1341 	struct intel_gt *gt;
1342 	bool flush;
1343 
1344 	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1345 		intel_gt_check_and_clear_faults(gt);
1346 
1347 	flush = i915_ggtt_resume_vm(&ggtt->vm);
1348 
1349 	if (drm_mm_node_allocated(&ggtt->error_capture))
1350 		ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start,
1351 				       ggtt->error_capture.size);
1352 
1353 	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1354 		intel_uc_resume_mappings(&gt->uc);
1355 
1356 	ggtt->invalidate(ggtt);
1357 
1358 	if (flush)
1359 		wbinvd_on_all_cpus();
1360 
1361 	intel_ggtt_restore_fences(ggtt);
1362 }
1363