1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include <asm/set_memory.h>
7 #include <asm/smp.h>
8 #include <linux/types.h>
9 #include <linux/stop_machine.h>
10 
11 #include <drm/i915_drm.h>
12 #include <drm/intel-gtt.h>
13 
14 #include "gem/i915_gem_lmem.h"
15 
16 #include "intel_ggtt_gmch.h"
17 #include "intel_gt.h"
18 #include "intel_gt_regs.h"
19 #include "intel_pci_config.h"
20 #include "i915_drv.h"
21 #include "i915_pci.h"
22 #include "i915_scatterlist.h"
23 #include "i915_utils.h"
24 #include "i915_vgpu.h"
25 
26 #include "intel_gtt.h"
27 #include "gen8_ppgtt.h"
28 
suspend_retains_ptes(struct i915_address_space * vm)29 static inline bool suspend_retains_ptes(struct i915_address_space *vm)
30 {
31 	return GRAPHICS_VER(vm->i915) >= 8 &&
32 		!HAS_LMEM(vm->i915) &&
33 		vm->is_ggtt;
34 }
35 
i915_ggtt_color_adjust(const struct drm_mm_node * node,unsigned long color,u64 * start,u64 * end)36 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
37 				   unsigned long color,
38 				   u64 *start,
39 				   u64 *end)
40 {
41 	if (i915_node_color_differs(node, color))
42 		*start += I915_GTT_PAGE_SIZE;
43 
44 	/*
45 	 * Also leave a space between the unallocated reserved node after the
46 	 * GTT and any objects within the GTT, i.e. we use the color adjustment
47 	 * to insert a guard page to prevent prefetches crossing over the
48 	 * GTT boundary.
49 	 */
50 	node = list_next_entry(node, node_list);
51 	if (node->color != color)
52 		*end -= I915_GTT_PAGE_SIZE;
53 }
54 
ggtt_init_hw(struct i915_ggtt * ggtt)55 static int ggtt_init_hw(struct i915_ggtt *ggtt)
56 {
57 	struct drm_i915_private *i915 = ggtt->vm.i915;
58 
59 	i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
60 
61 	ggtt->vm.is_ggtt = true;
62 
63 	/* Only VLV supports read-only GGTT mappings */
64 	ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
65 
66 	if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
67 		ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
68 
69 	if (ggtt->mappable_end) {
70 		if (!io_mapping_init_wc(&ggtt->iomap,
71 					ggtt->gmadr.start,
72 					ggtt->mappable_end)) {
73 			ggtt->vm.cleanup(&ggtt->vm);
74 			return -EIO;
75 		}
76 
77 		ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
78 					      ggtt->mappable_end);
79 	}
80 
81 	intel_ggtt_init_fences(ggtt);
82 
83 	return 0;
84 }
85 
86 /**
87  * i915_ggtt_init_hw - Initialize GGTT hardware
88  * @i915: i915 device
89  */
i915_ggtt_init_hw(struct drm_i915_private * i915)90 int i915_ggtt_init_hw(struct drm_i915_private *i915)
91 {
92 	int ret;
93 
94 	/*
95 	 * Note that we use page colouring to enforce a guard page at the
96 	 * end of the address space. This is required as the CS may prefetch
97 	 * beyond the end of the batch buffer, across the page boundary,
98 	 * and beyond the end of the GTT if we do not provide a guard.
99 	 */
100 	ret = ggtt_init_hw(to_gt(i915)->ggtt);
101 	if (ret)
102 		return ret;
103 
104 	return 0;
105 }
106 
107 /*
108  * Return the value of the last GGTT pte cast to an u64, if
109  * the system is supposed to retain ptes across resume. 0 otherwise.
110  */
read_last_pte(struct i915_address_space * vm)111 static u64 read_last_pte(struct i915_address_space *vm)
112 {
113 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
114 	gen8_pte_t __iomem *ptep;
115 
116 	if (!suspend_retains_ptes(vm))
117 		return 0;
118 
119 	GEM_BUG_ON(GRAPHICS_VER(vm->i915) < 8);
120 	ptep = (typeof(ptep))ggtt->gsm + (ggtt_total_entries(ggtt) - 1);
121 	return readq(ptep);
122 }
123 
124 /**
125  * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
126  * @vm: The VM to suspend the mappings for
127  *
128  * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
129  * DPT page table.
130  */
i915_ggtt_suspend_vm(struct i915_address_space * vm)131 void i915_ggtt_suspend_vm(struct i915_address_space *vm)
132 {
133 	struct i915_vma *vma, *vn;
134 	int save_skip_rewrite;
135 
136 	drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
137 
138 retry:
139 	i915_gem_drain_freed_objects(vm->i915);
140 
141 	mutex_lock(&vm->mutex);
142 
143 	/*
144 	 * Skip rewriting PTE on VMA unbind.
145 	 * FIXME: Use an argument to i915_vma_unbind() instead?
146 	 */
147 	save_skip_rewrite = vm->skip_pte_rewrite;
148 	vm->skip_pte_rewrite = true;
149 
150 	list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
151 		struct drm_i915_gem_object *obj = vma->obj;
152 
153 		GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
154 
155 		if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
156 			continue;
157 
158 		/* unlikely to race when GPU is idle, so no worry about slowpath.. */
159 		if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {
160 			/*
161 			 * No dead objects should appear here, GPU should be
162 			 * completely idle, and userspace suspended
163 			 */
164 			i915_gem_object_get(obj);
165 
166 			mutex_unlock(&vm->mutex);
167 
168 			i915_gem_object_lock(obj, NULL);
169 			GEM_WARN_ON(i915_vma_unbind(vma));
170 			i915_gem_object_unlock(obj);
171 			i915_gem_object_put(obj);
172 
173 			vm->skip_pte_rewrite = save_skip_rewrite;
174 			goto retry;
175 		}
176 
177 		if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
178 			i915_vma_wait_for_bind(vma);
179 
180 			__i915_vma_evict(vma, false);
181 			drm_mm_remove_node(&vma->node);
182 		}
183 
184 		i915_gem_object_unlock(obj);
185 	}
186 
187 	if (!suspend_retains_ptes(vm))
188 		vm->clear_range(vm, 0, vm->total);
189 	else
190 		i915_vm_to_ggtt(vm)->probed_pte = read_last_pte(vm);
191 
192 	vm->skip_pte_rewrite = save_skip_rewrite;
193 
194 	mutex_unlock(&vm->mutex);
195 }
196 
i915_ggtt_suspend(struct i915_ggtt * ggtt)197 void i915_ggtt_suspend(struct i915_ggtt *ggtt)
198 {
199 	i915_ggtt_suspend_vm(&ggtt->vm);
200 	ggtt->invalidate(ggtt);
201 
202 	intel_gt_check_and_clear_faults(ggtt->vm.gt);
203 }
204 
gen6_ggtt_invalidate(struct i915_ggtt * ggtt)205 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
206 {
207 	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
208 
209 	spin_lock_irq(&uncore->lock);
210 	intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
211 	intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
212 	spin_unlock_irq(&uncore->lock);
213 }
214 
gen8_ggtt_invalidate(struct i915_ggtt * ggtt)215 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
216 {
217 	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
218 
219 	/*
220 	 * Note that as an uncached mmio write, this will flush the
221 	 * WCB of the writes into the GGTT before it triggers the invalidate.
222 	 */
223 	intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
224 }
225 
guc_ggtt_invalidate(struct i915_ggtt * ggtt)226 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
227 {
228 	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
229 	struct drm_i915_private *i915 = ggtt->vm.i915;
230 
231 	gen8_ggtt_invalidate(ggtt);
232 
233 	if (GRAPHICS_VER(i915) >= 12)
234 		intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,
235 				      GEN12_GUC_TLB_INV_CR_INVALIDATE);
236 	else
237 		intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
238 }
239 
gen8_ggtt_pte_encode(dma_addr_t addr,enum i915_cache_level level,u32 flags)240 u64 gen8_ggtt_pte_encode(dma_addr_t addr,
241 			 enum i915_cache_level level,
242 			 u32 flags)
243 {
244 	gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
245 
246 	if (flags & PTE_LM)
247 		pte |= GEN12_GGTT_PTE_LM;
248 
249 	return pte;
250 }
251 
gen8_set_pte(void __iomem * addr,gen8_pte_t pte)252 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
253 {
254 	writeq(pte, addr);
255 }
256 
gen8_ggtt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,enum i915_cache_level level,u32 flags)257 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
258 				  dma_addr_t addr,
259 				  u64 offset,
260 				  enum i915_cache_level level,
261 				  u32 flags)
262 {
263 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
264 	gen8_pte_t __iomem *pte =
265 		(gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
266 
267 	gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, flags));
268 
269 	ggtt->invalidate(ggtt);
270 }
271 
gen8_ggtt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,enum i915_cache_level level,u32 flags)272 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
273 				     struct i915_vma_resource *vma_res,
274 				     enum i915_cache_level level,
275 				     u32 flags)
276 {
277 	const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, flags);
278 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
279 	gen8_pte_t __iomem *gte;
280 	gen8_pte_t __iomem *end;
281 	struct sgt_iter iter;
282 	dma_addr_t addr;
283 
284 	/*
285 	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
286 	 * not to allow the user to override access to a read only page.
287 	 */
288 
289 	gte = (gen8_pte_t __iomem *)ggtt->gsm;
290 	gte += vma_res->start / I915_GTT_PAGE_SIZE;
291 	end = gte + vma_res->node_size / I915_GTT_PAGE_SIZE;
292 
293 	for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
294 		gen8_set_pte(gte++, pte_encode | addr);
295 	GEM_BUG_ON(gte > end);
296 
297 	/* Fill the allocated but "unused" space beyond the end of the buffer */
298 	while (gte < end)
299 		gen8_set_pte(gte++, vm->scratch[0]->encode);
300 
301 	/*
302 	 * We want to flush the TLBs only after we're certain all the PTE
303 	 * updates have finished.
304 	 */
305 	ggtt->invalidate(ggtt);
306 }
307 
gen6_ggtt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,enum i915_cache_level level,u32 flags)308 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
309 				  dma_addr_t addr,
310 				  u64 offset,
311 				  enum i915_cache_level level,
312 				  u32 flags)
313 {
314 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
315 	gen6_pte_t __iomem *pte =
316 		(gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
317 
318 	iowrite32(vm->pte_encode(addr, level, flags), pte);
319 
320 	ggtt->invalidate(ggtt);
321 }
322 
323 /*
324  * Binds an object into the global gtt with the specified cache level.
325  * The object will be accessible to the GPU via commands whose operands
326  * reference offsets within the global GTT as well as accessible by the GPU
327  * through the GMADR mapped BAR (i915->mm.gtt->gtt).
328  */
gen6_ggtt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,enum i915_cache_level level,u32 flags)329 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
330 				     struct i915_vma_resource *vma_res,
331 				     enum i915_cache_level level,
332 				     u32 flags)
333 {
334 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
335 	gen6_pte_t __iomem *gte;
336 	gen6_pte_t __iomem *end;
337 	struct sgt_iter iter;
338 	dma_addr_t addr;
339 
340 	gte = (gen6_pte_t __iomem *)ggtt->gsm;
341 	gte += vma_res->start / I915_GTT_PAGE_SIZE;
342 	end = gte + vma_res->node_size / I915_GTT_PAGE_SIZE;
343 
344 	for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
345 		iowrite32(vm->pte_encode(addr, level, flags), gte++);
346 	GEM_BUG_ON(gte > end);
347 
348 	/* Fill the allocated but "unused" space beyond the end of the buffer */
349 	while (gte < end)
350 		iowrite32(vm->scratch[0]->encode, gte++);
351 
352 	/*
353 	 * We want to flush the TLBs only after we're certain all the PTE
354 	 * updates have finished.
355 	 */
356 	ggtt->invalidate(ggtt);
357 }
358 
nop_clear_range(struct i915_address_space * vm,u64 start,u64 length)359 static void nop_clear_range(struct i915_address_space *vm,
360 			    u64 start, u64 length)
361 {
362 }
363 
gen8_ggtt_clear_range(struct i915_address_space * vm,u64 start,u64 length)364 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
365 				  u64 start, u64 length)
366 {
367 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
368 	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
369 	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
370 	const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
371 	gen8_pte_t __iomem *gtt_base =
372 		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
373 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
374 	int i;
375 
376 	if (WARN(num_entries > max_entries,
377 		 "First entry = %d; Num entries = %d (max=%d)\n",
378 		 first_entry, num_entries, max_entries))
379 		num_entries = max_entries;
380 
381 	for (i = 0; i < num_entries; i++)
382 		gen8_set_pte(&gtt_base[i], scratch_pte);
383 }
384 
bxt_vtd_ggtt_wa(struct i915_address_space * vm)385 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
386 {
387 	/*
388 	 * Make sure the internal GAM fifo has been cleared of all GTT
389 	 * writes before exiting stop_machine(). This guarantees that
390 	 * any aperture accesses waiting to start in another process
391 	 * cannot back up behind the GTT writes causing a hang.
392 	 * The register can be any arbitrary GAM register.
393 	 */
394 	intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
395 }
396 
397 struct insert_page {
398 	struct i915_address_space *vm;
399 	dma_addr_t addr;
400 	u64 offset;
401 	enum i915_cache_level level;
402 };
403 
bxt_vtd_ggtt_insert_page__cb(void * _arg)404 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
405 {
406 	struct insert_page *arg = _arg;
407 
408 	gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
409 	bxt_vtd_ggtt_wa(arg->vm);
410 
411 	return 0;
412 }
413 
bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space * vm,dma_addr_t addr,u64 offset,enum i915_cache_level level,u32 unused)414 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
415 					  dma_addr_t addr,
416 					  u64 offset,
417 					  enum i915_cache_level level,
418 					  u32 unused)
419 {
420 	struct insert_page arg = { vm, addr, offset, level };
421 
422 	stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
423 }
424 
425 struct insert_entries {
426 	struct i915_address_space *vm;
427 	struct i915_vma_resource *vma_res;
428 	enum i915_cache_level level;
429 	u32 flags;
430 };
431 
bxt_vtd_ggtt_insert_entries__cb(void * _arg)432 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
433 {
434 	struct insert_entries *arg = _arg;
435 
436 	gen8_ggtt_insert_entries(arg->vm, arg->vma_res, arg->level, arg->flags);
437 	bxt_vtd_ggtt_wa(arg->vm);
438 
439 	return 0;
440 }
441 
bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space * vm,struct i915_vma_resource * vma_res,enum i915_cache_level level,u32 flags)442 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
443 					     struct i915_vma_resource *vma_res,
444 					     enum i915_cache_level level,
445 					     u32 flags)
446 {
447 	struct insert_entries arg = { vm, vma_res, level, flags };
448 
449 	stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
450 }
451 
gen6_ggtt_clear_range(struct i915_address_space * vm,u64 start,u64 length)452 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
453 				  u64 start, u64 length)
454 {
455 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
456 	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
457 	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
458 	gen6_pte_t scratch_pte, __iomem *gtt_base =
459 		(gen6_pte_t __iomem *)ggtt->gsm + first_entry;
460 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
461 	int i;
462 
463 	if (WARN(num_entries > max_entries,
464 		 "First entry = %d; Num entries = %d (max=%d)\n",
465 		 first_entry, num_entries, max_entries))
466 		num_entries = max_entries;
467 
468 	scratch_pte = vm->scratch[0]->encode;
469 	for (i = 0; i < num_entries; i++)
470 		iowrite32(scratch_pte, &gtt_base[i]);
471 }
472 
intel_ggtt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,enum i915_cache_level cache_level,u32 flags)473 void intel_ggtt_bind_vma(struct i915_address_space *vm,
474 			 struct i915_vm_pt_stash *stash,
475 			 struct i915_vma_resource *vma_res,
476 			 enum i915_cache_level cache_level,
477 			 u32 flags)
478 {
479 	u32 pte_flags;
480 
481 	if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))
482 		return;
483 
484 	vma_res->bound_flags |= flags;
485 
486 	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
487 	pte_flags = 0;
488 	if (vma_res->bi.readonly)
489 		pte_flags |= PTE_READ_ONLY;
490 	if (vma_res->bi.lmem)
491 		pte_flags |= PTE_LM;
492 
493 	vm->insert_entries(vm, vma_res, cache_level, pte_flags);
494 	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
495 }
496 
intel_ggtt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)497 void intel_ggtt_unbind_vma(struct i915_address_space *vm,
498 			   struct i915_vma_resource *vma_res)
499 {
500 	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
501 }
502 
ggtt_reserve_guc_top(struct i915_ggtt * ggtt)503 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
504 {
505 	u64 size;
506 	int ret;
507 
508 	if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
509 		return 0;
510 
511 	GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
512 	size = ggtt->vm.total - GUC_GGTT_TOP;
513 
514 	ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, size,
515 				   GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
516 				   PIN_NOEVICT);
517 	if (ret)
518 		drm_dbg(&ggtt->vm.i915->drm,
519 			"Failed to reserve top of GGTT for GuC\n");
520 
521 	return ret;
522 }
523 
ggtt_release_guc_top(struct i915_ggtt * ggtt)524 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
525 {
526 	if (drm_mm_node_allocated(&ggtt->uc_fw))
527 		drm_mm_remove_node(&ggtt->uc_fw);
528 }
529 
cleanup_init_ggtt(struct i915_ggtt * ggtt)530 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
531 {
532 	ggtt_release_guc_top(ggtt);
533 	if (drm_mm_node_allocated(&ggtt->error_capture))
534 		drm_mm_remove_node(&ggtt->error_capture);
535 	mutex_destroy(&ggtt->error_mutex);
536 }
537 
init_ggtt(struct i915_ggtt * ggtt)538 static int init_ggtt(struct i915_ggtt *ggtt)
539 {
540 	/*
541 	 * Let GEM Manage all of the aperture.
542 	 *
543 	 * However, leave one page at the end still bound to the scratch page.
544 	 * There are a number of places where the hardware apparently prefetches
545 	 * past the end of the object, and we've seen multiple hangs with the
546 	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
547 	 * aperture.  One page should be enough to keep any prefetching inside
548 	 * of the aperture.
549 	 */
550 	unsigned long hole_start, hole_end;
551 	struct drm_mm_node *entry;
552 	int ret;
553 
554 	ggtt->pte_lost = true;
555 
556 	/*
557 	 * GuC requires all resources that we're sharing with it to be placed in
558 	 * non-WOPCM memory. If GuC is not present or not in use we still need a
559 	 * small bias as ring wraparound at offset 0 sometimes hangs. No idea
560 	 * why.
561 	 */
562 	ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
563 			       intel_wopcm_guc_size(&ggtt->vm.i915->wopcm));
564 
565 	ret = intel_vgt_balloon(ggtt);
566 	if (ret)
567 		return ret;
568 
569 	mutex_init(&ggtt->error_mutex);
570 	if (ggtt->mappable_end) {
571 		/*
572 		 * Reserve a mappable slot for our lockless error capture.
573 		 *
574 		 * We strongly prefer taking address 0x0 in order to protect
575 		 * other critical buffers against accidental overwrites,
576 		 * as writing to address 0 is a very common mistake.
577 		 *
578 		 * Since 0 may already be in use by the system (e.g. the BIOS
579 		 * framebuffer), we let the reservation fail quietly and hope
580 		 * 0 remains reserved always.
581 		 *
582 		 * If we fail to reserve 0, and then fail to find any space
583 		 * for an error-capture, remain silent. We can afford not
584 		 * to reserve an error_capture node as we have fallback
585 		 * paths, and we trust that 0 will remain reserved. However,
586 		 * the only likely reason for failure to insert is a driver
587 		 * bug, which we expect to cause other failures...
588 		 */
589 		ggtt->error_capture.size = I915_GTT_PAGE_SIZE;
590 		ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
591 		if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
592 			drm_mm_insert_node_in_range(&ggtt->vm.mm,
593 						    &ggtt->error_capture,
594 						    ggtt->error_capture.size, 0,
595 						    ggtt->error_capture.color,
596 						    0, ggtt->mappable_end,
597 						    DRM_MM_INSERT_LOW);
598 	}
599 	if (drm_mm_node_allocated(&ggtt->error_capture))
600 		drm_dbg(&ggtt->vm.i915->drm,
601 			"Reserved GGTT:[%llx, %llx] for use by error capture\n",
602 			ggtt->error_capture.start,
603 			ggtt->error_capture.start + ggtt->error_capture.size);
604 
605 	/*
606 	 * The upper portion of the GuC address space has a sizeable hole
607 	 * (several MB) that is inaccessible by GuC. Reserve this range within
608 	 * GGTT as it can comfortably hold GuC/HuC firmware images.
609 	 */
610 	ret = ggtt_reserve_guc_top(ggtt);
611 	if (ret)
612 		goto err;
613 
614 	/* Clear any non-preallocated blocks */
615 	drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
616 		drm_dbg(&ggtt->vm.i915->drm,
617 			"clearing unused GTT space: [%lx, %lx]\n",
618 			hole_start, hole_end);
619 		ggtt->vm.clear_range(&ggtt->vm, hole_start,
620 				     hole_end - hole_start);
621 	}
622 
623 	/* And finally clear the reserved guard page */
624 	ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
625 
626 	return 0;
627 
628 err:
629 	cleanup_init_ggtt(ggtt);
630 	return ret;
631 }
632 
aliasing_gtt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,enum i915_cache_level cache_level,u32 flags)633 static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
634 				  struct i915_vm_pt_stash *stash,
635 				  struct i915_vma_resource *vma_res,
636 				  enum i915_cache_level cache_level,
637 				  u32 flags)
638 {
639 	u32 pte_flags;
640 
641 	/* Currently applicable only to VLV */
642 	pte_flags = 0;
643 	if (vma_res->bi.readonly)
644 		pte_flags |= PTE_READ_ONLY;
645 
646 	if (flags & I915_VMA_LOCAL_BIND)
647 		ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
648 			       stash, vma_res, cache_level, flags);
649 
650 	if (flags & I915_VMA_GLOBAL_BIND)
651 		vm->insert_entries(vm, vma_res, cache_level, pte_flags);
652 
653 	vma_res->bound_flags |= flags;
654 }
655 
aliasing_gtt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)656 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
657 				    struct i915_vma_resource *vma_res)
658 {
659 	if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)
660 		vm->clear_range(vm, vma_res->start, vma_res->vma_size);
661 
662 	if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)
663 		ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
664 }
665 
init_aliasing_ppgtt(struct i915_ggtt * ggtt)666 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
667 {
668 	struct i915_vm_pt_stash stash = {};
669 	struct i915_ppgtt *ppgtt;
670 	int err;
671 
672 	ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
673 	if (IS_ERR(ppgtt))
674 		return PTR_ERR(ppgtt);
675 
676 	if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
677 		err = -ENODEV;
678 		goto err_ppgtt;
679 	}
680 
681 	err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
682 	if (err)
683 		goto err_ppgtt;
684 
685 	i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
686 	err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
687 	i915_gem_object_unlock(ppgtt->vm.scratch[0]);
688 	if (err)
689 		goto err_stash;
690 
691 	/*
692 	 * Note we only pre-allocate as far as the end of the global
693 	 * GTT. On 48b / 4-level page-tables, the difference is very,
694 	 * very significant! We have to preallocate as GVT/vgpu does
695 	 * not like the page directory disappearing.
696 	 */
697 	ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
698 
699 	ggtt->alias = ppgtt;
700 	ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
701 
702 	GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);
703 	ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
704 
705 	GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);
706 	ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
707 
708 	i915_vm_free_pt_stash(&ppgtt->vm, &stash);
709 	return 0;
710 
711 err_stash:
712 	i915_vm_free_pt_stash(&ppgtt->vm, &stash);
713 err_ppgtt:
714 	i915_vm_put(&ppgtt->vm);
715 	return err;
716 }
717 
fini_aliasing_ppgtt(struct i915_ggtt * ggtt)718 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
719 {
720 	struct i915_ppgtt *ppgtt;
721 
722 	ppgtt = fetch_and_zero(&ggtt->alias);
723 	if (!ppgtt)
724 		return;
725 
726 	i915_vm_put(&ppgtt->vm);
727 
728 	ggtt->vm.vma_ops.bind_vma   = intel_ggtt_bind_vma;
729 	ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
730 }
731 
i915_init_ggtt(struct drm_i915_private * i915)732 int i915_init_ggtt(struct drm_i915_private *i915)
733 {
734 	int ret;
735 
736 	ret = init_ggtt(to_gt(i915)->ggtt);
737 	if (ret)
738 		return ret;
739 
740 	if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
741 		ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
742 		if (ret)
743 			cleanup_init_ggtt(to_gt(i915)->ggtt);
744 	}
745 
746 	return 0;
747 }
748 
ggtt_cleanup_hw(struct i915_ggtt * ggtt)749 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
750 {
751 	struct i915_vma *vma, *vn;
752 
753 	flush_workqueue(ggtt->vm.i915->wq);
754 	i915_gem_drain_freed_objects(ggtt->vm.i915);
755 
756 	mutex_lock(&ggtt->vm.mutex);
757 
758 	ggtt->vm.skip_pte_rewrite = true;
759 
760 	list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
761 		struct drm_i915_gem_object *obj = vma->obj;
762 		bool trylock;
763 
764 		trylock = i915_gem_object_trylock(obj, NULL);
765 		WARN_ON(!trylock);
766 
767 		WARN_ON(__i915_vma_unbind(vma));
768 		if (trylock)
769 			i915_gem_object_unlock(obj);
770 	}
771 
772 	if (drm_mm_node_allocated(&ggtt->error_capture))
773 		drm_mm_remove_node(&ggtt->error_capture);
774 	mutex_destroy(&ggtt->error_mutex);
775 
776 	ggtt_release_guc_top(ggtt);
777 	intel_vgt_deballoon(ggtt);
778 
779 	ggtt->vm.cleanup(&ggtt->vm);
780 
781 	mutex_unlock(&ggtt->vm.mutex);
782 	i915_address_space_fini(&ggtt->vm);
783 
784 	arch_phys_wc_del(ggtt->mtrr);
785 
786 	if (ggtt->iomap.size)
787 		io_mapping_fini(&ggtt->iomap);
788 }
789 
790 /**
791  * i915_ggtt_driver_release - Clean up GGTT hardware initialization
792  * @i915: i915 device
793  */
i915_ggtt_driver_release(struct drm_i915_private * i915)794 void i915_ggtt_driver_release(struct drm_i915_private *i915)
795 {
796 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
797 
798 	fini_aliasing_ppgtt(ggtt);
799 
800 	intel_ggtt_fini_fences(ggtt);
801 	ggtt_cleanup_hw(ggtt);
802 }
803 
804 /**
805  * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
806  * all free objects have been drained.
807  * @i915: i915 device
808  */
i915_ggtt_driver_late_release(struct drm_i915_private * i915)809 void i915_ggtt_driver_late_release(struct drm_i915_private *i915)
810 {
811 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
812 
813 	GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);
814 	dma_resv_fini(&ggtt->vm._resv);
815 }
816 
gen6_get_total_gtt_size(u16 snb_gmch_ctl)817 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
818 {
819 	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
820 	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
821 	return snb_gmch_ctl << 20;
822 }
823 
gen8_get_total_gtt_size(u16 bdw_gmch_ctl)824 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
825 {
826 	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
827 	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
828 	if (bdw_gmch_ctl)
829 		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
830 
831 #ifdef CONFIG_X86_32
832 	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
833 	if (bdw_gmch_ctl > 4)
834 		bdw_gmch_ctl = 4;
835 #endif
836 
837 	return bdw_gmch_ctl << 20;
838 }
839 
chv_get_total_gtt_size(u16 gmch_ctrl)840 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
841 {
842 	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
843 	gmch_ctrl &= SNB_GMCH_GGMS_MASK;
844 
845 	if (gmch_ctrl)
846 		return 1 << (20 + gmch_ctrl);
847 
848 	return 0;
849 }
850 
gen6_gttmmadr_size(struct drm_i915_private * i915)851 static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915)
852 {
853 	/*
854 	 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset
855 	 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset
856 	 */
857 	GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
858 	return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M;
859 }
860 
gen6_gttadr_offset(struct drm_i915_private * i915)861 static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915)
862 {
863 	return gen6_gttmmadr_size(i915) / 2;
864 }
865 
ggtt_probe_common(struct i915_ggtt * ggtt,u64 size)866 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
867 {
868 	struct drm_i915_private *i915 = ggtt->vm.i915;
869 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
870 	phys_addr_t phys_addr;
871 	u32 pte_flags;
872 	int ret;
873 
874 	GEM_WARN_ON(pci_resource_len(pdev, GTTMMADR_BAR) != gen6_gttmmadr_size(i915));
875 	phys_addr = pci_resource_start(pdev, GTTMMADR_BAR) + gen6_gttadr_offset(i915);
876 
877 	/*
878 	 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
879 	 * will be dropped. For WC mappings in general we have 64 byte burst
880 	 * writes when the WC buffer is flushed, so we can't use it, but have to
881 	 * resort to an uncached mapping. The WC issue is easily caught by the
882 	 * readback check when writing GTT PTE entries.
883 	 */
884 	if (IS_GEN9_LP(i915) || GRAPHICS_VER(i915) >= 11)
885 		ggtt->gsm = ioremap(phys_addr, size);
886 	else
887 		ggtt->gsm = ioremap_wc(phys_addr, size);
888 	if (!ggtt->gsm) {
889 		drm_err(&i915->drm, "Failed to map the ggtt page table\n");
890 		return -ENOMEM;
891 	}
892 
893 	kref_init(&ggtt->vm.resv_ref);
894 	ret = setup_scratch_page(&ggtt->vm);
895 	if (ret) {
896 		drm_err(&i915->drm, "Scratch setup failed\n");
897 		/* iounmap will also get called at remove, but meh */
898 		iounmap(ggtt->gsm);
899 		return ret;
900 	}
901 
902 	pte_flags = 0;
903 	if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
904 		pte_flags |= PTE_LM;
905 
906 	ggtt->vm.scratch[0]->encode =
907 		ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),
908 				    I915_CACHE_NONE, pte_flags);
909 
910 	return 0;
911 }
912 
gen6_gmch_remove(struct i915_address_space * vm)913 static void gen6_gmch_remove(struct i915_address_space *vm)
914 {
915 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
916 
917 	iounmap(ggtt->gsm);
918 	free_scratch(vm);
919 }
920 
pci_resource(struct pci_dev * pdev,int bar)921 static struct resource pci_resource(struct pci_dev *pdev, int bar)
922 {
923 	return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar),
924 					       pci_resource_len(pdev, bar));
925 }
926 
gen8_gmch_probe(struct i915_ggtt * ggtt)927 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
928 {
929 	struct drm_i915_private *i915 = ggtt->vm.i915;
930 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
931 	unsigned int size;
932 	u16 snb_gmch_ctl;
933 
934 	if (!HAS_LMEM(i915)) {
935 		if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR))
936 			return -ENXIO;
937 
938 		ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR);
939 		ggtt->mappable_end = resource_size(&ggtt->gmadr);
940 	}
941 
942 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
943 	if (IS_CHERRYVIEW(i915))
944 		size = chv_get_total_gtt_size(snb_gmch_ctl);
945 	else
946 		size = gen8_get_total_gtt_size(snb_gmch_ctl);
947 
948 	ggtt->vm.alloc_pt_dma = alloc_pt_dma;
949 	ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
950 	ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY;
951 
952 	ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
953 	ggtt->vm.cleanup = gen6_gmch_remove;
954 	ggtt->vm.insert_page = gen8_ggtt_insert_page;
955 	ggtt->vm.clear_range = nop_clear_range;
956 	if (intel_scanout_needs_vtd_wa(i915))
957 		ggtt->vm.clear_range = gen8_ggtt_clear_range;
958 
959 	ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
960 
961 	/*
962 	 * Serialize GTT updates with aperture access on BXT if VT-d is on,
963 	 * and always on CHV.
964 	 */
965 	if (intel_vm_no_concurrent_access_wa(i915)) {
966 		ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
967 		ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
968 
969 		/*
970 		 * Calling stop_machine() version of GGTT update function
971 		 * at error capture/reset path will raise lockdep warning.
972 		 * Allow calling gen8_ggtt_insert_* directly at reset path
973 		 * which is safe from parallel GGTT updates.
974 		 */
975 		ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
976 		ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;
977 
978 		ggtt->vm.bind_async_flags =
979 			I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
980 	}
981 
982 	ggtt->invalidate = gen8_ggtt_invalidate;
983 
984 	ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
985 	ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
986 
987 	ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
988 
989 	setup_private_pat(ggtt->vm.gt->uncore);
990 
991 	return ggtt_probe_common(ggtt, size);
992 }
993 
snb_pte_encode(dma_addr_t addr,enum i915_cache_level level,u32 flags)994 static u64 snb_pte_encode(dma_addr_t addr,
995 			  enum i915_cache_level level,
996 			  u32 flags)
997 {
998 	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
999 
1000 	switch (level) {
1001 	case I915_CACHE_L3_LLC:
1002 	case I915_CACHE_LLC:
1003 		pte |= GEN6_PTE_CACHE_LLC;
1004 		break;
1005 	case I915_CACHE_NONE:
1006 		pte |= GEN6_PTE_UNCACHED;
1007 		break;
1008 	default:
1009 		MISSING_CASE(level);
1010 	}
1011 
1012 	return pte;
1013 }
1014 
ivb_pte_encode(dma_addr_t addr,enum i915_cache_level level,u32 flags)1015 static u64 ivb_pte_encode(dma_addr_t addr,
1016 			  enum i915_cache_level level,
1017 			  u32 flags)
1018 {
1019 	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1020 
1021 	switch (level) {
1022 	case I915_CACHE_L3_LLC:
1023 		pte |= GEN7_PTE_CACHE_L3_LLC;
1024 		break;
1025 	case I915_CACHE_LLC:
1026 		pte |= GEN6_PTE_CACHE_LLC;
1027 		break;
1028 	case I915_CACHE_NONE:
1029 		pte |= GEN6_PTE_UNCACHED;
1030 		break;
1031 	default:
1032 		MISSING_CASE(level);
1033 	}
1034 
1035 	return pte;
1036 }
1037 
byt_pte_encode(dma_addr_t addr,enum i915_cache_level level,u32 flags)1038 static u64 byt_pte_encode(dma_addr_t addr,
1039 			  enum i915_cache_level level,
1040 			  u32 flags)
1041 {
1042 	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1043 
1044 	if (!(flags & PTE_READ_ONLY))
1045 		pte |= BYT_PTE_WRITEABLE;
1046 
1047 	if (level != I915_CACHE_NONE)
1048 		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
1049 
1050 	return pte;
1051 }
1052 
hsw_pte_encode(dma_addr_t addr,enum i915_cache_level level,u32 flags)1053 static u64 hsw_pte_encode(dma_addr_t addr,
1054 			  enum i915_cache_level level,
1055 			  u32 flags)
1056 {
1057 	gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1058 
1059 	if (level != I915_CACHE_NONE)
1060 		pte |= HSW_WB_LLC_AGE3;
1061 
1062 	return pte;
1063 }
1064 
iris_pte_encode(dma_addr_t addr,enum i915_cache_level level,u32 flags)1065 static u64 iris_pte_encode(dma_addr_t addr,
1066 			   enum i915_cache_level level,
1067 			   u32 flags)
1068 {
1069 	gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1070 
1071 	switch (level) {
1072 	case I915_CACHE_NONE:
1073 		break;
1074 	case I915_CACHE_WT:
1075 		pte |= HSW_WT_ELLC_LLC_AGE3;
1076 		break;
1077 	default:
1078 		pte |= HSW_WB_ELLC_LLC_AGE3;
1079 		break;
1080 	}
1081 
1082 	return pte;
1083 }
1084 
gen6_gmch_probe(struct i915_ggtt * ggtt)1085 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
1086 {
1087 	struct drm_i915_private *i915 = ggtt->vm.i915;
1088 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1089 	unsigned int size;
1090 	u16 snb_gmch_ctl;
1091 
1092 	if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR))
1093 		return -ENXIO;
1094 
1095 	ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR);
1096 	ggtt->mappable_end = resource_size(&ggtt->gmadr);
1097 
1098 	/*
1099 	 * 64/512MB is the current min/max we actually know of, but this is
1100 	 * just a coarse sanity check.
1101 	 */
1102 	if (ggtt->mappable_end < (64 << 20) ||
1103 	    ggtt->mappable_end > (512 << 20)) {
1104 		drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",
1105 			&ggtt->mappable_end);
1106 		return -ENXIO;
1107 	}
1108 
1109 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1110 
1111 	size = gen6_get_total_gtt_size(snb_gmch_ctl);
1112 	ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
1113 
1114 	ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1115 	ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1116 
1117 	ggtt->vm.clear_range = nop_clear_range;
1118 	if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915))
1119 		ggtt->vm.clear_range = gen6_ggtt_clear_range;
1120 	ggtt->vm.insert_page = gen6_ggtt_insert_page;
1121 	ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
1122 	ggtt->vm.cleanup = gen6_gmch_remove;
1123 
1124 	ggtt->invalidate = gen6_ggtt_invalidate;
1125 
1126 	if (HAS_EDRAM(i915))
1127 		ggtt->vm.pte_encode = iris_pte_encode;
1128 	else if (IS_HASWELL(i915))
1129 		ggtt->vm.pte_encode = hsw_pte_encode;
1130 	else if (IS_VALLEYVIEW(i915))
1131 		ggtt->vm.pte_encode = byt_pte_encode;
1132 	else if (GRAPHICS_VER(i915) >= 7)
1133 		ggtt->vm.pte_encode = ivb_pte_encode;
1134 	else
1135 		ggtt->vm.pte_encode = snb_pte_encode;
1136 
1137 	ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1138 	ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1139 
1140 	return ggtt_probe_common(ggtt, size);
1141 }
1142 
ggtt_probe_hw(struct i915_ggtt * ggtt,struct intel_gt * gt)1143 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1144 {
1145 	struct drm_i915_private *i915 = gt->i915;
1146 	int ret;
1147 
1148 	ggtt->vm.gt = gt;
1149 	ggtt->vm.i915 = i915;
1150 	ggtt->vm.dma = i915->drm.dev;
1151 	dma_resv_init(&ggtt->vm._resv);
1152 
1153 	if (GRAPHICS_VER(i915) >= 8)
1154 		ret = gen8_gmch_probe(ggtt);
1155 	else if (GRAPHICS_VER(i915) >= 6)
1156 		ret = gen6_gmch_probe(ggtt);
1157 	else
1158 		ret = intel_ggtt_gmch_probe(ggtt);
1159 
1160 	if (ret) {
1161 		dma_resv_fini(&ggtt->vm._resv);
1162 		return ret;
1163 	}
1164 
1165 	if ((ggtt->vm.total - 1) >> 32) {
1166 		drm_err(&i915->drm,
1167 			"We never expected a Global GTT with more than 32bits"
1168 			" of address space! Found %lldM!\n",
1169 			ggtt->vm.total >> 20);
1170 		ggtt->vm.total = 1ULL << 32;
1171 		ggtt->mappable_end =
1172 			min_t(u64, ggtt->mappable_end, ggtt->vm.total);
1173 	}
1174 
1175 	if (ggtt->mappable_end > ggtt->vm.total) {
1176 		drm_err(&i915->drm,
1177 			"mappable aperture extends past end of GGTT,"
1178 			" aperture=%pa, total=%llx\n",
1179 			&ggtt->mappable_end, ggtt->vm.total);
1180 		ggtt->mappable_end = ggtt->vm.total;
1181 	}
1182 
1183 	/* GMADR is the PCI mmio aperture into the global GTT. */
1184 	drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
1185 	drm_dbg(&i915->drm, "GMADR size = %lluM\n",
1186 		(u64)ggtt->mappable_end >> 20);
1187 	drm_dbg(&i915->drm, "DSM size = %lluM\n",
1188 		(u64)resource_size(&intel_graphics_stolen_res) >> 20);
1189 
1190 	return 0;
1191 }
1192 
1193 /**
1194  * i915_ggtt_probe_hw - Probe GGTT hardware location
1195  * @i915: i915 device
1196  */
i915_ggtt_probe_hw(struct drm_i915_private * i915)1197 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
1198 {
1199 	int ret;
1200 
1201 	ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
1202 	if (ret)
1203 		return ret;
1204 
1205 	if (i915_vtd_active(i915))
1206 		drm_info(&i915->drm, "VT-d active for gfx access\n");
1207 
1208 	return 0;
1209 }
1210 
i915_ggtt_enable_hw(struct drm_i915_private * i915)1211 int i915_ggtt_enable_hw(struct drm_i915_private *i915)
1212 {
1213 	if (GRAPHICS_VER(i915) < 6)
1214 		return intel_ggtt_gmch_enable_hw(i915);
1215 
1216 	return 0;
1217 }
1218 
i915_ggtt_enable_guc(struct i915_ggtt * ggtt)1219 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt)
1220 {
1221 	GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate);
1222 
1223 	ggtt->invalidate = guc_ggtt_invalidate;
1224 
1225 	ggtt->invalidate(ggtt);
1226 }
1227 
i915_ggtt_disable_guc(struct i915_ggtt * ggtt)1228 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt)
1229 {
1230 	/* XXX Temporary pardon for error unload */
1231 	if (ggtt->invalidate == gen8_ggtt_invalidate)
1232 		return;
1233 
1234 	/* We should only be called after i915_ggtt_enable_guc() */
1235 	GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate);
1236 
1237 	ggtt->invalidate = gen8_ggtt_invalidate;
1238 
1239 	ggtt->invalidate(ggtt);
1240 }
1241 
1242 /**
1243  * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
1244  * @vm: The VM to restore the mappings for
1245  *
1246  * Restore the memory mappings for all objects mapped to HW via the GGTT or a
1247  * DPT page table.
1248  *
1249  * Returns %true if restoring the mapping for any object that was in a write
1250  * domain before suspend.
1251  */
i915_ggtt_resume_vm(struct i915_address_space * vm)1252 bool i915_ggtt_resume_vm(struct i915_address_space *vm)
1253 {
1254 	struct i915_vma *vma;
1255 	bool write_domain_objs = false;
1256 	bool retained_ptes;
1257 
1258 	drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
1259 
1260 	/*
1261 	 * First fill our portion of the GTT with scratch pages if
1262 	 * they were not retained across suspend.
1263 	 */
1264 	retained_ptes = suspend_retains_ptes(vm) &&
1265 		!i915_vm_to_ggtt(vm)->pte_lost &&
1266 		!GEM_WARN_ON(i915_vm_to_ggtt(vm)->probed_pte != read_last_pte(vm));
1267 
1268 	if (!retained_ptes)
1269 		vm->clear_range(vm, 0, vm->total);
1270 
1271 	/* clflush objects bound into the GGTT and rebind them. */
1272 	list_for_each_entry(vma, &vm->bound_list, vm_link) {
1273 		struct drm_i915_gem_object *obj = vma->obj;
1274 		unsigned int was_bound =
1275 			atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
1276 
1277 		GEM_BUG_ON(!was_bound);
1278 		if (!retained_ptes) {
1279 			/*
1280 			 * Clear the bound flags of the vma resource to allow
1281 			 * ptes to be repopulated.
1282 			 */
1283 			vma->resource->bound_flags = 0;
1284 			vma->ops->bind_vma(vm, NULL, vma->resource,
1285 					   obj ? obj->cache_level : 0,
1286 					   was_bound);
1287 		}
1288 		if (obj) { /* only used during resume => exclusive access */
1289 			write_domain_objs |= fetch_and_zero(&obj->write_domain);
1290 			obj->read_domains |= I915_GEM_DOMAIN_GTT;
1291 		}
1292 	}
1293 
1294 	return write_domain_objs;
1295 }
1296 
i915_ggtt_resume(struct i915_ggtt * ggtt)1297 void i915_ggtt_resume(struct i915_ggtt *ggtt)
1298 {
1299 	bool flush;
1300 
1301 	intel_gt_check_and_clear_faults(ggtt->vm.gt);
1302 
1303 	flush = i915_ggtt_resume_vm(&ggtt->vm);
1304 
1305 	ggtt->invalidate(ggtt);
1306 
1307 	if (flush)
1308 		wbinvd_on_all_cpus();
1309 
1310 	if (GRAPHICS_VER(ggtt->vm.i915) >= 8)
1311 		setup_private_pat(ggtt->vm.gt->uncore);
1312 
1313 	intel_ggtt_restore_fences(ggtt);
1314 }
1315 
i915_ggtt_mark_pte_lost(struct drm_i915_private * i915,bool val)1316 void i915_ggtt_mark_pte_lost(struct drm_i915_private *i915, bool val)
1317 {
1318 	to_gt(i915)->ggtt->pte_lost = val;
1319 }
1320