1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Dynamic DMA mapping support for AMD Hammer.
4  *
5  * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
6  * This allows to use PCI devices that only support 32bit addresses on systems
7  * with more than 4GB.
8  *
9  * See Documentation/core-api/dma-api-howto.rst for the interface specification.
10  *
11  * Copyright 2002 Andi Kleen, SuSE Labs.
12  */
13 
14 #include <linux/types.h>
15 #include <linux/ctype.h>
16 #include <linux/agp_backend.h>
17 #include <linux/init.h>
18 #include <linux/mm.h>
19 #include <linux/sched.h>
20 #include <linux/sched/debug.h>
21 #include <linux/string.h>
22 #include <linux/spinlock.h>
23 #include <linux/pci.h>
24 #include <linux/topology.h>
25 #include <linux/interrupt.h>
26 #include <linux/bitmap.h>
27 #include <linux/kdebug.h>
28 #include <linux/scatterlist.h>
29 #include <linux/iommu-helper.h>
30 #include <linux/syscore_ops.h>
31 #include <linux/io.h>
32 #include <linux/gfp.h>
33 #include <linux/atomic.h>
34 #include <linux/dma-direct.h>
35 #include <linux/dma-map-ops.h>
36 #include <asm/mtrr.h>
37 #include <asm/proto.h>
38 #include <asm/iommu.h>
39 #include <asm/gart.h>
40 #include <asm/set_memory.h>
41 #include <asm/dma.h>
42 #include <asm/amd_nb.h>
43 #include <asm/x86_init.h>
44 
45 static unsigned long iommu_bus_base;	/* GART remapping area (physical) */
46 static unsigned long iommu_size;	/* size of remapping area bytes */
47 static unsigned long iommu_pages;	/* .. and in pages */
48 
49 static u32 *iommu_gatt_base;		/* Remapping table */
50 
51 /*
52  * If this is disabled the IOMMU will use an optimized flushing strategy
53  * of only flushing when an mapping is reused. With it true the GART is
54  * flushed for every mapping. Problem is that doing the lazy flush seems
55  * to trigger bugs with some popular PCI cards, in particular 3ware (but
56  * has been also also seen with Qlogic at least).
57  */
58 static int iommu_fullflush = 1;
59 
60 /* Allocation bitmap for the remapping area: */
61 static DEFINE_SPINLOCK(iommu_bitmap_lock);
62 /* Guarded by iommu_bitmap_lock: */
63 static unsigned long *iommu_gart_bitmap;
64 
65 static u32 gart_unmapped_entry;
66 
67 #define GPTE_VALID    1
68 #define GPTE_COHERENT 2
69 #define GPTE_ENCODE(x) \
70 	(((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT)
71 #define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28))
72 
73 #ifdef CONFIG_AGP
74 #define AGPEXTERN extern
75 #else
76 #define AGPEXTERN
77 #endif
78 
79 /* GART can only remap to physical addresses < 1TB */
80 #define GART_MAX_PHYS_ADDR	(1ULL << 40)
81 
82 /* backdoor interface to AGP driver */
83 AGPEXTERN int agp_memory_reserved;
84 AGPEXTERN __u32 *agp_gatt_table;
85 
86 static unsigned long next_bit;  /* protected by iommu_bitmap_lock */
87 static bool need_flush;		/* global flush state. set for each gart wrap */
88 
alloc_iommu(struct device * dev,int size,unsigned long align_mask)89 static unsigned long alloc_iommu(struct device *dev, int size,
90 				 unsigned long align_mask)
91 {
92 	unsigned long offset, flags;
93 	unsigned long boundary_size;
94 	unsigned long base_index;
95 
96 	base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev),
97 			   PAGE_SIZE) >> PAGE_SHIFT;
98 	boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
99 
100 	spin_lock_irqsave(&iommu_bitmap_lock, flags);
101 	offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit,
102 				  size, base_index, boundary_size, align_mask);
103 	if (offset == -1) {
104 		need_flush = true;
105 		offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, 0,
106 					  size, base_index, boundary_size,
107 					  align_mask);
108 	}
109 	if (offset != -1) {
110 		next_bit = offset+size;
111 		if (next_bit >= iommu_pages) {
112 			next_bit = 0;
113 			need_flush = true;
114 		}
115 	}
116 	if (iommu_fullflush)
117 		need_flush = true;
118 	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
119 
120 	return offset;
121 }
122 
free_iommu(unsigned long offset,int size)123 static void free_iommu(unsigned long offset, int size)
124 {
125 	unsigned long flags;
126 
127 	spin_lock_irqsave(&iommu_bitmap_lock, flags);
128 	bitmap_clear(iommu_gart_bitmap, offset, size);
129 	if (offset >= next_bit)
130 		next_bit = offset + size;
131 	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
132 }
133 
134 /*
135  * Use global flush state to avoid races with multiple flushers.
136  */
flush_gart(void)137 static void flush_gart(void)
138 {
139 	unsigned long flags;
140 
141 	spin_lock_irqsave(&iommu_bitmap_lock, flags);
142 	if (need_flush) {
143 		amd_flush_garts();
144 		need_flush = false;
145 	}
146 	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
147 }
148 
149 #ifdef CONFIG_IOMMU_LEAK
150 /* Debugging aid for drivers that don't free their IOMMU tables */
dump_leak(void)151 static void dump_leak(void)
152 {
153 	static int dump;
154 
155 	if (dump)
156 		return;
157 	dump = 1;
158 
159 	show_stack(NULL, NULL, KERN_ERR);
160 	debug_dma_dump_mappings(NULL);
161 }
162 #endif
163 
iommu_full(struct device * dev,size_t size,int dir)164 static void iommu_full(struct device *dev, size_t size, int dir)
165 {
166 	/*
167 	 * Ran out of IOMMU space for this operation. This is very bad.
168 	 * Unfortunately the drivers cannot handle this operation properly.
169 	 * Return some non mapped prereserved space in the aperture and
170 	 * let the Northbridge deal with it. This will result in garbage
171 	 * in the IO operation. When the size exceeds the prereserved space
172 	 * memory corruption will occur or random memory will be DMAed
173 	 * out. Hopefully no network devices use single mappings that big.
174 	 */
175 
176 	dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size);
177 #ifdef CONFIG_IOMMU_LEAK
178 	dump_leak();
179 #endif
180 }
181 
182 static inline int
need_iommu(struct device * dev,unsigned long addr,size_t size)183 need_iommu(struct device *dev, unsigned long addr, size_t size)
184 {
185 	return force_iommu || !dma_capable(dev, addr, size, true);
186 }
187 
188 static inline int
nonforced_iommu(struct device * dev,unsigned long addr,size_t size)189 nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
190 {
191 	return !dma_capable(dev, addr, size, true);
192 }
193 
194 /* Map a single continuous physical area into the IOMMU.
195  * Caller needs to check if the iommu is needed and flush.
196  */
dma_map_area(struct device * dev,dma_addr_t phys_mem,size_t size,int dir,unsigned long align_mask)197 static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
198 				size_t size, int dir, unsigned long align_mask)
199 {
200 	unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
201 	unsigned long iommu_page;
202 	int i;
203 
204 	if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR))
205 		return DMA_MAPPING_ERROR;
206 
207 	iommu_page = alloc_iommu(dev, npages, align_mask);
208 	if (iommu_page == -1) {
209 		if (!nonforced_iommu(dev, phys_mem, size))
210 			return phys_mem;
211 		if (panic_on_overflow)
212 			panic("dma_map_area overflow %lu bytes\n", size);
213 		iommu_full(dev, size, dir);
214 		return DMA_MAPPING_ERROR;
215 	}
216 
217 	for (i = 0; i < npages; i++) {
218 		iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem);
219 		phys_mem += PAGE_SIZE;
220 	}
221 	return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
222 }
223 
224 /* Map a single area into the IOMMU */
gart_map_page(struct device * dev,struct page * page,unsigned long offset,size_t size,enum dma_data_direction dir,unsigned long attrs)225 static dma_addr_t gart_map_page(struct device *dev, struct page *page,
226 				unsigned long offset, size_t size,
227 				enum dma_data_direction dir,
228 				unsigned long attrs)
229 {
230 	unsigned long bus;
231 	phys_addr_t paddr = page_to_phys(page) + offset;
232 
233 	if (!need_iommu(dev, paddr, size))
234 		return paddr;
235 
236 	bus = dma_map_area(dev, paddr, size, dir, 0);
237 	flush_gart();
238 
239 	return bus;
240 }
241 
242 /*
243  * Free a DMA mapping.
244  */
gart_unmap_page(struct device * dev,dma_addr_t dma_addr,size_t size,enum dma_data_direction dir,unsigned long attrs)245 static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr,
246 			    size_t size, enum dma_data_direction dir,
247 			    unsigned long attrs)
248 {
249 	unsigned long iommu_page;
250 	int npages;
251 	int i;
252 
253 	if (WARN_ON_ONCE(dma_addr == DMA_MAPPING_ERROR))
254 		return;
255 
256 	/*
257 	 * This driver will not always use a GART mapping, but might have
258 	 * created a direct mapping instead.  If that is the case there is
259 	 * nothing to unmap here.
260 	 */
261 	if (dma_addr < iommu_bus_base ||
262 	    dma_addr >= iommu_bus_base + iommu_size)
263 		return;
264 
265 	iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;
266 	npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
267 	for (i = 0; i < npages; i++) {
268 		iommu_gatt_base[iommu_page + i] = gart_unmapped_entry;
269 	}
270 	free_iommu(iommu_page, npages);
271 }
272 
273 /*
274  * Wrapper for pci_unmap_single working with scatterlists.
275  */
gart_unmap_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)276 static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
277 			  enum dma_data_direction dir, unsigned long attrs)
278 {
279 	struct scatterlist *s;
280 	int i;
281 
282 	for_each_sg(sg, s, nents, i) {
283 		if (!s->dma_length || !s->length)
284 			break;
285 		gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0);
286 	}
287 }
288 
289 /* Fallback for dma_map_sg in case of overflow */
dma_map_sg_nonforce(struct device * dev,struct scatterlist * sg,int nents,int dir)290 static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
291 			       int nents, int dir)
292 {
293 	struct scatterlist *s;
294 	int i;
295 
296 #ifdef CONFIG_IOMMU_DEBUG
297 	pr_debug("dma_map_sg overflow\n");
298 #endif
299 
300 	for_each_sg(sg, s, nents, i) {
301 		unsigned long addr = sg_phys(s);
302 
303 		if (nonforced_iommu(dev, addr, s->length)) {
304 			addr = dma_map_area(dev, addr, s->length, dir, 0);
305 			if (addr == DMA_MAPPING_ERROR) {
306 				if (i > 0)
307 					gart_unmap_sg(dev, sg, i, dir, 0);
308 				nents = 0;
309 				sg[0].dma_length = 0;
310 				break;
311 			}
312 		}
313 		s->dma_address = addr;
314 		s->dma_length = s->length;
315 	}
316 	flush_gart();
317 
318 	return nents;
319 }
320 
321 /* Map multiple scatterlist entries continuous into the first. */
__dma_map_cont(struct device * dev,struct scatterlist * start,int nelems,struct scatterlist * sout,unsigned long pages)322 static int __dma_map_cont(struct device *dev, struct scatterlist *start,
323 			  int nelems, struct scatterlist *sout,
324 			  unsigned long pages)
325 {
326 	unsigned long iommu_start = alloc_iommu(dev, pages, 0);
327 	unsigned long iommu_page = iommu_start;
328 	struct scatterlist *s;
329 	int i;
330 
331 	if (iommu_start == -1)
332 		return -ENOMEM;
333 
334 	for_each_sg(start, s, nelems, i) {
335 		unsigned long pages, addr;
336 		unsigned long phys_addr = s->dma_address;
337 
338 		BUG_ON(s != start && s->offset);
339 		if (s == start) {
340 			sout->dma_address = iommu_bus_base;
341 			sout->dma_address += iommu_page*PAGE_SIZE + s->offset;
342 			sout->dma_length = s->length;
343 		} else {
344 			sout->dma_length += s->length;
345 		}
346 
347 		addr = phys_addr;
348 		pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE);
349 		while (pages--) {
350 			iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr);
351 			addr += PAGE_SIZE;
352 			iommu_page++;
353 		}
354 	}
355 	BUG_ON(iommu_page - iommu_start != pages);
356 
357 	return 0;
358 }
359 
360 static inline int
dma_map_cont(struct device * dev,struct scatterlist * start,int nelems,struct scatterlist * sout,unsigned long pages,int need)361 dma_map_cont(struct device *dev, struct scatterlist *start, int nelems,
362 	     struct scatterlist *sout, unsigned long pages, int need)
363 {
364 	if (!need) {
365 		BUG_ON(nelems != 1);
366 		sout->dma_address = start->dma_address;
367 		sout->dma_length = start->length;
368 		return 0;
369 	}
370 	return __dma_map_cont(dev, start, nelems, sout, pages);
371 }
372 
373 /*
374  * DMA map all entries in a scatterlist.
375  * Merge chunks that have page aligned sizes into a continuous mapping.
376  */
gart_map_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)377 static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
378 		       enum dma_data_direction dir, unsigned long attrs)
379 {
380 	struct scatterlist *s, *ps, *start_sg, *sgmap;
381 	int need = 0, nextneed, i, out, start, ret;
382 	unsigned long pages = 0;
383 	unsigned int seg_size;
384 	unsigned int max_seg_size;
385 
386 	if (nents == 0)
387 		return -EINVAL;
388 
389 	out		= 0;
390 	start		= 0;
391 	start_sg	= sg;
392 	sgmap		= sg;
393 	seg_size	= 0;
394 	max_seg_size	= dma_get_max_seg_size(dev);
395 	ps		= NULL; /* shut up gcc */
396 
397 	for_each_sg(sg, s, nents, i) {
398 		dma_addr_t addr = sg_phys(s);
399 
400 		s->dma_address = addr;
401 		BUG_ON(s->length == 0);
402 
403 		nextneed = need_iommu(dev, addr, s->length);
404 
405 		/* Handle the previous not yet processed entries */
406 		if (i > start) {
407 			/*
408 			 * Can only merge when the last chunk ends on a
409 			 * page boundary and the new one doesn't have an
410 			 * offset.
411 			 */
412 			if (!iommu_merge || !nextneed || !need || s->offset ||
413 			    (s->length + seg_size > max_seg_size) ||
414 			    (ps->offset + ps->length) % PAGE_SIZE) {
415 				ret = dma_map_cont(dev, start_sg, i - start,
416 						   sgmap, pages, need);
417 				if (ret < 0)
418 					goto error;
419 				out++;
420 
421 				seg_size	= 0;
422 				sgmap		= sg_next(sgmap);
423 				pages		= 0;
424 				start		= i;
425 				start_sg	= s;
426 			}
427 		}
428 
429 		seg_size += s->length;
430 		need = nextneed;
431 		pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE);
432 		ps = s;
433 	}
434 	ret = dma_map_cont(dev, start_sg, i - start, sgmap, pages, need);
435 	if (ret < 0)
436 		goto error;
437 	out++;
438 	flush_gart();
439 	if (out < nents) {
440 		sgmap = sg_next(sgmap);
441 		sgmap->dma_length = 0;
442 	}
443 	return out;
444 
445 error:
446 	flush_gart();
447 	gart_unmap_sg(dev, sg, out, dir, 0);
448 
449 	/* When it was forced or merged try again in a dumb way */
450 	if (force_iommu || iommu_merge) {
451 		out = dma_map_sg_nonforce(dev, sg, nents, dir);
452 		if (out > 0)
453 			return out;
454 	}
455 	if (panic_on_overflow)
456 		panic("dma_map_sg: overflow on %lu pages\n", pages);
457 
458 	iommu_full(dev, pages << PAGE_SHIFT, dir);
459 	return ret;
460 }
461 
462 /* allocate and map a coherent mapping */
463 static void *
gart_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_addr,gfp_t flag,unsigned long attrs)464 gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
465 		    gfp_t flag, unsigned long attrs)
466 {
467 	void *vaddr;
468 
469 	vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
470 	if (!vaddr ||
471 	    !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
472 		return vaddr;
473 
474 	*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
475 			DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
476 	flush_gart();
477 	if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
478 		goto out_free;
479 	return vaddr;
480 out_free:
481 	dma_direct_free(dev, size, vaddr, *dma_addr, attrs);
482 	return NULL;
483 }
484 
485 /* free a coherent mapping */
486 static void
gart_free_coherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_addr,unsigned long attrs)487 gart_free_coherent(struct device *dev, size_t size, void *vaddr,
488 		   dma_addr_t dma_addr, unsigned long attrs)
489 {
490 	gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
491 	dma_direct_free(dev, size, vaddr, dma_addr, attrs);
492 }
493 
494 static int no_agp;
495 
check_iommu_size(unsigned long aper,u64 aper_size)496 static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
497 {
498 	unsigned long a;
499 
500 	if (!iommu_size) {
501 		iommu_size = aper_size;
502 		if (!no_agp)
503 			iommu_size /= 2;
504 	}
505 
506 	a = aper + iommu_size;
507 	iommu_size -= round_up(a, PMD_PAGE_SIZE) - a;
508 
509 	if (iommu_size < 64*1024*1024) {
510 		pr_warn("PCI-DMA: Warning: Small IOMMU %luMB."
511 			" Consider increasing the AGP aperture in BIOS\n",
512 			iommu_size >> 20);
513 	}
514 
515 	return iommu_size;
516 }
517 
read_aperture(struct pci_dev * dev,u32 * size)518 static __init unsigned read_aperture(struct pci_dev *dev, u32 *size)
519 {
520 	unsigned aper_size = 0, aper_base_32, aper_order;
521 	u64 aper_base;
522 
523 	pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, &aper_base_32);
524 	pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &aper_order);
525 	aper_order = (aper_order >> 1) & 7;
526 
527 	aper_base = aper_base_32 & 0x7fff;
528 	aper_base <<= 25;
529 
530 	aper_size = (32 * 1024 * 1024) << aper_order;
531 	if (aper_base + aper_size > 0x100000000UL || !aper_size)
532 		aper_base = 0;
533 
534 	*size = aper_size;
535 	return aper_base;
536 }
537 
enable_gart_translations(void)538 static void enable_gart_translations(void)
539 {
540 	int i;
541 
542 	if (!amd_nb_has_feature(AMD_NB_GART))
543 		return;
544 
545 	for (i = 0; i < amd_nb_num(); i++) {
546 		struct pci_dev *dev = node_to_amd_nb(i)->misc;
547 
548 		enable_gart_translation(dev, __pa(agp_gatt_table));
549 	}
550 
551 	/* Flush the GART-TLB to remove stale entries */
552 	amd_flush_garts();
553 }
554 
555 /*
556  * If fix_up_north_bridges is set, the north bridges have to be fixed up on
557  * resume in the same way as they are handled in gart_iommu_hole_init().
558  */
559 static bool fix_up_north_bridges;
560 static u32 aperture_order;
561 static u32 aperture_alloc;
562 
set_up_gart_resume(u32 aper_order,u32 aper_alloc)563 void set_up_gart_resume(u32 aper_order, u32 aper_alloc)
564 {
565 	fix_up_north_bridges = true;
566 	aperture_order = aper_order;
567 	aperture_alloc = aper_alloc;
568 }
569 
gart_fixup_northbridges(void)570 static void gart_fixup_northbridges(void)
571 {
572 	int i;
573 
574 	if (!fix_up_north_bridges)
575 		return;
576 
577 	if (!amd_nb_has_feature(AMD_NB_GART))
578 		return;
579 
580 	pr_info("PCI-DMA: Restoring GART aperture settings\n");
581 
582 	for (i = 0; i < amd_nb_num(); i++) {
583 		struct pci_dev *dev = node_to_amd_nb(i)->misc;
584 
585 		/*
586 		 * Don't enable translations just yet.  That is the next
587 		 * step.  Restore the pre-suspend aperture settings.
588 		 */
589 		gart_set_size_and_enable(dev, aperture_order);
590 		pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, aperture_alloc >> 25);
591 	}
592 }
593 
gart_resume(void)594 static void gart_resume(void)
595 {
596 	pr_info("PCI-DMA: Resuming GART IOMMU\n");
597 
598 	gart_fixup_northbridges();
599 
600 	enable_gart_translations();
601 }
602 
603 static struct syscore_ops gart_syscore_ops = {
604 	.resume		= gart_resume,
605 
606 };
607 
608 /*
609  * Private Northbridge GATT initialization in case we cannot use the
610  * AGP driver for some reason.
611  */
init_amd_gatt(struct agp_kern_info * info)612 static __init int init_amd_gatt(struct agp_kern_info *info)
613 {
614 	unsigned aper_size, gatt_size, new_aper_size;
615 	unsigned aper_base, new_aper_base;
616 	struct pci_dev *dev;
617 	void *gatt;
618 	int i;
619 
620 	pr_info("PCI-DMA: Disabling AGP.\n");
621 
622 	aper_size = aper_base = info->aper_size = 0;
623 	dev = NULL;
624 	for (i = 0; i < amd_nb_num(); i++) {
625 		dev = node_to_amd_nb(i)->misc;
626 		new_aper_base = read_aperture(dev, &new_aper_size);
627 		if (!new_aper_base)
628 			goto nommu;
629 
630 		if (!aper_base) {
631 			aper_size = new_aper_size;
632 			aper_base = new_aper_base;
633 		}
634 		if (aper_size != new_aper_size || aper_base != new_aper_base)
635 			goto nommu;
636 	}
637 	if (!aper_base)
638 		goto nommu;
639 
640 	info->aper_base = aper_base;
641 	info->aper_size = aper_size >> 20;
642 
643 	gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32);
644 	gatt = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
645 					get_order(gatt_size));
646 	if (!gatt)
647 		panic("Cannot allocate GATT table");
648 	if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT))
649 		panic("Could not set GART PTEs to uncacheable pages");
650 
651 	agp_gatt_table = gatt;
652 
653 	register_syscore_ops(&gart_syscore_ops);
654 
655 	flush_gart();
656 
657 	pr_info("PCI-DMA: aperture base @ %x size %u KB\n",
658 	       aper_base, aper_size>>10);
659 
660 	return 0;
661 
662  nommu:
663 	/* Should not happen anymore */
664 	pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n");
665 	return -1;
666 }
667 
668 static const struct dma_map_ops gart_dma_ops = {
669 	.map_sg				= gart_map_sg,
670 	.unmap_sg			= gart_unmap_sg,
671 	.map_page			= gart_map_page,
672 	.unmap_page			= gart_unmap_page,
673 	.alloc				= gart_alloc_coherent,
674 	.free				= gart_free_coherent,
675 	.mmap				= dma_common_mmap,
676 	.get_sgtable			= dma_common_get_sgtable,
677 	.dma_supported			= dma_direct_supported,
678 	.get_required_mask		= dma_direct_get_required_mask,
679 	.alloc_pages			= dma_direct_alloc_pages,
680 	.free_pages			= dma_direct_free_pages,
681 };
682 
gart_iommu_shutdown(void)683 static void gart_iommu_shutdown(void)
684 {
685 	struct pci_dev *dev;
686 	int i;
687 
688 	/* don't shutdown it if there is AGP installed */
689 	if (!no_agp)
690 		return;
691 
692 	if (!amd_nb_has_feature(AMD_NB_GART))
693 		return;
694 
695 	for (i = 0; i < amd_nb_num(); i++) {
696 		u32 ctl;
697 
698 		dev = node_to_amd_nb(i)->misc;
699 		pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl);
700 
701 		ctl &= ~GARTEN;
702 
703 		pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl);
704 	}
705 }
706 
gart_iommu_init(void)707 int __init gart_iommu_init(void)
708 {
709 	struct agp_kern_info info;
710 	unsigned long iommu_start;
711 	unsigned long aper_base, aper_size;
712 	unsigned long start_pfn, end_pfn;
713 	unsigned long scratch;
714 
715 	if (!amd_nb_has_feature(AMD_NB_GART))
716 		return 0;
717 
718 #ifndef CONFIG_AGP_AMD64
719 	no_agp = 1;
720 #else
721 	/* Makefile puts PCI initialization via subsys_initcall first. */
722 	/* Add other AMD AGP bridge drivers here */
723 	no_agp = no_agp ||
724 		(agp_amd64_init() < 0) ||
725 		(agp_copy_info(agp_bridge, &info) < 0);
726 #endif
727 
728 	if (no_iommu ||
729 	    (!force_iommu && max_pfn <= MAX_DMA32_PFN) ||
730 	    !gart_iommu_aperture ||
731 	    (no_agp && init_amd_gatt(&info) < 0)) {
732 		if (max_pfn > MAX_DMA32_PFN) {
733 			pr_warn("More than 4GB of memory but GART IOMMU not available.\n");
734 			pr_warn("falling back to iommu=soft.\n");
735 		}
736 		return 0;
737 	}
738 
739 	/* need to map that range */
740 	aper_size	= info.aper_size << 20;
741 	aper_base	= info.aper_base;
742 	end_pfn		= (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT);
743 
744 	start_pfn = PFN_DOWN(aper_base);
745 	if (!pfn_range_is_mapped(start_pfn, end_pfn))
746 		init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT,
747 				    PAGE_KERNEL);
748 
749 	pr_info("PCI-DMA: using GART IOMMU.\n");
750 	iommu_size = check_iommu_size(info.aper_base, aper_size);
751 	iommu_pages = iommu_size >> PAGE_SHIFT;
752 
753 	iommu_gart_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
754 						      get_order(iommu_pages/8));
755 	if (!iommu_gart_bitmap)
756 		panic("Cannot allocate iommu bitmap\n");
757 
758 	pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",
759 	       iommu_size >> 20);
760 
761 	agp_memory_reserved	= iommu_size;
762 	iommu_start		= aper_size - iommu_size;
763 	iommu_bus_base		= info.aper_base + iommu_start;
764 	iommu_gatt_base		= agp_gatt_table + (iommu_start>>PAGE_SHIFT);
765 
766 	/*
767 	 * Unmap the IOMMU part of the GART. The alias of the page is
768 	 * always mapped with cache enabled and there is no full cache
769 	 * coherency across the GART remapping. The unmapping avoids
770 	 * automatic prefetches from the CPU allocating cache lines in
771 	 * there. All CPU accesses are done via the direct mapping to
772 	 * the backing memory. The GART address is only used by PCI
773 	 * devices.
774 	 */
775 	set_memory_np((unsigned long)__va(iommu_bus_base),
776 				iommu_size >> PAGE_SHIFT);
777 	/*
778 	 * Tricky. The GART table remaps the physical memory range,
779 	 * so the CPU wont notice potential aliases and if the memory
780 	 * is remapped to UC later on, we might surprise the PCI devices
781 	 * with a stray writeout of a cacheline. So play it sure and
782 	 * do an explicit, full-scale wbinvd() _after_ having marked all
783 	 * the pages as Not-Present:
784 	 */
785 	wbinvd();
786 
787 	/*
788 	 * Now all caches are flushed and we can safely enable
789 	 * GART hardware.  Doing it early leaves the possibility
790 	 * of stale cache entries that can lead to GART PTE
791 	 * errors.
792 	 */
793 	enable_gart_translations();
794 
795 	/*
796 	 * Try to workaround a bug (thanks to BenH):
797 	 * Set unmapped entries to a scratch page instead of 0.
798 	 * Any prefetches that hit unmapped entries won't get an bus abort
799 	 * then. (P2P bridge may be prefetching on DMA reads).
800 	 */
801 	scratch = get_zeroed_page(GFP_KERNEL);
802 	if (!scratch)
803 		panic("Cannot allocate iommu scratch page");
804 	gart_unmapped_entry = GPTE_ENCODE(__pa(scratch));
805 
806 	flush_gart();
807 	dma_ops = &gart_dma_ops;
808 	x86_platform.iommu_shutdown = gart_iommu_shutdown;
809 	x86_swiotlb_enable = false;
810 
811 	return 0;
812 }
813 
gart_parse_options(char * p)814 void __init gart_parse_options(char *p)
815 {
816 	int arg;
817 
818 	if (isdigit(*p) && get_option(&p, &arg))
819 		iommu_size = arg;
820 	if (!strncmp(p, "fullflush", 9))
821 		iommu_fullflush = 1;
822 	if (!strncmp(p, "nofullflush", 11))
823 		iommu_fullflush = 0;
824 	if (!strncmp(p, "noagp", 5))
825 		no_agp = 1;
826 	if (!strncmp(p, "noaperture", 10))
827 		fix_aperture = 0;
828 	/* duplicated from pci-dma.c */
829 	if (!strncmp(p, "force", 5))
830 		gart_iommu_aperture_allowed = 1;
831 	if (!strncmp(p, "allowed", 7))
832 		gart_iommu_aperture_allowed = 1;
833 	if (!strncmp(p, "memaper", 7)) {
834 		fallback_aper_force = 1;
835 		p += 7;
836 		if (*p == '=') {
837 			++p;
838 			if (get_option(&p, &arg))
839 				fallback_aper_order = arg;
840 		}
841 	}
842 }
843