1242 static int iommu_map_page(struct protection_domain *dom,  in iommu_map_page()  argument
1257 	pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);  in iommu_map_page()
1277 	update_domain(dom);  in iommu_map_page()
1282 static unsigned long iommu_unmap_page(struct protection_domain *dom,  in iommu_unmap_page()  argument
1295 		pte = fetch_pte(dom, bus_addr);  in iommu_unmap_page()
1438 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,  in dma_ops_reserve_addresses()  argument
1442 	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;  in dma_ops_reserve_addresses()
1450 		__set_bit(page, dom->aperture[index]->bitmap);  in dma_ops_reserve_addresses()
1561 					struct dma_ops_domain *dom,  in dma_ops_area_alloc()  argument
1567 	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;  in dma_ops_area_alloc()
1568 	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;  in dma_ops_area_alloc()
1580 		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;  in dma_ops_area_alloc()
1582 		if (dom->aperture[i]->offset >= dma_mask)  in dma_ops_area_alloc()
1588 		address = iommu_area_alloc(dom->aperture[i]->bitmap,  in dma_ops_area_alloc()
1592 			address = dom->aperture[i]->offset +  in dma_ops_area_alloc()
1594 			dom->next_address = address + (pages << PAGE_SHIFT);  in dma_ops_area_alloc()
1605 					     struct dma_ops_domain *dom,  in dma_ops_alloc_addresses()  argument
1613 	dom->next_address = 0;  in dma_ops_alloc_addresses()
1614 	dom->need_flush = true;  in dma_ops_alloc_addresses()
1617 	address = dma_ops_area_alloc(dev, dom, pages, align_mask,  in dma_ops_alloc_addresses()
1618 				     dma_mask, dom->next_address);  in dma_ops_alloc_addresses()
1621 		dom->next_address = 0;  in dma_ops_alloc_addresses()
1622 		address = dma_ops_area_alloc(dev, dom, pages, align_mask,  in dma_ops_alloc_addresses()
1624 		dom->need_flush = true;  in dma_ops_alloc_addresses()
1630 	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);  in dma_ops_alloc_addresses()
1640 static void dma_ops_free_addresses(struct dma_ops_domain *dom,  in dma_ops_free_addresses()  argument
1645 	struct aperture_range *range = dom->aperture[i];  in dma_ops_free_addresses()
1654 	if (address >= dom->next_address)  in dma_ops_free_addresses()
1655 		dom->need_flush = true;  in dma_ops_free_addresses()
1801 static void dma_ops_domain_free(struct dma_ops_domain *dom)  in dma_ops_domain_free()  argument
1805 	if (!dom)  in dma_ops_domain_free()
1808 	del_domain_from_list(&dom->domain);  in dma_ops_domain_free()
1810 	free_pagetable(&dom->domain);  in dma_ops_domain_free()
1813 		if (!dom->aperture[i])  in dma_ops_domain_free()
1815 		free_page((unsigned long)dom->aperture[i]->bitmap);  in dma_ops_domain_free()
1816 		kfree(dom->aperture[i]);  in dma_ops_domain_free()
1819 	kfree(dom);  in dma_ops_domain_free()
2227 	struct protection_domain *dom = NULL;  in domain_for_device()  local
2241 			dom = alias_data->domain;  in domain_for_device()
2246 	return dom;  in domain_for_device()
2399 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,  in dma_ops_get_pte()  argument
2405 	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];  in dma_ops_get_pte()
2411 		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,  in dma_ops_get_pte()
2417 	update_domain(&dom->domain);  in dma_ops_get_pte()
2426 static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,  in dma_ops_domain_map()  argument
2433 	WARN_ON(address > dom->aperture_size);  in dma_ops_domain_map()
2437 	pte  = dma_ops_get_pte(dom, address);  in dma_ops_domain_map()
2460 static void dma_ops_domain_unmap(struct dma_ops_domain *dom,  in dma_ops_domain_unmap()  argument
2466 	if (address >= dom->aperture_size)  in dma_ops_domain_unmap()
2469 	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];  in dma_ops_domain_unmap()
3096 static int amd_iommu_domain_init(struct iommu_domain *dom)  in amd_iommu_domain_init()  argument
3109 	domain->iommu_domain = dom;  in amd_iommu_domain_init()
3111 	dom->priv = domain;  in amd_iommu_domain_init()
3121 static void amd_iommu_domain_destroy(struct iommu_domain *dom)  in amd_iommu_domain_destroy()  argument
3123 	struct protection_domain *domain = dom->priv;  in amd_iommu_domain_destroy()
3141 	dom->priv = NULL;  in amd_iommu_domain_destroy()
3144 static void amd_iommu_detach_device(struct iommu_domain *dom,  in amd_iommu_detach_device()  argument
3166 static int amd_iommu_attach_device(struct iommu_domain *dom,  in amd_iommu_attach_device()  argument
3169 	struct protection_domain *domain = dom->priv;  in amd_iommu_attach_device()
3193 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,  in amd_iommu_map()  argument
3196 	struct protection_domain *domain = dom->priv;  in amd_iommu_map()
3215 static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,  in amd_iommu_unmap()  argument
3218 	struct protection_domain *domain = dom->priv;  in amd_iommu_unmap()
3233 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,  in amd_iommu_iova_to_phys()  argument
3236 	struct protection_domain *domain = dom->priv;  in amd_iommu_iova_to_phys()
3362 void amd_iommu_domain_direct_map(struct iommu_domain *dom)  in amd_iommu_domain_direct_map()  argument
3364 	struct protection_domain *domain = dom->priv;  in amd_iommu_domain_direct_map()
3383 int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)  in amd_iommu_domain_enable_v2()  argument
3385 	struct protection_domain *domain = dom->priv;  in amd_iommu_domain_enable_v2()
3494 int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,  in amd_iommu_flush_page()  argument
3497 	struct protection_domain *domain = dom->priv;  in amd_iommu_flush_page()
3517 int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)  in amd_iommu_flush_tlb()  argument
3519 	struct protection_domain *domain = dom->priv;  in amd_iommu_flush_tlb()
3596 int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,  in amd_iommu_domain_set_gcr3()  argument
3599 	struct protection_domain *domain = dom->priv;  in amd_iommu_domain_set_gcr3()
3611 int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)  in amd_iommu_domain_clear_gcr3()  argument
3613 	struct protection_domain *domain = dom->priv;  in amd_iommu_domain_clear_gcr3()