1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * This header is for implementations of dma_map_ops and related code.
4  * It should not be included in drivers just using the DMA API.
5  */
6 #ifndef _LINUX_DMA_MAP_OPS_H
7 #define _LINUX_DMA_MAP_OPS_H
8 
9 #include <linux/dma-mapping.h>
10 #include <linux/pgtable.h>
11 
12 struct cma;
13 
14 struct dma_map_ops {
15 	void *(*alloc)(struct device *dev, size_t size,
16 			dma_addr_t *dma_handle, gfp_t gfp,
17 			unsigned long attrs);
18 	void (*free)(struct device *dev, size_t size, void *vaddr,
19 			dma_addr_t dma_handle, unsigned long attrs);
20 	struct page *(*alloc_pages)(struct device *dev, size_t size,
21 			dma_addr_t *dma_handle, enum dma_data_direction dir,
22 			gfp_t gfp);
23 	void (*free_pages)(struct device *dev, size_t size, struct page *vaddr,
24 			dma_addr_t dma_handle, enum dma_data_direction dir);
25 	struct sg_table *(*alloc_noncontiguous)(struct device *dev, size_t size,
26 			enum dma_data_direction dir, gfp_t gfp,
27 			unsigned long attrs);
28 	void (*free_noncontiguous)(struct device *dev, size_t size,
29 			struct sg_table *sgt, enum dma_data_direction dir);
30 	int (*mmap)(struct device *, struct vm_area_struct *,
31 			void *, dma_addr_t, size_t, unsigned long attrs);
32 
33 	int (*get_sgtable)(struct device *dev, struct sg_table *sgt,
34 			void *cpu_addr, dma_addr_t dma_addr, size_t size,
35 			unsigned long attrs);
36 
37 	dma_addr_t (*map_page)(struct device *dev, struct page *page,
38 			unsigned long offset, size_t size,
39 			enum dma_data_direction dir, unsigned long attrs);
40 	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
41 			size_t size, enum dma_data_direction dir,
42 			unsigned long attrs);
43 	/*
44 	 * map_sg should return a negative error code on error. See
45 	 * dma_map_sgtable() for a list of appropriate error codes
46 	 * and their meanings.
47 	 */
48 	int (*map_sg)(struct device *dev, struct scatterlist *sg, int nents,
49 			enum dma_data_direction dir, unsigned long attrs);
50 	void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents,
51 			enum dma_data_direction dir, unsigned long attrs);
52 	dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
53 			size_t size, enum dma_data_direction dir,
54 			unsigned long attrs);
55 	void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
56 			size_t size, enum dma_data_direction dir,
57 			unsigned long attrs);
58 	void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle,
59 			size_t size, enum dma_data_direction dir);
60 	void (*sync_single_for_device)(struct device *dev,
61 			dma_addr_t dma_handle, size_t size,
62 			enum dma_data_direction dir);
63 	void (*sync_sg_for_cpu)(struct device *dev, struct scatterlist *sg,
64 			int nents, enum dma_data_direction dir);
65 	void (*sync_sg_for_device)(struct device *dev, struct scatterlist *sg,
66 			int nents, enum dma_data_direction dir);
67 	void (*cache_sync)(struct device *dev, void *vaddr, size_t size,
68 			enum dma_data_direction direction);
69 	int (*dma_supported)(struct device *dev, u64 mask);
70 	u64 (*get_required_mask)(struct device *dev);
71 	size_t (*max_mapping_size)(struct device *dev);
72 	unsigned long (*get_merge_boundary)(struct device *dev);
73 };
74 
75 #ifdef CONFIG_DMA_OPS
76 #include <asm/dma-mapping.h>
77 
get_dma_ops(struct device * dev)78 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
79 {
80 	if (dev->dma_ops)
81 		return dev->dma_ops;
82 	return get_arch_dma_ops(dev->bus);
83 }
84 
set_dma_ops(struct device * dev,const struct dma_map_ops * dma_ops)85 static inline void set_dma_ops(struct device *dev,
86 			       const struct dma_map_ops *dma_ops)
87 {
88 	dev->dma_ops = dma_ops;
89 }
90 #else /* CONFIG_DMA_OPS */
get_dma_ops(struct device * dev)91 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
92 {
93 	return NULL;
94 }
set_dma_ops(struct device * dev,const struct dma_map_ops * dma_ops)95 static inline void set_dma_ops(struct device *dev,
96 			       const struct dma_map_ops *dma_ops)
97 {
98 }
99 #endif /* CONFIG_DMA_OPS */
100 
101 #ifdef CONFIG_DMA_CMA
102 extern struct cma *dma_contiguous_default_area;
103 
dev_get_cma_area(struct device * dev)104 static inline struct cma *dev_get_cma_area(struct device *dev)
105 {
106 	if (dev && dev->cma_area)
107 		return dev->cma_area;
108 	return dma_contiguous_default_area;
109 }
110 
111 void dma_contiguous_reserve(phys_addr_t addr_limit);
112 int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base,
113 		phys_addr_t limit, struct cma **res_cma, bool fixed);
114 
115 struct page *dma_alloc_from_contiguous(struct device *dev, size_t count,
116 				       unsigned int order, bool no_warn);
117 bool dma_release_from_contiguous(struct device *dev, struct page *pages,
118 				 int count);
119 struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp);
120 void dma_free_contiguous(struct device *dev, struct page *page, size_t size);
121 
122 void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size);
123 #else /* CONFIG_DMA_CMA */
dev_get_cma_area(struct device * dev)124 static inline struct cma *dev_get_cma_area(struct device *dev)
125 {
126 	return NULL;
127 }
dma_contiguous_reserve(phys_addr_t limit)128 static inline void dma_contiguous_reserve(phys_addr_t limit)
129 {
130 }
dma_contiguous_reserve_area(phys_addr_t size,phys_addr_t base,phys_addr_t limit,struct cma ** res_cma,bool fixed)131 static inline int dma_contiguous_reserve_area(phys_addr_t size,
132 		phys_addr_t base, phys_addr_t limit, struct cma **res_cma,
133 		bool fixed)
134 {
135 	return -ENOSYS;
136 }
dma_alloc_from_contiguous(struct device * dev,size_t count,unsigned int order,bool no_warn)137 static inline struct page *dma_alloc_from_contiguous(struct device *dev,
138 		size_t count, unsigned int order, bool no_warn)
139 {
140 	return NULL;
141 }
dma_release_from_contiguous(struct device * dev,struct page * pages,int count)142 static inline bool dma_release_from_contiguous(struct device *dev,
143 		struct page *pages, int count)
144 {
145 	return false;
146 }
147 /* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */
dma_alloc_contiguous(struct device * dev,size_t size,gfp_t gfp)148 static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size,
149 		gfp_t gfp)
150 {
151 	return NULL;
152 }
dma_free_contiguous(struct device * dev,struct page * page,size_t size)153 static inline void dma_free_contiguous(struct device *dev, struct page *page,
154 		size_t size)
155 {
156 	__free_pages(page, get_order(size));
157 }
158 #endif /* CONFIG_DMA_CMA*/
159 
160 #ifdef CONFIG_DMA_PERNUMA_CMA
161 void dma_pernuma_cma_reserve(void);
162 #else
dma_pernuma_cma_reserve(void)163 static inline void dma_pernuma_cma_reserve(void) { }
164 #endif /* CONFIG_DMA_PERNUMA_CMA */
165 
166 #ifdef CONFIG_DMA_DECLARE_COHERENT
167 int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
168 		dma_addr_t device_addr, size_t size);
169 int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
170 		dma_addr_t *dma_handle, void **ret);
171 int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
172 int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
173 		void *cpu_addr, size_t size, int *ret);
174 #else
dma_declare_coherent_memory(struct device * dev,phys_addr_t phys_addr,dma_addr_t device_addr,size_t size)175 static inline int dma_declare_coherent_memory(struct device *dev,
176 		phys_addr_t phys_addr, dma_addr_t device_addr, size_t size)
177 {
178 	return -ENOSYS;
179 }
180 #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
181 #define dma_release_from_dev_coherent(dev, order, vaddr) (0)
182 #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
183 #endif /* CONFIG_DMA_DECLARE_COHERENT */
184 
185 #ifdef CONFIG_DMA_GLOBAL_POOL
186 void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
187 		dma_addr_t *dma_handle);
188 int dma_release_from_global_coherent(int order, void *vaddr);
189 int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
190 		size_t size, int *ret);
191 int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
192 #else
dma_alloc_from_global_coherent(struct device * dev,ssize_t size,dma_addr_t * dma_handle)193 static inline void *dma_alloc_from_global_coherent(struct device *dev,
194 		ssize_t size, dma_addr_t *dma_handle)
195 {
196 	return NULL;
197 }
dma_release_from_global_coherent(int order,void * vaddr)198 static inline int dma_release_from_global_coherent(int order, void *vaddr)
199 {
200 	return 0;
201 }
dma_mmap_from_global_coherent(struct vm_area_struct * vma,void * cpu_addr,size_t size,int * ret)202 static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
203 		void *cpu_addr, size_t size, int *ret)
204 {
205 	return 0;
206 }
207 #endif /* CONFIG_DMA_GLOBAL_POOL */
208 
209 /*
210  * This is the actual return value from the ->alloc_noncontiguous method.
211  * The users of the DMA API should only care about the sg_table, but to make
212  * the DMA-API internal vmaping and freeing easier we stash away the page
213  * array as well (except for the fallback case).  This can go away any time,
214  * e.g. when a vmap-variant that takes a scatterlist comes along.
215  */
216 struct dma_sgt_handle {
217 	struct sg_table sgt;
218 	struct page **pages;
219 };
220 #define sgt_handle(sgt) \
221 	container_of((sgt), struct dma_sgt_handle, sgt)
222 
223 int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
224 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
225 		unsigned long attrs);
226 int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
227 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
228 		unsigned long attrs);
229 struct page *dma_common_alloc_pages(struct device *dev, size_t size,
230 		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
231 void dma_common_free_pages(struct device *dev, size_t size, struct page *vaddr,
232 		dma_addr_t dma_handle, enum dma_data_direction dir);
233 
234 struct page **dma_common_find_pages(void *cpu_addr);
235 void *dma_common_contiguous_remap(struct page *page, size_t size, pgprot_t prot,
236 		const void *caller);
237 void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
238 		const void *caller);
239 void dma_common_free_remap(void *cpu_addr, size_t size);
240 
241 struct page *dma_alloc_from_pool(struct device *dev, size_t size,
242 		void **cpu_addr, gfp_t flags,
243 		bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
244 bool dma_free_from_pool(struct device *dev, void *start, size_t size);
245 
246 int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
247 		dma_addr_t dma_start, u64 size);
248 
249 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
250 	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
251 	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
252 extern bool dma_default_coherent;
dev_is_dma_coherent(struct device * dev)253 static inline bool dev_is_dma_coherent(struct device *dev)
254 {
255 	return dev->dma_coherent;
256 }
257 #else
dev_is_dma_coherent(struct device * dev)258 static inline bool dev_is_dma_coherent(struct device *dev)
259 {
260 	return true;
261 }
262 #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
263 
264 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
265 		gfp_t gfp, unsigned long attrs);
266 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
267 		dma_addr_t dma_addr, unsigned long attrs);
268 
269 #ifdef CONFIG_MMU
270 /*
271  * Page protection so that devices that can't snoop CPU caches can use the
272  * memory coherently.  We default to pgprot_noncached which is usually used
273  * for ioremap as a safe bet, but architectures can override this with less
274  * strict semantics if possible.
275  */
276 #ifndef pgprot_dmacoherent
277 #define pgprot_dmacoherent(prot)	pgprot_noncached(prot)
278 #endif
279 
280 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs);
281 #else
dma_pgprot(struct device * dev,pgprot_t prot,unsigned long attrs)282 static inline pgprot_t dma_pgprot(struct device *dev, pgprot_t prot,
283 		unsigned long attrs)
284 {
285 	return prot;	/* no protection bits supported without page tables */
286 }
287 #endif /* CONFIG_MMU */
288 
289 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE
290 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
291 		enum dma_data_direction dir);
292 #else
arch_sync_dma_for_device(phys_addr_t paddr,size_t size,enum dma_data_direction dir)293 static inline void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
294 		enum dma_data_direction dir)
295 {
296 }
297 #endif /* ARCH_HAS_SYNC_DMA_FOR_DEVICE */
298 
299 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU
300 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
301 		enum dma_data_direction dir);
302 #else
arch_sync_dma_for_cpu(phys_addr_t paddr,size_t size,enum dma_data_direction dir)303 static inline void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
304 		enum dma_data_direction dir)
305 {
306 }
307 #endif /* ARCH_HAS_SYNC_DMA_FOR_CPU */
308 
309 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
310 void arch_sync_dma_for_cpu_all(void);
311 #else
arch_sync_dma_for_cpu_all(void)312 static inline void arch_sync_dma_for_cpu_all(void)
313 {
314 }
315 #endif /* CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL */
316 
317 #ifdef CONFIG_ARCH_HAS_DMA_PREP_COHERENT
318 void arch_dma_prep_coherent(struct page *page, size_t size);
319 #else
arch_dma_prep_coherent(struct page * page,size_t size)320 static inline void arch_dma_prep_coherent(struct page *page, size_t size)
321 {
322 }
323 #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
324 
325 #ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
326 void arch_dma_mark_clean(phys_addr_t paddr, size_t size);
327 #else
arch_dma_mark_clean(phys_addr_t paddr,size_t size)328 static inline void arch_dma_mark_clean(phys_addr_t paddr, size_t size)
329 {
330 }
331 #endif /* ARCH_HAS_DMA_MARK_CLEAN */
332 
333 void *arch_dma_set_uncached(void *addr, size_t size);
334 void arch_dma_clear_uncached(void *addr, size_t size);
335 
336 #ifdef CONFIG_ARCH_HAS_DMA_MAP_DIRECT
337 bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr);
338 bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle);
339 bool arch_dma_map_sg_direct(struct device *dev, struct scatterlist *sg,
340 		int nents);
341 bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,
342 		int nents);
343 #else
344 #define arch_dma_map_page_direct(d, a)		(false)
345 #define arch_dma_unmap_page_direct(d, a)	(false)
346 #define arch_dma_map_sg_direct(d, s, n)		(false)
347 #define arch_dma_unmap_sg_direct(d, s, n)	(false)
348 #endif
349 
350 #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
351 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
352 		const struct iommu_ops *iommu, bool coherent);
353 #else
arch_setup_dma_ops(struct device * dev,u64 dma_base,u64 size,const struct iommu_ops * iommu,bool coherent)354 static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
355 		u64 size, const struct iommu_ops *iommu, bool coherent)
356 {
357 }
358 #endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */
359 
360 #ifdef CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS
361 void arch_teardown_dma_ops(struct device *dev);
362 #else
arch_teardown_dma_ops(struct device * dev)363 static inline void arch_teardown_dma_ops(struct device *dev)
364 {
365 }
366 #endif /* CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS */
367 
368 #ifdef CONFIG_DMA_API_DEBUG
369 void dma_debug_add_bus(struct bus_type *bus);
370 void debug_dma_dump_mappings(struct device *dev);
371 #else
dma_debug_add_bus(struct bus_type * bus)372 static inline void dma_debug_add_bus(struct bus_type *bus)
373 {
374 }
debug_dma_dump_mappings(struct device * dev)375 static inline void debug_dma_dump_mappings(struct device *dev)
376 {
377 }
378 #endif /* CONFIG_DMA_API_DEBUG */
379 
380 extern const struct dma_map_ops dma_dummy_ops;
381 
382 #endif /* _LINUX_DMA_MAP_OPS_H */
383