1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/err.h>
5 #include <linux/mm.h>
6 #include <linux/slab.h>
7 #include <linux/vmalloc.h>
8 #include <linux/pagemap.h>
9 #include <linux/sched.h>
10
11 #include <media/frame_vector.h>
12
13 /**
14 * get_vaddr_frames() - map virtual addresses to pfns
15 * @start: starting user address
16 * @nr_frames: number of pages / pfns from start to map
17 * @vec: structure which receives pages / pfns of the addresses mapped.
18 * It should have space for at least nr_frames entries.
19 *
20 * This function maps virtual addresses from @start and fills @vec structure
21 * with page frame numbers or page pointers to corresponding pages (choice
22 * depends on the type of the vma underlying the virtual address). If @start
23 * belongs to a normal vma, the function grabs reference to each of the pages
24 * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't
25 * touch page structures and the caller must make sure pfns aren't reused for
26 * anything else while he is using them.
27 *
28 * The function returns number of pages mapped which may be less than
29 * @nr_frames. In particular we stop mapping if there are more vmas of
30 * different type underlying the specified range of virtual addresses.
31 * When the function isn't able to map a single page, it returns error.
32 *
33 * This function takes care of grabbing mmap_lock as necessary.
34 */
get_vaddr_frames(unsigned long start,unsigned int nr_frames,struct frame_vector * vec)35 int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
36 struct frame_vector *vec)
37 {
38 int ret;
39
40 if (nr_frames == 0)
41 return 0;
42
43 if (WARN_ON_ONCE(nr_frames > vec->nr_allocated))
44 nr_frames = vec->nr_allocated;
45
46 start = untagged_addr(start);
47
48 ret = pin_user_pages_fast(start, nr_frames,
49 FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM,
50 (struct page **)(vec->ptrs));
51 vec->got_ref = true;
52 vec->is_pfns = false;
53 vec->nr_frames = ret;
54
55 if (likely(ret > 0))
56 return ret;
57
58 /* This used to (racily) return non-refcounted pfns. Let people know */
59 WARN_ONCE(1, "get_vaddr_frames() cannot follow VM_IO mapping");
60 vec->nr_frames = 0;
61 return ret ? ret : -EFAULT;
62 }
63 EXPORT_SYMBOL(get_vaddr_frames);
64
65 /**
66 * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired
67 * them
68 * @vec: frame vector to put
69 *
70 * Drop references to pages if get_vaddr_frames() acquired them. We also
71 * invalidate the frame vector so that it is prepared for the next call into
72 * get_vaddr_frames().
73 */
put_vaddr_frames(struct frame_vector * vec)74 void put_vaddr_frames(struct frame_vector *vec)
75 {
76 struct page **pages;
77
78 if (!vec->got_ref)
79 goto out;
80 pages = frame_vector_pages(vec);
81 /*
82 * frame_vector_pages() might needed to do a conversion when
83 * get_vaddr_frames() got pages but vec was later converted to pfns.
84 * But it shouldn't really fail to convert pfns back...
85 */
86 if (WARN_ON(IS_ERR(pages)))
87 goto out;
88
89 unpin_user_pages(pages, vec->nr_frames);
90 vec->got_ref = false;
91 out:
92 vec->nr_frames = 0;
93 }
94 EXPORT_SYMBOL(put_vaddr_frames);
95
96 /**
97 * frame_vector_to_pages - convert frame vector to contain page pointers
98 * @vec: frame vector to convert
99 *
100 * Convert @vec to contain array of page pointers. If the conversion is
101 * successful, return 0. Otherwise return an error. Note that we do not grab
102 * page references for the page structures.
103 */
frame_vector_to_pages(struct frame_vector * vec)104 int frame_vector_to_pages(struct frame_vector *vec)
105 {
106 int i;
107 unsigned long *nums;
108 struct page **pages;
109
110 if (!vec->is_pfns)
111 return 0;
112 nums = frame_vector_pfns(vec);
113 for (i = 0; i < vec->nr_frames; i++)
114 if (!pfn_valid(nums[i]))
115 return -EINVAL;
116 pages = (struct page **)nums;
117 for (i = 0; i < vec->nr_frames; i++)
118 pages[i] = pfn_to_page(nums[i]);
119 vec->is_pfns = false;
120 return 0;
121 }
122 EXPORT_SYMBOL(frame_vector_to_pages);
123
124 /**
125 * frame_vector_to_pfns - convert frame vector to contain pfns
126 * @vec: frame vector to convert
127 *
128 * Convert @vec to contain array of pfns.
129 */
frame_vector_to_pfns(struct frame_vector * vec)130 void frame_vector_to_pfns(struct frame_vector *vec)
131 {
132 int i;
133 unsigned long *nums;
134 struct page **pages;
135
136 if (vec->is_pfns)
137 return;
138 pages = (struct page **)(vec->ptrs);
139 nums = (unsigned long *)pages;
140 for (i = 0; i < vec->nr_frames; i++)
141 nums[i] = page_to_pfn(pages[i]);
142 vec->is_pfns = true;
143 }
144 EXPORT_SYMBOL(frame_vector_to_pfns);
145
146 /**
147 * frame_vector_create() - allocate & initialize structure for pinned pfns
148 * @nr_frames: number of pfns slots we should reserve
149 *
150 * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns
151 * pfns.
152 */
frame_vector_create(unsigned int nr_frames)153 struct frame_vector *frame_vector_create(unsigned int nr_frames)
154 {
155 struct frame_vector *vec;
156 int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames;
157
158 if (WARN_ON_ONCE(nr_frames == 0))
159 return NULL;
160 /*
161 * This is absurdly high. It's here just to avoid strange effects when
162 * arithmetics overflows.
163 */
164 if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2))
165 return NULL;
166 /*
167 * Avoid higher order allocations, use vmalloc instead. It should
168 * be rare anyway.
169 */
170 vec = kvmalloc(size, GFP_KERNEL);
171 if (!vec)
172 return NULL;
173 vec->nr_allocated = nr_frames;
174 vec->nr_frames = 0;
175 return vec;
176 }
177 EXPORT_SYMBOL(frame_vector_create);
178
179 /**
180 * frame_vector_destroy() - free memory allocated to carry frame vector
181 * @vec: Frame vector to free
182 *
183 * Free structure allocated by frame_vector_create() to carry frames.
184 */
frame_vector_destroy(struct frame_vector * vec)185 void frame_vector_destroy(struct frame_vector *vec)
186 {
187 /* Make sure put_vaddr_frames() got called properly... */
188 VM_BUG_ON(vec->nr_frames > 0);
189 kvfree(vec);
190 }
191 EXPORT_SYMBOL(frame_vector_destroy);
192