1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
3
4 #include <linux/anon_inodes.h>
5 #include <linux/file.h>
6 #include <linux/fs.h>
7 #include <linux/highmem.h>
8 #include <linux/vfio.h>
9 #include <linux/vfio_pci_core.h>
10
11 #include "vfio_dev.h"
12 #include "cmds.h"
13
14 static struct pds_vfio_lm_file *
pds_vfio_get_lm_file(const struct file_operations * fops,int flags,u64 size)15 pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size)
16 {
17 struct pds_vfio_lm_file *lm_file = NULL;
18 unsigned long long npages;
19 struct page **pages;
20 void *page_mem;
21 const void *p;
22
23 if (!size)
24 return NULL;
25
26 /* Alloc file structure */
27 lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL);
28 if (!lm_file)
29 return NULL;
30
31 /* Create file */
32 lm_file->filep =
33 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags);
34 if (IS_ERR(lm_file->filep))
35 goto out_free_file;
36
37 stream_open(lm_file->filep->f_inode, lm_file->filep);
38 mutex_init(&lm_file->lock);
39
40 /* prevent file from being released before we are done with it */
41 get_file(lm_file->filep);
42
43 /* Allocate memory for file pages */
44 npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE);
45 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
46 if (!pages)
47 goto out_put_file;
48
49 page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);
50 if (!page_mem)
51 goto out_free_pages_array;
52
53 p = page_mem - offset_in_page(page_mem);
54 for (unsigned long long i = 0; i < npages; i++) {
55 if (is_vmalloc_addr(p))
56 pages[i] = vmalloc_to_page(p);
57 else
58 pages[i] = kmap_to_page((void *)p);
59 if (!pages[i])
60 goto out_free_page_mem;
61
62 p += PAGE_SIZE;
63 }
64
65 /* Create scatterlist of file pages to use for DMA mapping later */
66 if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0,
67 size, GFP_KERNEL))
68 goto out_free_page_mem;
69
70 lm_file->size = size;
71 lm_file->pages = pages;
72 lm_file->npages = npages;
73 lm_file->page_mem = page_mem;
74 lm_file->alloc_size = npages * PAGE_SIZE;
75
76 return lm_file;
77
78 out_free_page_mem:
79 kvfree(page_mem);
80 out_free_pages_array:
81 kfree(pages);
82 out_put_file:
83 fput(lm_file->filep);
84 mutex_destroy(&lm_file->lock);
85 out_free_file:
86 kfree(lm_file);
87
88 return NULL;
89 }
90
pds_vfio_put_lm_file(struct pds_vfio_lm_file * lm_file)91 static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file)
92 {
93 mutex_lock(&lm_file->lock);
94
95 lm_file->size = 0;
96 lm_file->alloc_size = 0;
97
98 /* Free scatter list of file pages */
99 sg_free_table(&lm_file->sg_table);
100
101 kvfree(lm_file->page_mem);
102 lm_file->page_mem = NULL;
103 kfree(lm_file->pages);
104 lm_file->pages = NULL;
105
106 mutex_unlock(&lm_file->lock);
107
108 /* allow file to be released since we are done with it */
109 fput(lm_file->filep);
110 }
111
pds_vfio_put_save_file(struct pds_vfio_pci_device * pds_vfio)112 void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio)
113 {
114 if (!pds_vfio->save_file)
115 return;
116
117 pds_vfio_put_lm_file(pds_vfio->save_file);
118 pds_vfio->save_file = NULL;
119 }
120
pds_vfio_put_restore_file(struct pds_vfio_pci_device * pds_vfio)121 void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio)
122 {
123 if (!pds_vfio->restore_file)
124 return;
125
126 pds_vfio_put_lm_file(pds_vfio->restore_file);
127 pds_vfio->restore_file = NULL;
128 }
129
pds_vfio_get_file_page(struct pds_vfio_lm_file * lm_file,unsigned long offset)130 static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file,
131 unsigned long offset)
132 {
133 unsigned long cur_offset = 0;
134 struct scatterlist *sg;
135 unsigned int i;
136
137 /* All accesses are sequential */
138 if (offset < lm_file->last_offset || !lm_file->last_offset_sg) {
139 lm_file->last_offset = 0;
140 lm_file->last_offset_sg = lm_file->sg_table.sgl;
141 lm_file->sg_last_entry = 0;
142 }
143
144 cur_offset = lm_file->last_offset;
145
146 for_each_sg(lm_file->last_offset_sg, sg,
147 lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) {
148 if (offset < sg->length + cur_offset) {
149 lm_file->last_offset_sg = sg;
150 lm_file->sg_last_entry += i;
151 lm_file->last_offset = cur_offset;
152 return nth_page(sg_page(sg),
153 (offset - cur_offset) / PAGE_SIZE);
154 }
155 cur_offset += sg->length;
156 }
157
158 return NULL;
159 }
160
pds_vfio_release_file(struct inode * inode,struct file * filp)161 static int pds_vfio_release_file(struct inode *inode, struct file *filp)
162 {
163 struct pds_vfio_lm_file *lm_file = filp->private_data;
164
165 mutex_lock(&lm_file->lock);
166 lm_file->filep->f_pos = 0;
167 lm_file->size = 0;
168 mutex_unlock(&lm_file->lock);
169 mutex_destroy(&lm_file->lock);
170 kfree(lm_file);
171
172 return 0;
173 }
174
pds_vfio_save_read(struct file * filp,char __user * buf,size_t len,loff_t * pos)175 static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf,
176 size_t len, loff_t *pos)
177 {
178 struct pds_vfio_lm_file *lm_file = filp->private_data;
179 ssize_t done = 0;
180
181 if (pos)
182 return -ESPIPE;
183 pos = &filp->f_pos;
184
185 mutex_lock(&lm_file->lock);
186 if (*pos > lm_file->size) {
187 done = -EINVAL;
188 goto out_unlock;
189 }
190
191 len = min_t(size_t, lm_file->size - *pos, len);
192 while (len) {
193 size_t page_offset;
194 struct page *page;
195 size_t page_len;
196 u8 *from_buff;
197 int err;
198
199 page_offset = (*pos) % PAGE_SIZE;
200 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
201 if (!page) {
202 if (done == 0)
203 done = -EINVAL;
204 goto out_unlock;
205 }
206
207 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
208 from_buff = kmap_local_page(page);
209 err = copy_to_user(buf, from_buff + page_offset, page_len);
210 kunmap_local(from_buff);
211 if (err) {
212 done = -EFAULT;
213 goto out_unlock;
214 }
215 *pos += page_len;
216 len -= page_len;
217 done += page_len;
218 buf += page_len;
219 }
220
221 out_unlock:
222 mutex_unlock(&lm_file->lock);
223 return done;
224 }
225
226 static const struct file_operations pds_vfio_save_fops = {
227 .owner = THIS_MODULE,
228 .read = pds_vfio_save_read,
229 .release = pds_vfio_release_file,
230 .llseek = no_llseek,
231 };
232
pds_vfio_get_save_file(struct pds_vfio_pci_device * pds_vfio)233 static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio)
234 {
235 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
236 struct pds_vfio_lm_file *lm_file;
237 u64 size;
238 int err;
239
240 /* Get live migration state size in this state */
241 err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size);
242 if (err) {
243 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err));
244 return err;
245 }
246
247 dev_dbg(dev, "save status, size = %lld\n", size);
248
249 if (!size) {
250 dev_err(dev, "invalid state size\n");
251 return -EIO;
252 }
253
254 lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size);
255 if (!lm_file) {
256 dev_err(dev, "failed to create save file\n");
257 return -ENOENT;
258 }
259
260 dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n",
261 lm_file->size, lm_file->alloc_size, lm_file->npages);
262
263 pds_vfio->save_file = lm_file;
264
265 return 0;
266 }
267
pds_vfio_restore_write(struct file * filp,const char __user * buf,size_t len,loff_t * pos)268 static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf,
269 size_t len, loff_t *pos)
270 {
271 struct pds_vfio_lm_file *lm_file = filp->private_data;
272 loff_t requested_length;
273 ssize_t done = 0;
274
275 if (pos)
276 return -ESPIPE;
277
278 pos = &filp->f_pos;
279
280 if (*pos < 0 ||
281 check_add_overflow((loff_t)len, *pos, &requested_length))
282 return -EINVAL;
283
284 mutex_lock(&lm_file->lock);
285
286 while (len) {
287 size_t page_offset;
288 struct page *page;
289 size_t page_len;
290 u8 *to_buff;
291 int err;
292
293 page_offset = (*pos) % PAGE_SIZE;
294 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
295 if (!page) {
296 if (done == 0)
297 done = -EINVAL;
298 goto out_unlock;
299 }
300
301 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
302 to_buff = kmap_local_page(page);
303 err = copy_from_user(to_buff + page_offset, buf, page_len);
304 kunmap_local(to_buff);
305 if (err) {
306 done = -EFAULT;
307 goto out_unlock;
308 }
309 *pos += page_len;
310 len -= page_len;
311 done += page_len;
312 buf += page_len;
313 lm_file->size += page_len;
314 }
315 out_unlock:
316 mutex_unlock(&lm_file->lock);
317 return done;
318 }
319
320 static const struct file_operations pds_vfio_restore_fops = {
321 .owner = THIS_MODULE,
322 .write = pds_vfio_restore_write,
323 .release = pds_vfio_release_file,
324 .llseek = no_llseek,
325 };
326
pds_vfio_get_restore_file(struct pds_vfio_pci_device * pds_vfio)327 static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio)
328 {
329 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
330 struct pds_vfio_lm_file *lm_file;
331 u64 size;
332
333 size = sizeof(union pds_lm_dev_state);
334 dev_dbg(dev, "restore status, size = %lld\n", size);
335
336 if (!size) {
337 dev_err(dev, "invalid state size");
338 return -EIO;
339 }
340
341 lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size);
342 if (!lm_file) {
343 dev_err(dev, "failed to create restore file");
344 return -ENOENT;
345 }
346 pds_vfio->restore_file = lm_file;
347
348 return 0;
349 }
350
351 struct file *
pds_vfio_step_device_state_locked(struct pds_vfio_pci_device * pds_vfio,enum vfio_device_mig_state next)352 pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio,
353 enum vfio_device_mig_state next)
354 {
355 enum vfio_device_mig_state cur = pds_vfio->state;
356 int err;
357
358 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) {
359 err = pds_vfio_get_save_file(pds_vfio);
360 if (err)
361 return ERR_PTR(err);
362
363 err = pds_vfio_get_lm_state_cmd(pds_vfio);
364 if (err) {
365 pds_vfio_put_save_file(pds_vfio);
366 return ERR_PTR(err);
367 }
368
369 return pds_vfio->save_file->filep;
370 }
371
372 if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) {
373 pds_vfio_put_save_file(pds_vfio);
374 pds_vfio_dirty_disable(pds_vfio, true);
375 return NULL;
376 }
377
378 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) {
379 err = pds_vfio_get_restore_file(pds_vfio);
380 if (err)
381 return ERR_PTR(err);
382
383 return pds_vfio->restore_file->filep;
384 }
385
386 if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) {
387 err = pds_vfio_set_lm_state_cmd(pds_vfio);
388 if (err)
389 return ERR_PTR(err);
390
391 pds_vfio_put_restore_file(pds_vfio);
392 return NULL;
393 }
394
395 if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
396 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio,
397 PDS_LM_STA_IN_PROGRESS);
398 err = pds_vfio_suspend_device_cmd(pds_vfio,
399 PDS_LM_SUSPEND_RESUME_TYPE_P2P);
400 if (err)
401 return ERR_PTR(err);
402
403 return NULL;
404 }
405
406 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) {
407 err = pds_vfio_resume_device_cmd(pds_vfio,
408 PDS_LM_SUSPEND_RESUME_TYPE_FULL);
409 if (err)
410 return ERR_PTR(err);
411
412 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);
413 return NULL;
414 }
415
416 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
417 err = pds_vfio_resume_device_cmd(pds_vfio,
418 PDS_LM_SUSPEND_RESUME_TYPE_P2P);
419 if (err)
420 return ERR_PTR(err);
421
422 return NULL;
423 }
424
425 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) {
426 err = pds_vfio_suspend_device_cmd(pds_vfio,
427 PDS_LM_SUSPEND_RESUME_TYPE_FULL);
428 if (err)
429 return ERR_PTR(err);
430 return NULL;
431 }
432
433 return ERR_PTR(-EINVAL);
434 }
435