1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * linux/fs/nfs/read.c
4  *
5  * Block I/O for NFS
6  *
7  * Partial copy of Linus' read cache modifications to fs/nfs/file.c
8  * modified for async RPC by okir@monad.swb.de
9  */
10 
11 #include <linux/time.h>
12 #include <linux/kernel.h>
13 #include <linux/errno.h>
14 #include <linux/fcntl.h>
15 #include <linux/stat.h>
16 #include <linux/mm.h>
17 #include <linux/slab.h>
18 #include <linux/pagemap.h>
19 #include <linux/sunrpc/clnt.h>
20 #include <linux/nfs_fs.h>
21 #include <linux/nfs_page.h>
22 #include <linux/module.h>
23 
24 #include "nfs4_fs.h"
25 #include "internal.h"
26 #include "iostat.h"
27 #include "fscache.h"
28 #include "pnfs.h"
29 #include "nfstrace.h"
30 
31 #define NFSDBG_FACILITY		NFSDBG_PAGECACHE
32 
33 static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops;
34 static const struct nfs_rw_ops nfs_rw_read_ops;
35 
36 static struct kmem_cache *nfs_rdata_cachep;
37 
nfs_readhdr_alloc(void)38 static struct nfs_pgio_header *nfs_readhdr_alloc(void)
39 {
40 	struct nfs_pgio_header *p = kmem_cache_zalloc(nfs_rdata_cachep, GFP_KERNEL);
41 
42 	if (p)
43 		p->rw_mode = FMODE_READ;
44 	return p;
45 }
46 
nfs_readhdr_free(struct nfs_pgio_header * rhdr)47 static void nfs_readhdr_free(struct nfs_pgio_header *rhdr)
48 {
49 	kmem_cache_free(nfs_rdata_cachep, rhdr);
50 }
51 
52 static
nfs_return_empty_page(struct page * page)53 int nfs_return_empty_page(struct page *page)
54 {
55 	zero_user(page, 0, PAGE_SIZE);
56 	SetPageUptodate(page);
57 	unlock_page(page);
58 	return 0;
59 }
60 
nfs_pageio_init_read(struct nfs_pageio_descriptor * pgio,struct inode * inode,bool force_mds,const struct nfs_pgio_completion_ops * compl_ops)61 void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio,
62 			      struct inode *inode, bool force_mds,
63 			      const struct nfs_pgio_completion_ops *compl_ops)
64 {
65 	struct nfs_server *server = NFS_SERVER(inode);
66 	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
67 
68 #ifdef CONFIG_NFS_V4_1
69 	if (server->pnfs_curr_ld && !force_mds)
70 		pg_ops = server->pnfs_curr_ld->pg_read_ops;
71 #endif
72 	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_read_ops,
73 			server->rsize, 0);
74 }
75 EXPORT_SYMBOL_GPL(nfs_pageio_init_read);
76 
nfs_pageio_complete_read(struct nfs_pageio_descriptor * pgio)77 static void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio)
78 {
79 	struct nfs_pgio_mirror *pgm;
80 	unsigned long npages;
81 
82 	nfs_pageio_complete(pgio);
83 
84 	/* It doesn't make sense to do mirrored reads! */
85 	WARN_ON_ONCE(pgio->pg_mirror_count != 1);
86 
87 	pgm = &pgio->pg_mirrors[0];
88 	NFS_I(pgio->pg_inode)->read_io += pgm->pg_bytes_written;
89 	npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >> PAGE_SHIFT;
90 	nfs_add_stats(pgio->pg_inode, NFSIOS_READPAGES, npages);
91 }
92 
93 
nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor * pgio)94 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
95 {
96 	struct nfs_pgio_mirror *mirror;
97 
98 	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
99 		pgio->pg_ops->pg_cleanup(pgio);
100 
101 	pgio->pg_ops = &nfs_pgio_rw_ops;
102 
103 	/* read path should never have more than one mirror */
104 	WARN_ON_ONCE(pgio->pg_mirror_count != 1);
105 
106 	mirror = &pgio->pg_mirrors[0];
107 	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
108 }
109 EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
110 
nfs_readpage_release(struct nfs_page * req,int error)111 static void nfs_readpage_release(struct nfs_page *req, int error)
112 {
113 	struct inode *inode = d_inode(nfs_req_openctx(req)->dentry);
114 	struct page *page = req->wb_page;
115 
116 	dprintk("NFS: read done (%s/%llu %d@%lld)\n", inode->i_sb->s_id,
117 		(unsigned long long)NFS_FILEID(inode), req->wb_bytes,
118 		(long long)req_offset(req));
119 
120 	if (nfs_error_is_fatal_on_server(error) && error != -ETIMEDOUT)
121 		SetPageError(page);
122 	if (nfs_page_group_sync_on_bit(req, PG_UNLOCKPAGE)) {
123 		if (PageUptodate(page))
124 			nfs_fscache_write_page(inode, page);
125 		unlock_page(page);
126 	}
127 	nfs_release_request(req);
128 }
129 
130 struct nfs_readdesc {
131 	struct nfs_pageio_descriptor pgio;
132 	struct nfs_open_context *ctx;
133 };
134 
nfs_page_group_set_uptodate(struct nfs_page * req)135 static void nfs_page_group_set_uptodate(struct nfs_page *req)
136 {
137 	if (nfs_page_group_sync_on_bit(req, PG_UPTODATE))
138 		SetPageUptodate(req->wb_page);
139 }
140 
nfs_read_completion(struct nfs_pgio_header * hdr)141 static void nfs_read_completion(struct nfs_pgio_header *hdr)
142 {
143 	unsigned long bytes = 0;
144 	int error;
145 
146 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
147 		goto out;
148 	while (!list_empty(&hdr->pages)) {
149 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);
150 		struct page *page = req->wb_page;
151 		unsigned long start = req->wb_pgbase;
152 		unsigned long end = req->wb_pgbase + req->wb_bytes;
153 
154 		if (test_bit(NFS_IOHDR_EOF, &hdr->flags)) {
155 			/* note: regions of the page not covered by a
156 			 * request are zeroed in readpage_async_filler */
157 			if (bytes > hdr->good_bytes) {
158 				/* nothing in this request was good, so zero
159 				 * the full extent of the request */
160 				zero_user_segment(page, start, end);
161 
162 			} else if (hdr->good_bytes - bytes < req->wb_bytes) {
163 				/* part of this request has good bytes, but
164 				 * not all. zero the bad bytes */
165 				start += hdr->good_bytes - bytes;
166 				WARN_ON(start < req->wb_pgbase);
167 				zero_user_segment(page, start, end);
168 			}
169 		}
170 		error = 0;
171 		bytes += req->wb_bytes;
172 		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
173 			if (bytes <= hdr->good_bytes)
174 				nfs_page_group_set_uptodate(req);
175 			else {
176 				error = hdr->error;
177 				xchg(&nfs_req_openctx(req)->error, error);
178 			}
179 		} else
180 			nfs_page_group_set_uptodate(req);
181 		nfs_list_remove_request(req);
182 		nfs_readpage_release(req, error);
183 	}
184 out:
185 	hdr->release(hdr);
186 }
187 
nfs_initiate_read(struct nfs_pgio_header * hdr,struct rpc_message * msg,const struct nfs_rpc_ops * rpc_ops,struct rpc_task_setup * task_setup_data,int how)188 static void nfs_initiate_read(struct nfs_pgio_header *hdr,
189 			      struct rpc_message *msg,
190 			      const struct nfs_rpc_ops *rpc_ops,
191 			      struct rpc_task_setup *task_setup_data, int how)
192 {
193 	rpc_ops->read_setup(hdr, msg);
194 	trace_nfs_initiate_read(hdr);
195 }
196 
197 static void
nfs_async_read_error(struct list_head * head,int error)198 nfs_async_read_error(struct list_head *head, int error)
199 {
200 	struct nfs_page	*req;
201 
202 	while (!list_empty(head)) {
203 		req = nfs_list_entry(head->next);
204 		nfs_list_remove_request(req);
205 		nfs_readpage_release(req, error);
206 	}
207 }
208 
209 static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops = {
210 	.error_cleanup = nfs_async_read_error,
211 	.completion = nfs_read_completion,
212 };
213 
214 /*
215  * This is the callback from RPC telling us whether a reply was
216  * received or some error occurred (timeout or socket shutdown).
217  */
nfs_readpage_done(struct rpc_task * task,struct nfs_pgio_header * hdr,struct inode * inode)218 static int nfs_readpage_done(struct rpc_task *task,
219 			     struct nfs_pgio_header *hdr,
220 			     struct inode *inode)
221 {
222 	int status = NFS_PROTO(inode)->read_done(task, hdr);
223 	if (status != 0)
224 		return status;
225 
226 	nfs_add_stats(inode, NFSIOS_SERVERREADBYTES, hdr->res.count);
227 	trace_nfs_readpage_done(task, hdr);
228 
229 	if (task->tk_status == -ESTALE) {
230 		nfs_set_inode_stale(inode);
231 		nfs_mark_for_revalidate(inode);
232 	}
233 	return 0;
234 }
235 
nfs_readpage_retry(struct rpc_task * task,struct nfs_pgio_header * hdr)236 static void nfs_readpage_retry(struct rpc_task *task,
237 			       struct nfs_pgio_header *hdr)
238 {
239 	struct nfs_pgio_args *argp = &hdr->args;
240 	struct nfs_pgio_res  *resp = &hdr->res;
241 
242 	/* This is a short read! */
243 	nfs_inc_stats(hdr->inode, NFSIOS_SHORTREAD);
244 	trace_nfs_readpage_short(task, hdr);
245 
246 	/* Has the server at least made some progress? */
247 	if (resp->count == 0) {
248 		nfs_set_pgio_error(hdr, -EIO, argp->offset);
249 		return;
250 	}
251 
252 	/* For non rpc-based layout drivers, retry-through-MDS */
253 	if (!task->tk_ops) {
254 		hdr->pnfs_error = -EAGAIN;
255 		return;
256 	}
257 
258 	/* Yes, so retry the read at the end of the hdr */
259 	hdr->mds_offset += resp->count;
260 	argp->offset += resp->count;
261 	argp->pgbase += resp->count;
262 	argp->count -= resp->count;
263 	resp->count = 0;
264 	resp->eof = 0;
265 	rpc_restart_call_prepare(task);
266 }
267 
nfs_readpage_result(struct rpc_task * task,struct nfs_pgio_header * hdr)268 static void nfs_readpage_result(struct rpc_task *task,
269 				struct nfs_pgio_header *hdr)
270 {
271 	if (hdr->res.eof) {
272 		loff_t pos = hdr->args.offset + hdr->res.count;
273 		unsigned int new = pos - hdr->io_start;
274 
275 		if (hdr->good_bytes > new) {
276 			hdr->good_bytes = new;
277 			set_bit(NFS_IOHDR_EOF, &hdr->flags);
278 			clear_bit(NFS_IOHDR_ERROR, &hdr->flags);
279 		}
280 	} else if (hdr->res.count < hdr->args.count)
281 		nfs_readpage_retry(task, hdr);
282 }
283 
284 static int
readpage_async_filler(struct nfs_readdesc * desc,struct page * page)285 readpage_async_filler(struct nfs_readdesc *desc, struct page *page)
286 {
287 	struct inode *inode = page_file_mapping(page)->host;
288 	unsigned int rsize = NFS_SERVER(inode)->rsize;
289 	struct nfs_page *new;
290 	unsigned int len, aligned_len;
291 	int error;
292 
293 	len = nfs_page_length(page);
294 	if (len == 0)
295 		return nfs_return_empty_page(page);
296 
297 	aligned_len = min_t(unsigned int, ALIGN(len, rsize), PAGE_SIZE);
298 
299 	if (!IS_SYNC(page->mapping->host)) {
300 		error = nfs_fscache_read_page(page->mapping->host, page);
301 		if (error == 0)
302 			goto out_unlock;
303 	}
304 
305 	new = nfs_create_request(desc->ctx, page, 0, aligned_len);
306 	if (IS_ERR(new))
307 		goto out_error;
308 
309 	if (len < PAGE_SIZE)
310 		zero_user_segment(page, len, PAGE_SIZE);
311 	if (!nfs_pageio_add_request(&desc->pgio, new)) {
312 		nfs_list_remove_request(new);
313 		error = desc->pgio.pg_error;
314 		nfs_readpage_release(new, error);
315 		goto out;
316 	}
317 	return 0;
318 out_error:
319 	error = PTR_ERR(new);
320 out_unlock:
321 	unlock_page(page);
322 out:
323 	return error;
324 }
325 
326 /*
327  * Read a page over NFS.
328  * We read the page synchronously in the following case:
329  *  -	The error flag is set for this page. This happens only when a
330  *	previous async read operation failed.
331  */
nfs_read_folio(struct file * file,struct folio * folio)332 int nfs_read_folio(struct file *file, struct folio *folio)
333 {
334 	struct page *page = &folio->page;
335 	struct nfs_readdesc desc;
336 	struct inode *inode = page_file_mapping(page)->host;
337 	int ret;
338 
339 	trace_nfs_aop_readpage(inode, page);
340 	nfs_inc_stats(inode, NFSIOS_VFSREADPAGE);
341 
342 	/*
343 	 * Try to flush any pending writes to the file..
344 	 *
345 	 * NOTE! Because we own the page lock, there cannot
346 	 * be any new pending writes generated at this point
347 	 * for this page (other pages can be written to).
348 	 */
349 	ret = nfs_wb_page(inode, page);
350 	if (ret)
351 		goto out_unlock;
352 	if (PageUptodate(page))
353 		goto out_unlock;
354 
355 	ret = -ESTALE;
356 	if (NFS_STALE(inode))
357 		goto out_unlock;
358 
359 	if (file == NULL) {
360 		ret = -EBADF;
361 		desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
362 		if (desc.ctx == NULL)
363 			goto out_unlock;
364 	} else
365 		desc.ctx = get_nfs_open_context(nfs_file_open_context(file));
366 
367 	xchg(&desc.ctx->error, 0);
368 	nfs_pageio_init_read(&desc.pgio, inode, false,
369 			     &nfs_async_read_completion_ops);
370 
371 	ret = readpage_async_filler(&desc, page);
372 	if (ret)
373 		goto out;
374 
375 	nfs_pageio_complete_read(&desc.pgio);
376 	ret = desc.pgio.pg_error < 0 ? desc.pgio.pg_error : 0;
377 	if (!ret) {
378 		ret = wait_on_page_locked_killable(page);
379 		if (!PageUptodate(page) && !ret)
380 			ret = xchg(&desc.ctx->error, 0);
381 	}
382 out:
383 	put_nfs_open_context(desc.ctx);
384 	trace_nfs_aop_readpage_done(inode, page, ret);
385 	return ret;
386 out_unlock:
387 	unlock_page(page);
388 	trace_nfs_aop_readpage_done(inode, page, ret);
389 	return ret;
390 }
391 
nfs_readahead(struct readahead_control * ractl)392 void nfs_readahead(struct readahead_control *ractl)
393 {
394 	unsigned int nr_pages = readahead_count(ractl);
395 	struct file *file = ractl->file;
396 	struct nfs_readdesc desc;
397 	struct inode *inode = ractl->mapping->host;
398 	struct page *page;
399 	int ret;
400 
401 	trace_nfs_aop_readahead(inode, readahead_pos(ractl), nr_pages);
402 	nfs_inc_stats(inode, NFSIOS_VFSREADPAGES);
403 
404 	ret = -ESTALE;
405 	if (NFS_STALE(inode))
406 		goto out;
407 
408 	if (file == NULL) {
409 		ret = -EBADF;
410 		desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
411 		if (desc.ctx == NULL)
412 			goto out;
413 	} else
414 		desc.ctx = get_nfs_open_context(nfs_file_open_context(file));
415 
416 	nfs_pageio_init_read(&desc.pgio, inode, false,
417 			     &nfs_async_read_completion_ops);
418 
419 	while ((page = readahead_page(ractl)) != NULL) {
420 		ret = readpage_async_filler(&desc, page);
421 		put_page(page);
422 		if (ret)
423 			break;
424 	}
425 
426 	nfs_pageio_complete_read(&desc.pgio);
427 
428 	put_nfs_open_context(desc.ctx);
429 out:
430 	trace_nfs_aop_readahead_done(inode, nr_pages, ret);
431 }
432 
nfs_init_readpagecache(void)433 int __init nfs_init_readpagecache(void)
434 {
435 	nfs_rdata_cachep = kmem_cache_create("nfs_read_data",
436 					     sizeof(struct nfs_pgio_header),
437 					     0, SLAB_HWCACHE_ALIGN,
438 					     NULL);
439 	if (nfs_rdata_cachep == NULL)
440 		return -ENOMEM;
441 
442 	return 0;
443 }
444 
nfs_destroy_readpagecache(void)445 void nfs_destroy_readpagecache(void)
446 {
447 	kmem_cache_destroy(nfs_rdata_cachep);
448 }
449 
450 static const struct nfs_rw_ops nfs_rw_read_ops = {
451 	.rw_alloc_header	= nfs_readhdr_alloc,
452 	.rw_free_header		= nfs_readhdr_free,
453 	.rw_done		= nfs_readpage_done,
454 	.rw_result		= nfs_readpage_result,
455 	.rw_initiate		= nfs_initiate_read,
456 };
457