1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * The NFSD open file cache.
4  *
5  * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
6  *
7  * An nfsd_file object is a per-file collection of open state that binds
8  * together:
9  *   - a struct file *
10  *   - a user credential
11  *   - a network namespace
12  *   - a read-ahead context
13  *   - monitoring for writeback errors
14  *
15  * nfsd_file objects are reference-counted. Consumers acquire a new
16  * object via the nfsd_file_acquire API. They manage their interest in
17  * the acquired object, and hence the object's reference count, via
18  * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file
19  * object:
20  *
21  *  * non-garbage-collected: When a consumer wants to precisely control
22  *    the lifetime of a file's open state, it acquires a non-garbage-
23  *    collected nfsd_file. The final nfsd_file_put releases the open
24  *    state immediately.
25  *
26  *  * garbage-collected: When a consumer does not control the lifetime
27  *    of open state, it acquires a garbage-collected nfsd_file. The
28  *    final nfsd_file_put allows the open state to linger for a period
29  *    during which it may be re-used.
30  */
31 
32 #include <linux/hash.h>
33 #include <linux/slab.h>
34 #include <linux/file.h>
35 #include <linux/pagemap.h>
36 #include <linux/sched.h>
37 #include <linux/list_lru.h>
38 #include <linux/fsnotify_backend.h>
39 #include <linux/fsnotify.h>
40 #include <linux/seq_file.h>
41 #include <linux/rhashtable.h>
42 
43 #include "vfs.h"
44 #include "nfsd.h"
45 #include "nfsfh.h"
46 #include "netns.h"
47 #include "filecache.h"
48 #include "trace.h"
49 
50 #define NFSD_LAUNDRETTE_DELAY		     (2 * HZ)
51 
52 #define NFSD_FILE_CACHE_UP		     (0)
53 
54 /* We only care about NFSD_MAY_READ/WRITE for this cache */
55 #define NFSD_FILE_MAY_MASK	(NFSD_MAY_READ|NFSD_MAY_WRITE)
56 
57 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
58 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
59 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
60 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
61 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
62 
63 struct nfsd_fcache_disposal {
64 	struct work_struct work;
65 	spinlock_t lock;
66 	struct list_head freeme;
67 };
68 
69 static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
70 
71 static struct kmem_cache		*nfsd_file_slab;
72 static struct kmem_cache		*nfsd_file_mark_slab;
73 static struct list_lru			nfsd_file_lru;
74 static unsigned long			nfsd_file_flags;
75 static struct fsnotify_group		*nfsd_file_fsnotify_group;
76 static struct delayed_work		nfsd_filecache_laundrette;
77 static struct rhltable			nfsd_file_rhltable
78 						____cacheline_aligned_in_smp;
79 
80 static bool
nfsd_match_cred(const struct cred * c1,const struct cred * c2)81 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
82 {
83 	int i;
84 
85 	if (!uid_eq(c1->fsuid, c2->fsuid))
86 		return false;
87 	if (!gid_eq(c1->fsgid, c2->fsgid))
88 		return false;
89 	if (c1->group_info == NULL || c2->group_info == NULL)
90 		return c1->group_info == c2->group_info;
91 	if (c1->group_info->ngroups != c2->group_info->ngroups)
92 		return false;
93 	for (i = 0; i < c1->group_info->ngroups; i++) {
94 		if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
95 			return false;
96 	}
97 	return true;
98 }
99 
100 static const struct rhashtable_params nfsd_file_rhash_params = {
101 	.key_len		= sizeof_field(struct nfsd_file, nf_inode),
102 	.key_offset		= offsetof(struct nfsd_file, nf_inode),
103 	.head_offset		= offsetof(struct nfsd_file, nf_rlist),
104 
105 	/*
106 	 * Start with a single page hash table to reduce resizing churn
107 	 * on light workloads.
108 	 */
109 	.min_size		= 256,
110 	.automatic_shrinking	= true,
111 };
112 
113 static void
nfsd_file_schedule_laundrette(void)114 nfsd_file_schedule_laundrette(void)
115 {
116 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))
117 		queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
118 				   NFSD_LAUNDRETTE_DELAY);
119 }
120 
121 static void
nfsd_file_slab_free(struct rcu_head * rcu)122 nfsd_file_slab_free(struct rcu_head *rcu)
123 {
124 	struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
125 
126 	put_cred(nf->nf_cred);
127 	kmem_cache_free(nfsd_file_slab, nf);
128 }
129 
130 static void
nfsd_file_mark_free(struct fsnotify_mark * mark)131 nfsd_file_mark_free(struct fsnotify_mark *mark)
132 {
133 	struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
134 						  nfm_mark);
135 
136 	kmem_cache_free(nfsd_file_mark_slab, nfm);
137 }
138 
139 static struct nfsd_file_mark *
nfsd_file_mark_get(struct nfsd_file_mark * nfm)140 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
141 {
142 	if (!refcount_inc_not_zero(&nfm->nfm_ref))
143 		return NULL;
144 	return nfm;
145 }
146 
147 static void
nfsd_file_mark_put(struct nfsd_file_mark * nfm)148 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
149 {
150 	if (refcount_dec_and_test(&nfm->nfm_ref)) {
151 		fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
152 		fsnotify_put_mark(&nfm->nfm_mark);
153 	}
154 }
155 
156 static struct nfsd_file_mark *
nfsd_file_mark_find_or_create(struct nfsd_file * nf,struct inode * inode)157 nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
158 {
159 	int			err;
160 	struct fsnotify_mark	*mark;
161 	struct nfsd_file_mark	*nfm = NULL, *new;
162 
163 	do {
164 		fsnotify_group_lock(nfsd_file_fsnotify_group);
165 		mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
166 					  nfsd_file_fsnotify_group);
167 		if (mark) {
168 			nfm = nfsd_file_mark_get(container_of(mark,
169 						 struct nfsd_file_mark,
170 						 nfm_mark));
171 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
172 			if (nfm) {
173 				fsnotify_put_mark(mark);
174 				break;
175 			}
176 			/* Avoid soft lockup race with nfsd_file_mark_put() */
177 			fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
178 			fsnotify_put_mark(mark);
179 		} else {
180 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
181 		}
182 
183 		/* allocate a new nfm */
184 		new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
185 		if (!new)
186 			return NULL;
187 		fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
188 		new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
189 		refcount_set(&new->nfm_ref, 1);
190 
191 		err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
192 
193 		/*
194 		 * If the add was successful, then return the object.
195 		 * Otherwise, we need to put the reference we hold on the
196 		 * nfm_mark. The fsnotify code will take a reference and put
197 		 * it on failure, so we can't just free it directly. It's also
198 		 * not safe to call fsnotify_destroy_mark on it as the
199 		 * mark->group will be NULL. Thus, we can't let the nfm_ref
200 		 * counter drive the destruction at this point.
201 		 */
202 		if (likely(!err))
203 			nfm = new;
204 		else
205 			fsnotify_put_mark(&new->nfm_mark);
206 	} while (unlikely(err == -EEXIST));
207 
208 	return nfm;
209 }
210 
211 static struct nfsd_file *
nfsd_file_alloc(struct net * net,struct inode * inode,unsigned char need,bool want_gc)212 nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need,
213 		bool want_gc)
214 {
215 	struct nfsd_file *nf;
216 
217 	nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
218 	if (unlikely(!nf))
219 		return NULL;
220 
221 	INIT_LIST_HEAD(&nf->nf_lru);
222 	nf->nf_birthtime = ktime_get();
223 	nf->nf_file = NULL;
224 	nf->nf_cred = get_current_cred();
225 	nf->nf_net = net;
226 	nf->nf_flags = want_gc ?
227 		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) :
228 		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING);
229 	nf->nf_inode = inode;
230 	refcount_set(&nf->nf_ref, 1);
231 	nf->nf_may = need;
232 	nf->nf_mark = NULL;
233 	return nf;
234 }
235 
236 /**
237  * nfsd_file_check_write_error - check for writeback errors on a file
238  * @nf: nfsd_file to check for writeback errors
239  *
240  * Check whether a nfsd_file has an unseen error. Reset the write
241  * verifier if so.
242  */
243 static void
nfsd_file_check_write_error(struct nfsd_file * nf)244 nfsd_file_check_write_error(struct nfsd_file *nf)
245 {
246 	struct file *file = nf->nf_file;
247 
248 	if ((file->f_mode & FMODE_WRITE) &&
249 	    filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
250 		nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
251 }
252 
253 static void
nfsd_file_hash_remove(struct nfsd_file * nf)254 nfsd_file_hash_remove(struct nfsd_file *nf)
255 {
256 	trace_nfsd_file_unhash(nf);
257 	rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,
258 			nfsd_file_rhash_params);
259 }
260 
261 static bool
nfsd_file_unhash(struct nfsd_file * nf)262 nfsd_file_unhash(struct nfsd_file *nf)
263 {
264 	if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
265 		nfsd_file_hash_remove(nf);
266 		return true;
267 	}
268 	return false;
269 }
270 
271 static void
nfsd_file_free(struct nfsd_file * nf)272 nfsd_file_free(struct nfsd_file *nf)
273 {
274 	s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
275 
276 	trace_nfsd_file_free(nf);
277 
278 	this_cpu_inc(nfsd_file_releases);
279 	this_cpu_add(nfsd_file_total_age, age);
280 
281 	nfsd_file_unhash(nf);
282 	if (nf->nf_mark)
283 		nfsd_file_mark_put(nf->nf_mark);
284 	if (nf->nf_file) {
285 		nfsd_file_check_write_error(nf);
286 		filp_close(nf->nf_file, NULL);
287 	}
288 
289 	/*
290 	 * If this item is still linked via nf_lru, that's a bug.
291 	 * WARN and leak it to preserve system stability.
292 	 */
293 	if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
294 		return;
295 
296 	call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
297 }
298 
299 static bool
nfsd_file_check_writeback(struct nfsd_file * nf)300 nfsd_file_check_writeback(struct nfsd_file *nf)
301 {
302 	struct file *file = nf->nf_file;
303 	struct address_space *mapping;
304 
305 	/* File not open for write? */
306 	if (!(file->f_mode & FMODE_WRITE))
307 		return false;
308 
309 	/*
310 	 * Some filesystems (e.g. NFS) flush all dirty data on close.
311 	 * On others, there is no need to wait for writeback.
312 	 */
313 	if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE))
314 		return false;
315 
316 	mapping = file->f_mapping;
317 	return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
318 		mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
319 }
320 
321 
nfsd_file_lru_add(struct nfsd_file * nf)322 static bool nfsd_file_lru_add(struct nfsd_file *nf)
323 {
324 	set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
325 	if (list_lru_add(&nfsd_file_lru, &nf->nf_lru)) {
326 		trace_nfsd_file_lru_add(nf);
327 		return true;
328 	}
329 	return false;
330 }
331 
nfsd_file_lru_remove(struct nfsd_file * nf)332 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
333 {
334 	if (list_lru_del(&nfsd_file_lru, &nf->nf_lru)) {
335 		trace_nfsd_file_lru_del(nf);
336 		return true;
337 	}
338 	return false;
339 }
340 
341 struct nfsd_file *
nfsd_file_get(struct nfsd_file * nf)342 nfsd_file_get(struct nfsd_file *nf)
343 {
344 	if (nf && refcount_inc_not_zero(&nf->nf_ref))
345 		return nf;
346 	return NULL;
347 }
348 
349 /**
350  * nfsd_file_put - put the reference to a nfsd_file
351  * @nf: nfsd_file of which to put the reference
352  *
353  * Put a reference to a nfsd_file. In the non-GC case, we just put the
354  * reference immediately. In the GC case, if the reference would be
355  * the last one, the put it on the LRU instead to be cleaned up later.
356  */
357 void
nfsd_file_put(struct nfsd_file * nf)358 nfsd_file_put(struct nfsd_file *nf)
359 {
360 	might_sleep();
361 	trace_nfsd_file_put(nf);
362 
363 	if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
364 	    test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
365 		/*
366 		 * If this is the last reference (nf_ref == 1), then try to
367 		 * transfer it to the LRU.
368 		 */
369 		if (refcount_dec_not_one(&nf->nf_ref))
370 			return;
371 
372 		/* Try to add it to the LRU.  If that fails, decrement. */
373 		if (nfsd_file_lru_add(nf)) {
374 			/* If it's still hashed, we're done */
375 			if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
376 				nfsd_file_schedule_laundrette();
377 				return;
378 			}
379 
380 			/*
381 			 * We're racing with unhashing, so try to remove it from
382 			 * the LRU. If removal fails, then someone else already
383 			 * has our reference.
384 			 */
385 			if (!nfsd_file_lru_remove(nf))
386 				return;
387 		}
388 	}
389 	if (refcount_dec_and_test(&nf->nf_ref))
390 		nfsd_file_free(nf);
391 }
392 
393 static void
nfsd_file_dispose_list(struct list_head * dispose)394 nfsd_file_dispose_list(struct list_head *dispose)
395 {
396 	struct nfsd_file *nf;
397 
398 	while (!list_empty(dispose)) {
399 		nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
400 		list_del_init(&nf->nf_lru);
401 		nfsd_file_free(nf);
402 	}
403 }
404 
405 /**
406  * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list
407  * @dispose: list of nfsd_files to be disposed
408  *
409  * Transfers each file to the "freeme" list for its nfsd_net, to eventually
410  * be disposed of by the per-net garbage collector.
411  */
412 static void
nfsd_file_dispose_list_delayed(struct list_head * dispose)413 nfsd_file_dispose_list_delayed(struct list_head *dispose)
414 {
415 	while(!list_empty(dispose)) {
416 		struct nfsd_file *nf = list_first_entry(dispose,
417 						struct nfsd_file, nf_lru);
418 		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
419 		struct nfsd_fcache_disposal *l = nn->fcache_disposal;
420 
421 		spin_lock(&l->lock);
422 		list_move_tail(&nf->nf_lru, &l->freeme);
423 		spin_unlock(&l->lock);
424 		queue_work(nfsd_filecache_wq, &l->work);
425 	}
426 }
427 
428 /**
429  * nfsd_file_lru_cb - Examine an entry on the LRU list
430  * @item: LRU entry to examine
431  * @lru: controlling LRU
432  * @lock: LRU list lock (unused)
433  * @arg: dispose list
434  *
435  * Return values:
436  *   %LRU_REMOVED: @item was removed from the LRU
437  *   %LRU_ROTATE: @item is to be moved to the LRU tail
438  *   %LRU_SKIP: @item cannot be evicted
439  */
440 static enum lru_status
nfsd_file_lru_cb(struct list_head * item,struct list_lru_one * lru,spinlock_t * lock,void * arg)441 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
442 		 spinlock_t *lock, void *arg)
443 	__releases(lock)
444 	__acquires(lock)
445 {
446 	struct list_head *head = arg;
447 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
448 
449 	/* We should only be dealing with GC entries here */
450 	WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
451 
452 	/*
453 	 * Don't throw out files that are still undergoing I/O or
454 	 * that have uncleared errors pending.
455 	 */
456 	if (nfsd_file_check_writeback(nf)) {
457 		trace_nfsd_file_gc_writeback(nf);
458 		return LRU_SKIP;
459 	}
460 
461 	/* If it was recently added to the list, skip it */
462 	if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
463 		trace_nfsd_file_gc_referenced(nf);
464 		return LRU_ROTATE;
465 	}
466 
467 	/*
468 	 * Put the reference held on behalf of the LRU. If it wasn't the last
469 	 * one, then just remove it from the LRU and ignore it.
470 	 */
471 	if (!refcount_dec_and_test(&nf->nf_ref)) {
472 		trace_nfsd_file_gc_in_use(nf);
473 		list_lru_isolate(lru, &nf->nf_lru);
474 		return LRU_REMOVED;
475 	}
476 
477 	/* Refcount went to zero. Unhash it and queue it to the dispose list */
478 	nfsd_file_unhash(nf);
479 	list_lru_isolate_move(lru, &nf->nf_lru, head);
480 	this_cpu_inc(nfsd_file_evictions);
481 	trace_nfsd_file_gc_disposed(nf);
482 	return LRU_REMOVED;
483 }
484 
485 static void
nfsd_file_gc(void)486 nfsd_file_gc(void)
487 {
488 	LIST_HEAD(dispose);
489 	unsigned long ret;
490 
491 	ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
492 			    &dispose, list_lru_count(&nfsd_file_lru));
493 	trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
494 	nfsd_file_dispose_list_delayed(&dispose);
495 }
496 
497 static void
nfsd_file_gc_worker(struct work_struct * work)498 nfsd_file_gc_worker(struct work_struct *work)
499 {
500 	nfsd_file_gc();
501 	if (list_lru_count(&nfsd_file_lru))
502 		nfsd_file_schedule_laundrette();
503 }
504 
505 static unsigned long
nfsd_file_lru_count(struct shrinker * s,struct shrink_control * sc)506 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
507 {
508 	return list_lru_count(&nfsd_file_lru);
509 }
510 
511 static unsigned long
nfsd_file_lru_scan(struct shrinker * s,struct shrink_control * sc)512 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
513 {
514 	LIST_HEAD(dispose);
515 	unsigned long ret;
516 
517 	ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
518 				   nfsd_file_lru_cb, &dispose);
519 	trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
520 	nfsd_file_dispose_list_delayed(&dispose);
521 	return ret;
522 }
523 
524 static struct shrinker	nfsd_file_shrinker = {
525 	.scan_objects = nfsd_file_lru_scan,
526 	.count_objects = nfsd_file_lru_count,
527 	.seeks = 1,
528 };
529 
530 /**
531  * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
532  * @nf: nfsd_file to attempt to queue
533  * @dispose: private list to queue successfully-put objects
534  *
535  * Unhash an nfsd_file, try to get a reference to it, and then put that
536  * reference. If it's the last reference, queue it to the dispose list.
537  */
538 static void
nfsd_file_cond_queue(struct nfsd_file * nf,struct list_head * dispose)539 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
540 	__must_hold(RCU)
541 {
542 	int decrement = 1;
543 
544 	/* If we raced with someone else unhashing, ignore it */
545 	if (!nfsd_file_unhash(nf))
546 		return;
547 
548 	/* If we can't get a reference, ignore it */
549 	if (!nfsd_file_get(nf))
550 		return;
551 
552 	/* Extra decrement if we remove from the LRU */
553 	if (nfsd_file_lru_remove(nf))
554 		++decrement;
555 
556 	/* If refcount goes to 0, then put on the dispose list */
557 	if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
558 		list_add(&nf->nf_lru, dispose);
559 		trace_nfsd_file_closing(nf);
560 	}
561 }
562 
563 /**
564  * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
565  * @inode:   inode on which to close out nfsd_files
566  * @dispose: list on which to gather nfsd_files to close out
567  *
568  * An nfsd_file represents a struct file being held open on behalf of nfsd.
569  * An open file however can block other activity (such as leases), or cause
570  * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
571  *
572  * This function is intended to find open nfsd_files when this sort of
573  * conflicting access occurs and then attempt to close those files out.
574  *
575  * Populates the dispose list with entries that have already had their
576  * refcounts go to zero. The actual free of an nfsd_file can be expensive,
577  * so we leave it up to the caller whether it wants to wait or not.
578  */
579 static void
nfsd_file_queue_for_close(struct inode * inode,struct list_head * dispose)580 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
581 {
582 	struct rhlist_head *tmp, *list;
583 	struct nfsd_file *nf;
584 
585 	rcu_read_lock();
586 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
587 			       nfsd_file_rhash_params);
588 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
589 		if (!test_bit(NFSD_FILE_GC, &nf->nf_flags))
590 			continue;
591 		nfsd_file_cond_queue(nf, dispose);
592 	}
593 	rcu_read_unlock();
594 }
595 
596 /**
597  * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
598  * @inode: inode of the file to attempt to remove
599  *
600  * Close out any open nfsd_files that can be reaped for @inode. The
601  * actual freeing is deferred to the dispose_list_delayed infrastructure.
602  *
603  * This is used by the fsnotify callbacks and setlease notifier.
604  */
605 static void
nfsd_file_close_inode(struct inode * inode)606 nfsd_file_close_inode(struct inode *inode)
607 {
608 	LIST_HEAD(dispose);
609 
610 	nfsd_file_queue_for_close(inode, &dispose);
611 	nfsd_file_dispose_list_delayed(&dispose);
612 }
613 
614 /**
615  * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
616  * @inode: inode of the file to attempt to remove
617  *
618  * Close out any open nfsd_files that can be reaped for @inode. The
619  * nfsd_files are closed out synchronously.
620  *
621  * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
622  * when reexporting NFS.
623  */
624 void
nfsd_file_close_inode_sync(struct inode * inode)625 nfsd_file_close_inode_sync(struct inode *inode)
626 {
627 	struct nfsd_file *nf;
628 	LIST_HEAD(dispose);
629 
630 	trace_nfsd_file_close(inode);
631 
632 	nfsd_file_queue_for_close(inode, &dispose);
633 	while (!list_empty(&dispose)) {
634 		nf = list_first_entry(&dispose, struct nfsd_file, nf_lru);
635 		list_del_init(&nf->nf_lru);
636 		nfsd_file_free(nf);
637 	}
638 	flush_delayed_fput();
639 }
640 
641 /**
642  * nfsd_file_delayed_close - close unused nfsd_files
643  * @work: dummy
644  *
645  * Scrape the freeme list for this nfsd_net, and then dispose of them
646  * all.
647  */
648 static void
nfsd_file_delayed_close(struct work_struct * work)649 nfsd_file_delayed_close(struct work_struct *work)
650 {
651 	LIST_HEAD(head);
652 	struct nfsd_fcache_disposal *l = container_of(work,
653 			struct nfsd_fcache_disposal, work);
654 
655 	spin_lock(&l->lock);
656 	list_splice_init(&l->freeme, &head);
657 	spin_unlock(&l->lock);
658 
659 	nfsd_file_dispose_list(&head);
660 }
661 
662 static int
nfsd_file_lease_notifier_call(struct notifier_block * nb,unsigned long arg,void * data)663 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
664 			    void *data)
665 {
666 	struct file_lock *fl = data;
667 
668 	/* Only close files for F_SETLEASE leases */
669 	if (fl->fl_flags & FL_LEASE)
670 		nfsd_file_close_inode(file_inode(fl->fl_file));
671 	return 0;
672 }
673 
674 static struct notifier_block nfsd_file_lease_notifier = {
675 	.notifier_call = nfsd_file_lease_notifier_call,
676 };
677 
678 static int
nfsd_file_fsnotify_handle_event(struct fsnotify_mark * mark,u32 mask,struct inode * inode,struct inode * dir,const struct qstr * name,u32 cookie)679 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
680 				struct inode *inode, struct inode *dir,
681 				const struct qstr *name, u32 cookie)
682 {
683 	if (WARN_ON_ONCE(!inode))
684 		return 0;
685 
686 	trace_nfsd_file_fsnotify_handle_event(inode, mask);
687 
688 	/* Should be no marks on non-regular files */
689 	if (!S_ISREG(inode->i_mode)) {
690 		WARN_ON_ONCE(1);
691 		return 0;
692 	}
693 
694 	/* don't close files if this was not the last link */
695 	if (mask & FS_ATTRIB) {
696 		if (inode->i_nlink)
697 			return 0;
698 	}
699 
700 	nfsd_file_close_inode(inode);
701 	return 0;
702 }
703 
704 
705 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
706 	.handle_inode_event = nfsd_file_fsnotify_handle_event,
707 	.free_mark = nfsd_file_mark_free,
708 };
709 
710 int
nfsd_file_cache_init(void)711 nfsd_file_cache_init(void)
712 {
713 	int ret;
714 
715 	lockdep_assert_held(&nfsd_mutex);
716 	if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
717 		return 0;
718 
719 	ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params);
720 	if (ret)
721 		return ret;
722 
723 	ret = -ENOMEM;
724 	nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
725 	if (!nfsd_filecache_wq)
726 		goto out;
727 
728 	nfsd_file_slab = kmem_cache_create("nfsd_file",
729 				sizeof(struct nfsd_file), 0, 0, NULL);
730 	if (!nfsd_file_slab) {
731 		pr_err("nfsd: unable to create nfsd_file_slab\n");
732 		goto out_err;
733 	}
734 
735 	nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
736 					sizeof(struct nfsd_file_mark), 0, 0, NULL);
737 	if (!nfsd_file_mark_slab) {
738 		pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
739 		goto out_err;
740 	}
741 
742 
743 	ret = list_lru_init(&nfsd_file_lru);
744 	if (ret) {
745 		pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
746 		goto out_err;
747 	}
748 
749 	ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
750 	if (ret) {
751 		pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
752 		goto out_lru;
753 	}
754 
755 	ret = lease_register_notifier(&nfsd_file_lease_notifier);
756 	if (ret) {
757 		pr_err("nfsd: unable to register lease notifier: %d\n", ret);
758 		goto out_shrinker;
759 	}
760 
761 	nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
762 							FSNOTIFY_GROUP_NOFS);
763 	if (IS_ERR(nfsd_file_fsnotify_group)) {
764 		pr_err("nfsd: unable to create fsnotify group: %ld\n",
765 			PTR_ERR(nfsd_file_fsnotify_group));
766 		ret = PTR_ERR(nfsd_file_fsnotify_group);
767 		nfsd_file_fsnotify_group = NULL;
768 		goto out_notifier;
769 	}
770 
771 	INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
772 out:
773 	return ret;
774 out_notifier:
775 	lease_unregister_notifier(&nfsd_file_lease_notifier);
776 out_shrinker:
777 	unregister_shrinker(&nfsd_file_shrinker);
778 out_lru:
779 	list_lru_destroy(&nfsd_file_lru);
780 out_err:
781 	kmem_cache_destroy(nfsd_file_slab);
782 	nfsd_file_slab = NULL;
783 	kmem_cache_destroy(nfsd_file_mark_slab);
784 	nfsd_file_mark_slab = NULL;
785 	destroy_workqueue(nfsd_filecache_wq);
786 	nfsd_filecache_wq = NULL;
787 	rhltable_destroy(&nfsd_file_rhltable);
788 	goto out;
789 }
790 
791 /**
792  * __nfsd_file_cache_purge: clean out the cache for shutdown
793  * @net: net-namespace to shut down the cache (may be NULL)
794  *
795  * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
796  * then close out everything. Called when an nfsd instance is being shut down,
797  * and when the exports table is flushed.
798  */
799 static void
__nfsd_file_cache_purge(struct net * net)800 __nfsd_file_cache_purge(struct net *net)
801 {
802 	struct rhashtable_iter iter;
803 	struct nfsd_file *nf;
804 	LIST_HEAD(dispose);
805 
806 	rhltable_walk_enter(&nfsd_file_rhltable, &iter);
807 	do {
808 		rhashtable_walk_start(&iter);
809 
810 		nf = rhashtable_walk_next(&iter);
811 		while (!IS_ERR_OR_NULL(nf)) {
812 			if (!net || nf->nf_net == net)
813 				nfsd_file_cond_queue(nf, &dispose);
814 			nf = rhashtable_walk_next(&iter);
815 		}
816 
817 		rhashtable_walk_stop(&iter);
818 	} while (nf == ERR_PTR(-EAGAIN));
819 	rhashtable_walk_exit(&iter);
820 
821 	nfsd_file_dispose_list(&dispose);
822 }
823 
824 static struct nfsd_fcache_disposal *
nfsd_alloc_fcache_disposal(void)825 nfsd_alloc_fcache_disposal(void)
826 {
827 	struct nfsd_fcache_disposal *l;
828 
829 	l = kmalloc(sizeof(*l), GFP_KERNEL);
830 	if (!l)
831 		return NULL;
832 	INIT_WORK(&l->work, nfsd_file_delayed_close);
833 	spin_lock_init(&l->lock);
834 	INIT_LIST_HEAD(&l->freeme);
835 	return l;
836 }
837 
838 static void
nfsd_free_fcache_disposal(struct nfsd_fcache_disposal * l)839 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
840 {
841 	cancel_work_sync(&l->work);
842 	nfsd_file_dispose_list(&l->freeme);
843 	kfree(l);
844 }
845 
846 static void
nfsd_free_fcache_disposal_net(struct net * net)847 nfsd_free_fcache_disposal_net(struct net *net)
848 {
849 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
850 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
851 
852 	nfsd_free_fcache_disposal(l);
853 }
854 
855 int
nfsd_file_cache_start_net(struct net * net)856 nfsd_file_cache_start_net(struct net *net)
857 {
858 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
859 
860 	nn->fcache_disposal = nfsd_alloc_fcache_disposal();
861 	return nn->fcache_disposal ? 0 : -ENOMEM;
862 }
863 
864 /**
865  * nfsd_file_cache_purge - Remove all cache items associated with @net
866  * @net: target net namespace
867  *
868  */
869 void
nfsd_file_cache_purge(struct net * net)870 nfsd_file_cache_purge(struct net *net)
871 {
872 	lockdep_assert_held(&nfsd_mutex);
873 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
874 		__nfsd_file_cache_purge(net);
875 }
876 
877 void
nfsd_file_cache_shutdown_net(struct net * net)878 nfsd_file_cache_shutdown_net(struct net *net)
879 {
880 	nfsd_file_cache_purge(net);
881 	nfsd_free_fcache_disposal_net(net);
882 }
883 
884 void
nfsd_file_cache_shutdown(void)885 nfsd_file_cache_shutdown(void)
886 {
887 	int i;
888 
889 	lockdep_assert_held(&nfsd_mutex);
890 	if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
891 		return;
892 
893 	lease_unregister_notifier(&nfsd_file_lease_notifier);
894 	unregister_shrinker(&nfsd_file_shrinker);
895 	/*
896 	 * make sure all callers of nfsd_file_lru_cb are done before
897 	 * calling nfsd_file_cache_purge
898 	 */
899 	cancel_delayed_work_sync(&nfsd_filecache_laundrette);
900 	__nfsd_file_cache_purge(NULL);
901 	list_lru_destroy(&nfsd_file_lru);
902 	rcu_barrier();
903 	fsnotify_put_group(nfsd_file_fsnotify_group);
904 	nfsd_file_fsnotify_group = NULL;
905 	kmem_cache_destroy(nfsd_file_slab);
906 	nfsd_file_slab = NULL;
907 	fsnotify_wait_marks_destroyed();
908 	kmem_cache_destroy(nfsd_file_mark_slab);
909 	nfsd_file_mark_slab = NULL;
910 	destroy_workqueue(nfsd_filecache_wq);
911 	nfsd_filecache_wq = NULL;
912 	rhltable_destroy(&nfsd_file_rhltable);
913 
914 	for_each_possible_cpu(i) {
915 		per_cpu(nfsd_file_cache_hits, i) = 0;
916 		per_cpu(nfsd_file_acquisitions, i) = 0;
917 		per_cpu(nfsd_file_releases, i) = 0;
918 		per_cpu(nfsd_file_total_age, i) = 0;
919 		per_cpu(nfsd_file_evictions, i) = 0;
920 	}
921 }
922 
923 static struct nfsd_file *
nfsd_file_lookup_locked(const struct net * net,const struct cred * cred,struct inode * inode,unsigned char need,bool want_gc)924 nfsd_file_lookup_locked(const struct net *net, const struct cred *cred,
925 			struct inode *inode, unsigned char need,
926 			bool want_gc)
927 {
928 	struct rhlist_head *tmp, *list;
929 	struct nfsd_file *nf;
930 
931 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
932 			       nfsd_file_rhash_params);
933 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
934 		if (nf->nf_may != need)
935 			continue;
936 		if (nf->nf_net != net)
937 			continue;
938 		if (!nfsd_match_cred(nf->nf_cred, cred))
939 			continue;
940 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc)
941 			continue;
942 		if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
943 			continue;
944 
945 		if (!nfsd_file_get(nf))
946 			continue;
947 		return nf;
948 	}
949 	return NULL;
950 }
951 
952 /**
953  * nfsd_file_is_cached - are there any cached open files for this inode?
954  * @inode: inode to check
955  *
956  * The lookup matches inodes in all net namespaces and is atomic wrt
957  * nfsd_file_acquire().
958  *
959  * Return values:
960  *   %true: filecache contains at least one file matching this inode
961  *   %false: filecache contains no files matching this inode
962  */
963 bool
nfsd_file_is_cached(struct inode * inode)964 nfsd_file_is_cached(struct inode *inode)
965 {
966 	struct rhlist_head *tmp, *list;
967 	struct nfsd_file *nf;
968 	bool ret = false;
969 
970 	rcu_read_lock();
971 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
972 			       nfsd_file_rhash_params);
973 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist)
974 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) {
975 			ret = true;
976 			break;
977 		}
978 	rcu_read_unlock();
979 
980 	trace_nfsd_file_is_cached(inode, (int)ret);
981 	return ret;
982 }
983 
984 static __be32
nfsd_file_do_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf,bool want_gc)985 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
986 		     unsigned int may_flags, struct file *file,
987 		     struct nfsd_file **pnf, bool want_gc)
988 {
989 	unsigned char need = may_flags & NFSD_FILE_MAY_MASK;
990 	struct net *net = SVC_NET(rqstp);
991 	struct nfsd_file *new, *nf;
992 	bool stale_retry = true;
993 	bool open_retry = true;
994 	struct inode *inode;
995 	__be32 status;
996 	int ret;
997 
998 retry:
999 	status = fh_verify(rqstp, fhp, S_IFREG,
1000 				may_flags|NFSD_MAY_OWNER_OVERRIDE);
1001 	if (status != nfs_ok)
1002 		return status;
1003 	inode = d_inode(fhp->fh_dentry);
1004 
1005 	rcu_read_lock();
1006 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1007 	rcu_read_unlock();
1008 
1009 	if (nf) {
1010 		/*
1011 		 * If the nf is on the LRU then it holds an extra reference
1012 		 * that must be put if it's removed. It had better not be
1013 		 * the last one however, since we should hold another.
1014 		 */
1015 		if (nfsd_file_lru_remove(nf))
1016 			WARN_ON_ONCE(refcount_dec_and_test(&nf->nf_ref));
1017 		goto wait_for_construction;
1018 	}
1019 
1020 	new = nfsd_file_alloc(net, inode, need, want_gc);
1021 	if (!new) {
1022 		status = nfserr_jukebox;
1023 		goto out;
1024 	}
1025 
1026 	rcu_read_lock();
1027 	spin_lock(&inode->i_lock);
1028 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1029 	if (unlikely(nf)) {
1030 		spin_unlock(&inode->i_lock);
1031 		rcu_read_unlock();
1032 		nfsd_file_slab_free(&new->nf_rcu);
1033 		goto wait_for_construction;
1034 	}
1035 	nf = new;
1036 	ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist,
1037 			      nfsd_file_rhash_params);
1038 	spin_unlock(&inode->i_lock);
1039 	rcu_read_unlock();
1040 	if (likely(ret == 0))
1041 		goto open_file;
1042 
1043 	if (ret == -EEXIST)
1044 		goto retry;
1045 	trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret);
1046 	status = nfserr_jukebox;
1047 	goto construction_err;
1048 
1049 wait_for_construction:
1050 	wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1051 
1052 	/* Did construction of this file fail? */
1053 	if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1054 		trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf);
1055 		if (!open_retry) {
1056 			status = nfserr_jukebox;
1057 			goto construction_err;
1058 		}
1059 		open_retry = false;
1060 		fh_put(fhp);
1061 		goto retry;
1062 	}
1063 	this_cpu_inc(nfsd_file_cache_hits);
1064 
1065 	status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1066 	if (status != nfs_ok) {
1067 		nfsd_file_put(nf);
1068 		nf = NULL;
1069 	}
1070 
1071 out:
1072 	if (status == nfs_ok) {
1073 		this_cpu_inc(nfsd_file_acquisitions);
1074 		nfsd_file_check_write_error(nf);
1075 		*pnf = nf;
1076 	}
1077 	trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status);
1078 	return status;
1079 
1080 open_file:
1081 	trace_nfsd_file_alloc(nf);
1082 	nf->nf_mark = nfsd_file_mark_find_or_create(nf, inode);
1083 	if (nf->nf_mark) {
1084 		if (file) {
1085 			get_file(file);
1086 			nf->nf_file = file;
1087 			status = nfs_ok;
1088 			trace_nfsd_file_opened(nf, status);
1089 		} else {
1090 			ret = nfsd_open_verified(rqstp, fhp, may_flags,
1091 						 &nf->nf_file);
1092 			if (ret == -EOPENSTALE && stale_retry) {
1093 				stale_retry = false;
1094 				nfsd_file_unhash(nf);
1095 				clear_and_wake_up_bit(NFSD_FILE_PENDING,
1096 						      &nf->nf_flags);
1097 				if (refcount_dec_and_test(&nf->nf_ref))
1098 					nfsd_file_free(nf);
1099 				nf = NULL;
1100 				fh_put(fhp);
1101 				goto retry;
1102 			}
1103 			status = nfserrno(ret);
1104 			trace_nfsd_file_open(nf, status);
1105 		}
1106 	} else
1107 		status = nfserr_jukebox;
1108 	/*
1109 	 * If construction failed, or we raced with a call to unlink()
1110 	 * then unhash.
1111 	 */
1112 	if (status != nfs_ok || inode->i_nlink == 0)
1113 		nfsd_file_unhash(nf);
1114 	clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags);
1115 	if (status == nfs_ok)
1116 		goto out;
1117 
1118 construction_err:
1119 	if (refcount_dec_and_test(&nf->nf_ref))
1120 		nfsd_file_free(nf);
1121 	nf = NULL;
1122 	goto out;
1123 }
1124 
1125 /**
1126  * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1127  * @rqstp: the RPC transaction being executed
1128  * @fhp: the NFS filehandle of the file to be opened
1129  * @may_flags: NFSD_MAY_ settings for the file
1130  * @pnf: OUT: new or found "struct nfsd_file" object
1131  *
1132  * The nfsd_file object returned by this API is reference-counted
1133  * and garbage-collected. The object is retained for a few
1134  * seconds after the final nfsd_file_put() in case the caller
1135  * wants to re-use it.
1136  *
1137  * Return values:
1138  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1139  *   count boosted.
1140  *
1141  * On error, an nfsstat value in network byte order is returned.
1142  */
1143 __be32
nfsd_file_acquire_gc(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1144 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1145 		     unsigned int may_flags, struct nfsd_file **pnf)
1146 {
1147 	return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, true);
1148 }
1149 
1150 /**
1151  * nfsd_file_acquire - Get a struct nfsd_file with an open file
1152  * @rqstp: the RPC transaction being executed
1153  * @fhp: the NFS filehandle of the file to be opened
1154  * @may_flags: NFSD_MAY_ settings for the file
1155  * @pnf: OUT: new or found "struct nfsd_file" object
1156  *
1157  * The nfsd_file_object returned by this API is reference-counted
1158  * but not garbage-collected. The object is unhashed after the
1159  * final nfsd_file_put().
1160  *
1161  * Return values:
1162  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1163  *   count boosted.
1164  *
1165  * On error, an nfsstat value in network byte order is returned.
1166  */
1167 __be32
nfsd_file_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1168 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1169 		  unsigned int may_flags, struct nfsd_file **pnf)
1170 {
1171 	return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, false);
1172 }
1173 
1174 /**
1175  * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1176  * @rqstp: the RPC transaction being executed
1177  * @fhp: the NFS filehandle of the file just created
1178  * @may_flags: NFSD_MAY_ settings for the file
1179  * @file: cached, already-open file (may be NULL)
1180  * @pnf: OUT: new or found "struct nfsd_file" object
1181  *
1182  * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1183  * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1184  * opening a new one.
1185  *
1186  * Return values:
1187  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1188  *   count boosted.
1189  *
1190  * On error, an nfsstat value in network byte order is returned.
1191  */
1192 __be32
nfsd_file_acquire_opened(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf)1193 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1194 			 unsigned int may_flags, struct file *file,
1195 			 struct nfsd_file **pnf)
1196 {
1197 	return nfsd_file_do_acquire(rqstp, fhp, may_flags, file, pnf, false);
1198 }
1199 
1200 /*
1201  * Note that fields may be added, removed or reordered in the future. Programs
1202  * scraping this file for info should test the labels to ensure they're
1203  * getting the correct field.
1204  */
nfsd_file_cache_stats_show(struct seq_file * m,void * v)1205 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1206 {
1207 	unsigned long releases = 0, evictions = 0;
1208 	unsigned long hits = 0, acquisitions = 0;
1209 	unsigned int i, count = 0, buckets = 0;
1210 	unsigned long lru = 0, total_age = 0;
1211 
1212 	/* Serialize with server shutdown */
1213 	mutex_lock(&nfsd_mutex);
1214 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1215 		struct bucket_table *tbl;
1216 		struct rhashtable *ht;
1217 
1218 		lru = list_lru_count(&nfsd_file_lru);
1219 
1220 		rcu_read_lock();
1221 		ht = &nfsd_file_rhltable.ht;
1222 		count = atomic_read(&ht->nelems);
1223 		tbl = rht_dereference_rcu(ht->tbl, ht);
1224 		buckets = tbl->size;
1225 		rcu_read_unlock();
1226 	}
1227 	mutex_unlock(&nfsd_mutex);
1228 
1229 	for_each_possible_cpu(i) {
1230 		hits += per_cpu(nfsd_file_cache_hits, i);
1231 		acquisitions += per_cpu(nfsd_file_acquisitions, i);
1232 		releases += per_cpu(nfsd_file_releases, i);
1233 		total_age += per_cpu(nfsd_file_total_age, i);
1234 		evictions += per_cpu(nfsd_file_evictions, i);
1235 	}
1236 
1237 	seq_printf(m, "total inodes:  %u\n", count);
1238 	seq_printf(m, "hash buckets:  %u\n", buckets);
1239 	seq_printf(m, "lru entries:   %lu\n", lru);
1240 	seq_printf(m, "cache hits:    %lu\n", hits);
1241 	seq_printf(m, "acquisitions:  %lu\n", acquisitions);
1242 	seq_printf(m, "releases:      %lu\n", releases);
1243 	seq_printf(m, "evictions:     %lu\n", evictions);
1244 	if (releases)
1245 		seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1246 	else
1247 		seq_printf(m, "mean age (ms): -\n");
1248 	return 0;
1249 }
1250