1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * The NFSD open file cache.
4 *
5 * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
6 *
7 * An nfsd_file object is a per-file collection of open state that binds
8 * together:
9 * - a struct file *
10 * - a user credential
11 * - a network namespace
12 * - a read-ahead context
13 * - monitoring for writeback errors
14 *
15 * nfsd_file objects are reference-counted. Consumers acquire a new
16 * object via the nfsd_file_acquire API. They manage their interest in
17 * the acquired object, and hence the object's reference count, via
18 * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file
19 * object:
20 *
21 * * non-garbage-collected: When a consumer wants to precisely control
22 * the lifetime of a file's open state, it acquires a non-garbage-
23 * collected nfsd_file. The final nfsd_file_put releases the open
24 * state immediately.
25 *
26 * * garbage-collected: When a consumer does not control the lifetime
27 * of open state, it acquires a garbage-collected nfsd_file. The
28 * final nfsd_file_put allows the open state to linger for a period
29 * during which it may be re-used.
30 */
31
32 #include <linux/hash.h>
33 #include <linux/slab.h>
34 #include <linux/file.h>
35 #include <linux/pagemap.h>
36 #include <linux/sched.h>
37 #include <linux/list_lru.h>
38 #include <linux/fsnotify_backend.h>
39 #include <linux/fsnotify.h>
40 #include <linux/seq_file.h>
41 #include <linux/rhashtable.h>
42
43 #include "vfs.h"
44 #include "nfsd.h"
45 #include "nfsfh.h"
46 #include "netns.h"
47 #include "filecache.h"
48 #include "trace.h"
49
50 #define NFSD_LAUNDRETTE_DELAY (2 * HZ)
51
52 #define NFSD_FILE_CACHE_UP (0)
53
54 /* We only care about NFSD_MAY_READ/WRITE for this cache */
55 #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE)
56
57 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
58 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
59 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
60 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
61 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
62
63 struct nfsd_fcache_disposal {
64 struct work_struct work;
65 spinlock_t lock;
66 struct list_head freeme;
67 };
68
69 static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
70
71 static struct kmem_cache *nfsd_file_slab;
72 static struct kmem_cache *nfsd_file_mark_slab;
73 static struct list_lru nfsd_file_lru;
74 static unsigned long nfsd_file_flags;
75 static struct fsnotify_group *nfsd_file_fsnotify_group;
76 static struct delayed_work nfsd_filecache_laundrette;
77 static struct rhltable nfsd_file_rhltable
78 ____cacheline_aligned_in_smp;
79
80 static bool
nfsd_match_cred(const struct cred * c1,const struct cred * c2)81 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
82 {
83 int i;
84
85 if (!uid_eq(c1->fsuid, c2->fsuid))
86 return false;
87 if (!gid_eq(c1->fsgid, c2->fsgid))
88 return false;
89 if (c1->group_info == NULL || c2->group_info == NULL)
90 return c1->group_info == c2->group_info;
91 if (c1->group_info->ngroups != c2->group_info->ngroups)
92 return false;
93 for (i = 0; i < c1->group_info->ngroups; i++) {
94 if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
95 return false;
96 }
97 return true;
98 }
99
100 static const struct rhashtable_params nfsd_file_rhash_params = {
101 .key_len = sizeof_field(struct nfsd_file, nf_inode),
102 .key_offset = offsetof(struct nfsd_file, nf_inode),
103 .head_offset = offsetof(struct nfsd_file, nf_rlist),
104
105 /*
106 * Start with a single page hash table to reduce resizing churn
107 * on light workloads.
108 */
109 .min_size = 256,
110 .automatic_shrinking = true,
111 };
112
113 static void
nfsd_file_schedule_laundrette(void)114 nfsd_file_schedule_laundrette(void)
115 {
116 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))
117 queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
118 NFSD_LAUNDRETTE_DELAY);
119 }
120
121 static void
nfsd_file_slab_free(struct rcu_head * rcu)122 nfsd_file_slab_free(struct rcu_head *rcu)
123 {
124 struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
125
126 put_cred(nf->nf_cred);
127 kmem_cache_free(nfsd_file_slab, nf);
128 }
129
130 static void
nfsd_file_mark_free(struct fsnotify_mark * mark)131 nfsd_file_mark_free(struct fsnotify_mark *mark)
132 {
133 struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
134 nfm_mark);
135
136 kmem_cache_free(nfsd_file_mark_slab, nfm);
137 }
138
139 static struct nfsd_file_mark *
nfsd_file_mark_get(struct nfsd_file_mark * nfm)140 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
141 {
142 if (!refcount_inc_not_zero(&nfm->nfm_ref))
143 return NULL;
144 return nfm;
145 }
146
147 static void
nfsd_file_mark_put(struct nfsd_file_mark * nfm)148 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
149 {
150 if (refcount_dec_and_test(&nfm->nfm_ref)) {
151 fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
152 fsnotify_put_mark(&nfm->nfm_mark);
153 }
154 }
155
156 static struct nfsd_file_mark *
nfsd_file_mark_find_or_create(struct nfsd_file * nf,struct inode * inode)157 nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
158 {
159 int err;
160 struct fsnotify_mark *mark;
161 struct nfsd_file_mark *nfm = NULL, *new;
162
163 do {
164 fsnotify_group_lock(nfsd_file_fsnotify_group);
165 mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
166 nfsd_file_fsnotify_group);
167 if (mark) {
168 nfm = nfsd_file_mark_get(container_of(mark,
169 struct nfsd_file_mark,
170 nfm_mark));
171 fsnotify_group_unlock(nfsd_file_fsnotify_group);
172 if (nfm) {
173 fsnotify_put_mark(mark);
174 break;
175 }
176 /* Avoid soft lockup race with nfsd_file_mark_put() */
177 fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
178 fsnotify_put_mark(mark);
179 } else {
180 fsnotify_group_unlock(nfsd_file_fsnotify_group);
181 }
182
183 /* allocate a new nfm */
184 new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
185 if (!new)
186 return NULL;
187 fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
188 new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
189 refcount_set(&new->nfm_ref, 1);
190
191 err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
192
193 /*
194 * If the add was successful, then return the object.
195 * Otherwise, we need to put the reference we hold on the
196 * nfm_mark. The fsnotify code will take a reference and put
197 * it on failure, so we can't just free it directly. It's also
198 * not safe to call fsnotify_destroy_mark on it as the
199 * mark->group will be NULL. Thus, we can't let the nfm_ref
200 * counter drive the destruction at this point.
201 */
202 if (likely(!err))
203 nfm = new;
204 else
205 fsnotify_put_mark(&new->nfm_mark);
206 } while (unlikely(err == -EEXIST));
207
208 return nfm;
209 }
210
211 static struct nfsd_file *
nfsd_file_alloc(struct net * net,struct inode * inode,unsigned char need,bool want_gc)212 nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need,
213 bool want_gc)
214 {
215 struct nfsd_file *nf;
216
217 nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
218 if (unlikely(!nf))
219 return NULL;
220
221 INIT_LIST_HEAD(&nf->nf_lru);
222 nf->nf_birthtime = ktime_get();
223 nf->nf_file = NULL;
224 nf->nf_cred = get_current_cred();
225 nf->nf_net = net;
226 nf->nf_flags = want_gc ?
227 BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) :
228 BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING);
229 nf->nf_inode = inode;
230 refcount_set(&nf->nf_ref, 1);
231 nf->nf_may = need;
232 nf->nf_mark = NULL;
233 return nf;
234 }
235
236 /**
237 * nfsd_file_check_write_error - check for writeback errors on a file
238 * @nf: nfsd_file to check for writeback errors
239 *
240 * Check whether a nfsd_file has an unseen error. Reset the write
241 * verifier if so.
242 */
243 static void
nfsd_file_check_write_error(struct nfsd_file * nf)244 nfsd_file_check_write_error(struct nfsd_file *nf)
245 {
246 struct file *file = nf->nf_file;
247
248 if ((file->f_mode & FMODE_WRITE) &&
249 filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
250 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
251 }
252
253 static void
nfsd_file_hash_remove(struct nfsd_file * nf)254 nfsd_file_hash_remove(struct nfsd_file *nf)
255 {
256 trace_nfsd_file_unhash(nf);
257 rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,
258 nfsd_file_rhash_params);
259 }
260
261 static bool
nfsd_file_unhash(struct nfsd_file * nf)262 nfsd_file_unhash(struct nfsd_file *nf)
263 {
264 if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
265 nfsd_file_hash_remove(nf);
266 return true;
267 }
268 return false;
269 }
270
271 static void
nfsd_file_free(struct nfsd_file * nf)272 nfsd_file_free(struct nfsd_file *nf)
273 {
274 s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
275
276 trace_nfsd_file_free(nf);
277
278 this_cpu_inc(nfsd_file_releases);
279 this_cpu_add(nfsd_file_total_age, age);
280
281 nfsd_file_unhash(nf);
282 if (nf->nf_mark)
283 nfsd_file_mark_put(nf->nf_mark);
284 if (nf->nf_file) {
285 nfsd_file_check_write_error(nf);
286 filp_close(nf->nf_file, NULL);
287 }
288
289 /*
290 * If this item is still linked via nf_lru, that's a bug.
291 * WARN and leak it to preserve system stability.
292 */
293 if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
294 return;
295
296 call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
297 }
298
299 static bool
nfsd_file_check_writeback(struct nfsd_file * nf)300 nfsd_file_check_writeback(struct nfsd_file *nf)
301 {
302 struct file *file = nf->nf_file;
303 struct address_space *mapping;
304
305 /* File not open for write? */
306 if (!(file->f_mode & FMODE_WRITE))
307 return false;
308
309 /*
310 * Some filesystems (e.g. NFS) flush all dirty data on close.
311 * On others, there is no need to wait for writeback.
312 */
313 if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE))
314 return false;
315
316 mapping = file->f_mapping;
317 return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
318 mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
319 }
320
321
nfsd_file_lru_add(struct nfsd_file * nf)322 static bool nfsd_file_lru_add(struct nfsd_file *nf)
323 {
324 set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
325 if (list_lru_add(&nfsd_file_lru, &nf->nf_lru)) {
326 trace_nfsd_file_lru_add(nf);
327 return true;
328 }
329 return false;
330 }
331
nfsd_file_lru_remove(struct nfsd_file * nf)332 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
333 {
334 if (list_lru_del(&nfsd_file_lru, &nf->nf_lru)) {
335 trace_nfsd_file_lru_del(nf);
336 return true;
337 }
338 return false;
339 }
340
341 struct nfsd_file *
nfsd_file_get(struct nfsd_file * nf)342 nfsd_file_get(struct nfsd_file *nf)
343 {
344 if (nf && refcount_inc_not_zero(&nf->nf_ref))
345 return nf;
346 return NULL;
347 }
348
349 /**
350 * nfsd_file_put - put the reference to a nfsd_file
351 * @nf: nfsd_file of which to put the reference
352 *
353 * Put a reference to a nfsd_file. In the non-GC case, we just put the
354 * reference immediately. In the GC case, if the reference would be
355 * the last one, the put it on the LRU instead to be cleaned up later.
356 */
357 void
nfsd_file_put(struct nfsd_file * nf)358 nfsd_file_put(struct nfsd_file *nf)
359 {
360 might_sleep();
361 trace_nfsd_file_put(nf);
362
363 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
364 test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
365 /*
366 * If this is the last reference (nf_ref == 1), then try to
367 * transfer it to the LRU.
368 */
369 if (refcount_dec_not_one(&nf->nf_ref))
370 return;
371
372 /* Try to add it to the LRU. If that fails, decrement. */
373 if (nfsd_file_lru_add(nf)) {
374 /* If it's still hashed, we're done */
375 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
376 nfsd_file_schedule_laundrette();
377 return;
378 }
379
380 /*
381 * We're racing with unhashing, so try to remove it from
382 * the LRU. If removal fails, then someone else already
383 * has our reference.
384 */
385 if (!nfsd_file_lru_remove(nf))
386 return;
387 }
388 }
389 if (refcount_dec_and_test(&nf->nf_ref))
390 nfsd_file_free(nf);
391 }
392
393 static void
nfsd_file_dispose_list(struct list_head * dispose)394 nfsd_file_dispose_list(struct list_head *dispose)
395 {
396 struct nfsd_file *nf;
397
398 while (!list_empty(dispose)) {
399 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
400 list_del_init(&nf->nf_lru);
401 nfsd_file_free(nf);
402 }
403 }
404
405 /**
406 * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list
407 * @dispose: list of nfsd_files to be disposed
408 *
409 * Transfers each file to the "freeme" list for its nfsd_net, to eventually
410 * be disposed of by the per-net garbage collector.
411 */
412 static void
nfsd_file_dispose_list_delayed(struct list_head * dispose)413 nfsd_file_dispose_list_delayed(struct list_head *dispose)
414 {
415 while(!list_empty(dispose)) {
416 struct nfsd_file *nf = list_first_entry(dispose,
417 struct nfsd_file, nf_lru);
418 struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
419 struct nfsd_fcache_disposal *l = nn->fcache_disposal;
420
421 spin_lock(&l->lock);
422 list_move_tail(&nf->nf_lru, &l->freeme);
423 spin_unlock(&l->lock);
424 queue_work(nfsd_filecache_wq, &l->work);
425 }
426 }
427
428 /**
429 * nfsd_file_lru_cb - Examine an entry on the LRU list
430 * @item: LRU entry to examine
431 * @lru: controlling LRU
432 * @lock: LRU list lock (unused)
433 * @arg: dispose list
434 *
435 * Return values:
436 * %LRU_REMOVED: @item was removed from the LRU
437 * %LRU_ROTATE: @item is to be moved to the LRU tail
438 * %LRU_SKIP: @item cannot be evicted
439 */
440 static enum lru_status
nfsd_file_lru_cb(struct list_head * item,struct list_lru_one * lru,spinlock_t * lock,void * arg)441 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
442 spinlock_t *lock, void *arg)
443 __releases(lock)
444 __acquires(lock)
445 {
446 struct list_head *head = arg;
447 struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
448
449 /* We should only be dealing with GC entries here */
450 WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
451
452 /*
453 * Don't throw out files that are still undergoing I/O or
454 * that have uncleared errors pending.
455 */
456 if (nfsd_file_check_writeback(nf)) {
457 trace_nfsd_file_gc_writeback(nf);
458 return LRU_SKIP;
459 }
460
461 /* If it was recently added to the list, skip it */
462 if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
463 trace_nfsd_file_gc_referenced(nf);
464 return LRU_ROTATE;
465 }
466
467 /*
468 * Put the reference held on behalf of the LRU. If it wasn't the last
469 * one, then just remove it from the LRU and ignore it.
470 */
471 if (!refcount_dec_and_test(&nf->nf_ref)) {
472 trace_nfsd_file_gc_in_use(nf);
473 list_lru_isolate(lru, &nf->nf_lru);
474 return LRU_REMOVED;
475 }
476
477 /* Refcount went to zero. Unhash it and queue it to the dispose list */
478 nfsd_file_unhash(nf);
479 list_lru_isolate_move(lru, &nf->nf_lru, head);
480 this_cpu_inc(nfsd_file_evictions);
481 trace_nfsd_file_gc_disposed(nf);
482 return LRU_REMOVED;
483 }
484
485 static void
nfsd_file_gc(void)486 nfsd_file_gc(void)
487 {
488 LIST_HEAD(dispose);
489 unsigned long ret;
490
491 ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
492 &dispose, list_lru_count(&nfsd_file_lru));
493 trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
494 nfsd_file_dispose_list_delayed(&dispose);
495 }
496
497 static void
nfsd_file_gc_worker(struct work_struct * work)498 nfsd_file_gc_worker(struct work_struct *work)
499 {
500 nfsd_file_gc();
501 if (list_lru_count(&nfsd_file_lru))
502 nfsd_file_schedule_laundrette();
503 }
504
505 static unsigned long
nfsd_file_lru_count(struct shrinker * s,struct shrink_control * sc)506 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
507 {
508 return list_lru_count(&nfsd_file_lru);
509 }
510
511 static unsigned long
nfsd_file_lru_scan(struct shrinker * s,struct shrink_control * sc)512 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
513 {
514 LIST_HEAD(dispose);
515 unsigned long ret;
516
517 ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
518 nfsd_file_lru_cb, &dispose);
519 trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
520 nfsd_file_dispose_list_delayed(&dispose);
521 return ret;
522 }
523
524 static struct shrinker nfsd_file_shrinker = {
525 .scan_objects = nfsd_file_lru_scan,
526 .count_objects = nfsd_file_lru_count,
527 .seeks = 1,
528 };
529
530 /**
531 * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
532 * @nf: nfsd_file to attempt to queue
533 * @dispose: private list to queue successfully-put objects
534 *
535 * Unhash an nfsd_file, try to get a reference to it, and then put that
536 * reference. If it's the last reference, queue it to the dispose list.
537 */
538 static void
nfsd_file_cond_queue(struct nfsd_file * nf,struct list_head * dispose)539 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
540 __must_hold(RCU)
541 {
542 int decrement = 1;
543
544 /* If we raced with someone else unhashing, ignore it */
545 if (!nfsd_file_unhash(nf))
546 return;
547
548 /* If we can't get a reference, ignore it */
549 if (!nfsd_file_get(nf))
550 return;
551
552 /* Extra decrement if we remove from the LRU */
553 if (nfsd_file_lru_remove(nf))
554 ++decrement;
555
556 /* If refcount goes to 0, then put on the dispose list */
557 if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
558 list_add(&nf->nf_lru, dispose);
559 trace_nfsd_file_closing(nf);
560 }
561 }
562
563 /**
564 * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
565 * @inode: inode on which to close out nfsd_files
566 * @dispose: list on which to gather nfsd_files to close out
567 *
568 * An nfsd_file represents a struct file being held open on behalf of nfsd.
569 * An open file however can block other activity (such as leases), or cause
570 * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
571 *
572 * This function is intended to find open nfsd_files when this sort of
573 * conflicting access occurs and then attempt to close those files out.
574 *
575 * Populates the dispose list with entries that have already had their
576 * refcounts go to zero. The actual free of an nfsd_file can be expensive,
577 * so we leave it up to the caller whether it wants to wait or not.
578 */
579 static void
nfsd_file_queue_for_close(struct inode * inode,struct list_head * dispose)580 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
581 {
582 struct rhlist_head *tmp, *list;
583 struct nfsd_file *nf;
584
585 rcu_read_lock();
586 list = rhltable_lookup(&nfsd_file_rhltable, &inode,
587 nfsd_file_rhash_params);
588 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
589 if (!test_bit(NFSD_FILE_GC, &nf->nf_flags))
590 continue;
591 nfsd_file_cond_queue(nf, dispose);
592 }
593 rcu_read_unlock();
594 }
595
596 /**
597 * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
598 * @inode: inode of the file to attempt to remove
599 *
600 * Close out any open nfsd_files that can be reaped for @inode. The
601 * actual freeing is deferred to the dispose_list_delayed infrastructure.
602 *
603 * This is used by the fsnotify callbacks and setlease notifier.
604 */
605 static void
nfsd_file_close_inode(struct inode * inode)606 nfsd_file_close_inode(struct inode *inode)
607 {
608 LIST_HEAD(dispose);
609
610 nfsd_file_queue_for_close(inode, &dispose);
611 nfsd_file_dispose_list_delayed(&dispose);
612 }
613
614 /**
615 * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
616 * @inode: inode of the file to attempt to remove
617 *
618 * Close out any open nfsd_files that can be reaped for @inode. The
619 * nfsd_files are closed out synchronously.
620 *
621 * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
622 * when reexporting NFS.
623 */
624 void
nfsd_file_close_inode_sync(struct inode * inode)625 nfsd_file_close_inode_sync(struct inode *inode)
626 {
627 struct nfsd_file *nf;
628 LIST_HEAD(dispose);
629
630 trace_nfsd_file_close(inode);
631
632 nfsd_file_queue_for_close(inode, &dispose);
633 while (!list_empty(&dispose)) {
634 nf = list_first_entry(&dispose, struct nfsd_file, nf_lru);
635 list_del_init(&nf->nf_lru);
636 nfsd_file_free(nf);
637 }
638 flush_delayed_fput();
639 }
640
641 /**
642 * nfsd_file_delayed_close - close unused nfsd_files
643 * @work: dummy
644 *
645 * Scrape the freeme list for this nfsd_net, and then dispose of them
646 * all.
647 */
648 static void
nfsd_file_delayed_close(struct work_struct * work)649 nfsd_file_delayed_close(struct work_struct *work)
650 {
651 LIST_HEAD(head);
652 struct nfsd_fcache_disposal *l = container_of(work,
653 struct nfsd_fcache_disposal, work);
654
655 spin_lock(&l->lock);
656 list_splice_init(&l->freeme, &head);
657 spin_unlock(&l->lock);
658
659 nfsd_file_dispose_list(&head);
660 }
661
662 static int
nfsd_file_lease_notifier_call(struct notifier_block * nb,unsigned long arg,void * data)663 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
664 void *data)
665 {
666 struct file_lock *fl = data;
667
668 /* Only close files for F_SETLEASE leases */
669 if (fl->fl_flags & FL_LEASE)
670 nfsd_file_close_inode(file_inode(fl->fl_file));
671 return 0;
672 }
673
674 static struct notifier_block nfsd_file_lease_notifier = {
675 .notifier_call = nfsd_file_lease_notifier_call,
676 };
677
678 static int
nfsd_file_fsnotify_handle_event(struct fsnotify_mark * mark,u32 mask,struct inode * inode,struct inode * dir,const struct qstr * name,u32 cookie)679 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
680 struct inode *inode, struct inode *dir,
681 const struct qstr *name, u32 cookie)
682 {
683 if (WARN_ON_ONCE(!inode))
684 return 0;
685
686 trace_nfsd_file_fsnotify_handle_event(inode, mask);
687
688 /* Should be no marks on non-regular files */
689 if (!S_ISREG(inode->i_mode)) {
690 WARN_ON_ONCE(1);
691 return 0;
692 }
693
694 /* don't close files if this was not the last link */
695 if (mask & FS_ATTRIB) {
696 if (inode->i_nlink)
697 return 0;
698 }
699
700 nfsd_file_close_inode(inode);
701 return 0;
702 }
703
704
705 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
706 .handle_inode_event = nfsd_file_fsnotify_handle_event,
707 .free_mark = nfsd_file_mark_free,
708 };
709
710 int
nfsd_file_cache_init(void)711 nfsd_file_cache_init(void)
712 {
713 int ret;
714
715 lockdep_assert_held(&nfsd_mutex);
716 if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
717 return 0;
718
719 ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params);
720 if (ret)
721 return ret;
722
723 ret = -ENOMEM;
724 nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
725 if (!nfsd_filecache_wq)
726 goto out;
727
728 nfsd_file_slab = kmem_cache_create("nfsd_file",
729 sizeof(struct nfsd_file), 0, 0, NULL);
730 if (!nfsd_file_slab) {
731 pr_err("nfsd: unable to create nfsd_file_slab\n");
732 goto out_err;
733 }
734
735 nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
736 sizeof(struct nfsd_file_mark), 0, 0, NULL);
737 if (!nfsd_file_mark_slab) {
738 pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
739 goto out_err;
740 }
741
742
743 ret = list_lru_init(&nfsd_file_lru);
744 if (ret) {
745 pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
746 goto out_err;
747 }
748
749 ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
750 if (ret) {
751 pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
752 goto out_lru;
753 }
754
755 ret = lease_register_notifier(&nfsd_file_lease_notifier);
756 if (ret) {
757 pr_err("nfsd: unable to register lease notifier: %d\n", ret);
758 goto out_shrinker;
759 }
760
761 nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
762 FSNOTIFY_GROUP_NOFS);
763 if (IS_ERR(nfsd_file_fsnotify_group)) {
764 pr_err("nfsd: unable to create fsnotify group: %ld\n",
765 PTR_ERR(nfsd_file_fsnotify_group));
766 ret = PTR_ERR(nfsd_file_fsnotify_group);
767 nfsd_file_fsnotify_group = NULL;
768 goto out_notifier;
769 }
770
771 INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
772 out:
773 return ret;
774 out_notifier:
775 lease_unregister_notifier(&nfsd_file_lease_notifier);
776 out_shrinker:
777 unregister_shrinker(&nfsd_file_shrinker);
778 out_lru:
779 list_lru_destroy(&nfsd_file_lru);
780 out_err:
781 kmem_cache_destroy(nfsd_file_slab);
782 nfsd_file_slab = NULL;
783 kmem_cache_destroy(nfsd_file_mark_slab);
784 nfsd_file_mark_slab = NULL;
785 destroy_workqueue(nfsd_filecache_wq);
786 nfsd_filecache_wq = NULL;
787 rhltable_destroy(&nfsd_file_rhltable);
788 goto out;
789 }
790
791 /**
792 * __nfsd_file_cache_purge: clean out the cache for shutdown
793 * @net: net-namespace to shut down the cache (may be NULL)
794 *
795 * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
796 * then close out everything. Called when an nfsd instance is being shut down,
797 * and when the exports table is flushed.
798 */
799 static void
__nfsd_file_cache_purge(struct net * net)800 __nfsd_file_cache_purge(struct net *net)
801 {
802 struct rhashtable_iter iter;
803 struct nfsd_file *nf;
804 LIST_HEAD(dispose);
805
806 rhltable_walk_enter(&nfsd_file_rhltable, &iter);
807 do {
808 rhashtable_walk_start(&iter);
809
810 nf = rhashtable_walk_next(&iter);
811 while (!IS_ERR_OR_NULL(nf)) {
812 if (!net || nf->nf_net == net)
813 nfsd_file_cond_queue(nf, &dispose);
814 nf = rhashtable_walk_next(&iter);
815 }
816
817 rhashtable_walk_stop(&iter);
818 } while (nf == ERR_PTR(-EAGAIN));
819 rhashtable_walk_exit(&iter);
820
821 nfsd_file_dispose_list(&dispose);
822 }
823
824 static struct nfsd_fcache_disposal *
nfsd_alloc_fcache_disposal(void)825 nfsd_alloc_fcache_disposal(void)
826 {
827 struct nfsd_fcache_disposal *l;
828
829 l = kmalloc(sizeof(*l), GFP_KERNEL);
830 if (!l)
831 return NULL;
832 INIT_WORK(&l->work, nfsd_file_delayed_close);
833 spin_lock_init(&l->lock);
834 INIT_LIST_HEAD(&l->freeme);
835 return l;
836 }
837
838 static void
nfsd_free_fcache_disposal(struct nfsd_fcache_disposal * l)839 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
840 {
841 cancel_work_sync(&l->work);
842 nfsd_file_dispose_list(&l->freeme);
843 kfree(l);
844 }
845
846 static void
nfsd_free_fcache_disposal_net(struct net * net)847 nfsd_free_fcache_disposal_net(struct net *net)
848 {
849 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
850 struct nfsd_fcache_disposal *l = nn->fcache_disposal;
851
852 nfsd_free_fcache_disposal(l);
853 }
854
855 int
nfsd_file_cache_start_net(struct net * net)856 nfsd_file_cache_start_net(struct net *net)
857 {
858 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
859
860 nn->fcache_disposal = nfsd_alloc_fcache_disposal();
861 return nn->fcache_disposal ? 0 : -ENOMEM;
862 }
863
864 /**
865 * nfsd_file_cache_purge - Remove all cache items associated with @net
866 * @net: target net namespace
867 *
868 */
869 void
nfsd_file_cache_purge(struct net * net)870 nfsd_file_cache_purge(struct net *net)
871 {
872 lockdep_assert_held(&nfsd_mutex);
873 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
874 __nfsd_file_cache_purge(net);
875 }
876
877 void
nfsd_file_cache_shutdown_net(struct net * net)878 nfsd_file_cache_shutdown_net(struct net *net)
879 {
880 nfsd_file_cache_purge(net);
881 nfsd_free_fcache_disposal_net(net);
882 }
883
884 void
nfsd_file_cache_shutdown(void)885 nfsd_file_cache_shutdown(void)
886 {
887 int i;
888
889 lockdep_assert_held(&nfsd_mutex);
890 if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
891 return;
892
893 lease_unregister_notifier(&nfsd_file_lease_notifier);
894 unregister_shrinker(&nfsd_file_shrinker);
895 /*
896 * make sure all callers of nfsd_file_lru_cb are done before
897 * calling nfsd_file_cache_purge
898 */
899 cancel_delayed_work_sync(&nfsd_filecache_laundrette);
900 __nfsd_file_cache_purge(NULL);
901 list_lru_destroy(&nfsd_file_lru);
902 rcu_barrier();
903 fsnotify_put_group(nfsd_file_fsnotify_group);
904 nfsd_file_fsnotify_group = NULL;
905 kmem_cache_destroy(nfsd_file_slab);
906 nfsd_file_slab = NULL;
907 fsnotify_wait_marks_destroyed();
908 kmem_cache_destroy(nfsd_file_mark_slab);
909 nfsd_file_mark_slab = NULL;
910 destroy_workqueue(nfsd_filecache_wq);
911 nfsd_filecache_wq = NULL;
912 rhltable_destroy(&nfsd_file_rhltable);
913
914 for_each_possible_cpu(i) {
915 per_cpu(nfsd_file_cache_hits, i) = 0;
916 per_cpu(nfsd_file_acquisitions, i) = 0;
917 per_cpu(nfsd_file_releases, i) = 0;
918 per_cpu(nfsd_file_total_age, i) = 0;
919 per_cpu(nfsd_file_evictions, i) = 0;
920 }
921 }
922
923 static struct nfsd_file *
nfsd_file_lookup_locked(const struct net * net,const struct cred * cred,struct inode * inode,unsigned char need,bool want_gc)924 nfsd_file_lookup_locked(const struct net *net, const struct cred *cred,
925 struct inode *inode, unsigned char need,
926 bool want_gc)
927 {
928 struct rhlist_head *tmp, *list;
929 struct nfsd_file *nf;
930
931 list = rhltable_lookup(&nfsd_file_rhltable, &inode,
932 nfsd_file_rhash_params);
933 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
934 if (nf->nf_may != need)
935 continue;
936 if (nf->nf_net != net)
937 continue;
938 if (!nfsd_match_cred(nf->nf_cred, cred))
939 continue;
940 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc)
941 continue;
942 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
943 continue;
944
945 if (!nfsd_file_get(nf))
946 continue;
947 return nf;
948 }
949 return NULL;
950 }
951
952 /**
953 * nfsd_file_is_cached - are there any cached open files for this inode?
954 * @inode: inode to check
955 *
956 * The lookup matches inodes in all net namespaces and is atomic wrt
957 * nfsd_file_acquire().
958 *
959 * Return values:
960 * %true: filecache contains at least one file matching this inode
961 * %false: filecache contains no files matching this inode
962 */
963 bool
nfsd_file_is_cached(struct inode * inode)964 nfsd_file_is_cached(struct inode *inode)
965 {
966 struct rhlist_head *tmp, *list;
967 struct nfsd_file *nf;
968 bool ret = false;
969
970 rcu_read_lock();
971 list = rhltable_lookup(&nfsd_file_rhltable, &inode,
972 nfsd_file_rhash_params);
973 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist)
974 if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) {
975 ret = true;
976 break;
977 }
978 rcu_read_unlock();
979
980 trace_nfsd_file_is_cached(inode, (int)ret);
981 return ret;
982 }
983
984 static __be32
nfsd_file_do_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf,bool want_gc)985 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
986 unsigned int may_flags, struct file *file,
987 struct nfsd_file **pnf, bool want_gc)
988 {
989 unsigned char need = may_flags & NFSD_FILE_MAY_MASK;
990 struct net *net = SVC_NET(rqstp);
991 struct nfsd_file *new, *nf;
992 bool stale_retry = true;
993 bool open_retry = true;
994 struct inode *inode;
995 __be32 status;
996 int ret;
997
998 retry:
999 status = fh_verify(rqstp, fhp, S_IFREG,
1000 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1001 if (status != nfs_ok)
1002 return status;
1003 inode = d_inode(fhp->fh_dentry);
1004
1005 rcu_read_lock();
1006 nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1007 rcu_read_unlock();
1008
1009 if (nf) {
1010 /*
1011 * If the nf is on the LRU then it holds an extra reference
1012 * that must be put if it's removed. It had better not be
1013 * the last one however, since we should hold another.
1014 */
1015 if (nfsd_file_lru_remove(nf))
1016 WARN_ON_ONCE(refcount_dec_and_test(&nf->nf_ref));
1017 goto wait_for_construction;
1018 }
1019
1020 new = nfsd_file_alloc(net, inode, need, want_gc);
1021 if (!new) {
1022 status = nfserr_jukebox;
1023 goto out;
1024 }
1025
1026 rcu_read_lock();
1027 spin_lock(&inode->i_lock);
1028 nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1029 if (unlikely(nf)) {
1030 spin_unlock(&inode->i_lock);
1031 rcu_read_unlock();
1032 nfsd_file_slab_free(&new->nf_rcu);
1033 goto wait_for_construction;
1034 }
1035 nf = new;
1036 ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist,
1037 nfsd_file_rhash_params);
1038 spin_unlock(&inode->i_lock);
1039 rcu_read_unlock();
1040 if (likely(ret == 0))
1041 goto open_file;
1042
1043 if (ret == -EEXIST)
1044 goto retry;
1045 trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret);
1046 status = nfserr_jukebox;
1047 goto construction_err;
1048
1049 wait_for_construction:
1050 wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1051
1052 /* Did construction of this file fail? */
1053 if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1054 trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf);
1055 if (!open_retry) {
1056 status = nfserr_jukebox;
1057 goto construction_err;
1058 }
1059 open_retry = false;
1060 fh_put(fhp);
1061 goto retry;
1062 }
1063 this_cpu_inc(nfsd_file_cache_hits);
1064
1065 status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1066 if (status != nfs_ok) {
1067 nfsd_file_put(nf);
1068 nf = NULL;
1069 }
1070
1071 out:
1072 if (status == nfs_ok) {
1073 this_cpu_inc(nfsd_file_acquisitions);
1074 nfsd_file_check_write_error(nf);
1075 *pnf = nf;
1076 }
1077 trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status);
1078 return status;
1079
1080 open_file:
1081 trace_nfsd_file_alloc(nf);
1082 nf->nf_mark = nfsd_file_mark_find_or_create(nf, inode);
1083 if (nf->nf_mark) {
1084 if (file) {
1085 get_file(file);
1086 nf->nf_file = file;
1087 status = nfs_ok;
1088 trace_nfsd_file_opened(nf, status);
1089 } else {
1090 ret = nfsd_open_verified(rqstp, fhp, may_flags,
1091 &nf->nf_file);
1092 if (ret == -EOPENSTALE && stale_retry) {
1093 stale_retry = false;
1094 nfsd_file_unhash(nf);
1095 clear_and_wake_up_bit(NFSD_FILE_PENDING,
1096 &nf->nf_flags);
1097 if (refcount_dec_and_test(&nf->nf_ref))
1098 nfsd_file_free(nf);
1099 nf = NULL;
1100 fh_put(fhp);
1101 goto retry;
1102 }
1103 status = nfserrno(ret);
1104 trace_nfsd_file_open(nf, status);
1105 }
1106 } else
1107 status = nfserr_jukebox;
1108 /*
1109 * If construction failed, or we raced with a call to unlink()
1110 * then unhash.
1111 */
1112 if (status != nfs_ok || inode->i_nlink == 0)
1113 nfsd_file_unhash(nf);
1114 clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags);
1115 if (status == nfs_ok)
1116 goto out;
1117
1118 construction_err:
1119 if (refcount_dec_and_test(&nf->nf_ref))
1120 nfsd_file_free(nf);
1121 nf = NULL;
1122 goto out;
1123 }
1124
1125 /**
1126 * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1127 * @rqstp: the RPC transaction being executed
1128 * @fhp: the NFS filehandle of the file to be opened
1129 * @may_flags: NFSD_MAY_ settings for the file
1130 * @pnf: OUT: new or found "struct nfsd_file" object
1131 *
1132 * The nfsd_file object returned by this API is reference-counted
1133 * and garbage-collected. The object is retained for a few
1134 * seconds after the final nfsd_file_put() in case the caller
1135 * wants to re-use it.
1136 *
1137 * Return values:
1138 * %nfs_ok - @pnf points to an nfsd_file with its reference
1139 * count boosted.
1140 *
1141 * On error, an nfsstat value in network byte order is returned.
1142 */
1143 __be32
nfsd_file_acquire_gc(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1144 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1145 unsigned int may_flags, struct nfsd_file **pnf)
1146 {
1147 return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, true);
1148 }
1149
1150 /**
1151 * nfsd_file_acquire - Get a struct nfsd_file with an open file
1152 * @rqstp: the RPC transaction being executed
1153 * @fhp: the NFS filehandle of the file to be opened
1154 * @may_flags: NFSD_MAY_ settings for the file
1155 * @pnf: OUT: new or found "struct nfsd_file" object
1156 *
1157 * The nfsd_file_object returned by this API is reference-counted
1158 * but not garbage-collected. The object is unhashed after the
1159 * final nfsd_file_put().
1160 *
1161 * Return values:
1162 * %nfs_ok - @pnf points to an nfsd_file with its reference
1163 * count boosted.
1164 *
1165 * On error, an nfsstat value in network byte order is returned.
1166 */
1167 __be32
nfsd_file_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1168 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1169 unsigned int may_flags, struct nfsd_file **pnf)
1170 {
1171 return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, false);
1172 }
1173
1174 /**
1175 * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1176 * @rqstp: the RPC transaction being executed
1177 * @fhp: the NFS filehandle of the file just created
1178 * @may_flags: NFSD_MAY_ settings for the file
1179 * @file: cached, already-open file (may be NULL)
1180 * @pnf: OUT: new or found "struct nfsd_file" object
1181 *
1182 * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1183 * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1184 * opening a new one.
1185 *
1186 * Return values:
1187 * %nfs_ok - @pnf points to an nfsd_file with its reference
1188 * count boosted.
1189 *
1190 * On error, an nfsstat value in network byte order is returned.
1191 */
1192 __be32
nfsd_file_acquire_opened(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf)1193 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1194 unsigned int may_flags, struct file *file,
1195 struct nfsd_file **pnf)
1196 {
1197 return nfsd_file_do_acquire(rqstp, fhp, may_flags, file, pnf, false);
1198 }
1199
1200 /*
1201 * Note that fields may be added, removed or reordered in the future. Programs
1202 * scraping this file for info should test the labels to ensure they're
1203 * getting the correct field.
1204 */
nfsd_file_cache_stats_show(struct seq_file * m,void * v)1205 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1206 {
1207 unsigned long releases = 0, evictions = 0;
1208 unsigned long hits = 0, acquisitions = 0;
1209 unsigned int i, count = 0, buckets = 0;
1210 unsigned long lru = 0, total_age = 0;
1211
1212 /* Serialize with server shutdown */
1213 mutex_lock(&nfsd_mutex);
1214 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1215 struct bucket_table *tbl;
1216 struct rhashtable *ht;
1217
1218 lru = list_lru_count(&nfsd_file_lru);
1219
1220 rcu_read_lock();
1221 ht = &nfsd_file_rhltable.ht;
1222 count = atomic_read(&ht->nelems);
1223 tbl = rht_dereference_rcu(ht->tbl, ht);
1224 buckets = tbl->size;
1225 rcu_read_unlock();
1226 }
1227 mutex_unlock(&nfsd_mutex);
1228
1229 for_each_possible_cpu(i) {
1230 hits += per_cpu(nfsd_file_cache_hits, i);
1231 acquisitions += per_cpu(nfsd_file_acquisitions, i);
1232 releases += per_cpu(nfsd_file_releases, i);
1233 total_age += per_cpu(nfsd_file_total_age, i);
1234 evictions += per_cpu(nfsd_file_evictions, i);
1235 }
1236
1237 seq_printf(m, "total inodes: %u\n", count);
1238 seq_printf(m, "hash buckets: %u\n", buckets);
1239 seq_printf(m, "lru entries: %lu\n", lru);
1240 seq_printf(m, "cache hits: %lu\n", hits);
1241 seq_printf(m, "acquisitions: %lu\n", acquisitions);
1242 seq_printf(m, "releases: %lu\n", releases);
1243 seq_printf(m, "evictions: %lu\n", evictions);
1244 if (releases)
1245 seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1246 else
1247 seq_printf(m, "mean age (ms): -\n");
1248 return 0;
1249 }
1250