89 			struct nfsd_net *nn)  in nfsd_reply_cache_alloc()  argument
115 				struct nfsd_net *nn)  in nfsd_reply_cache_free_locked()  argument
118 		nfsd_stats_drc_mem_usage_sub(nn, rp->c_replvec.iov_len);  in nfsd_reply_cache_free_locked()
124 		atomic_dec(&nn->num_drc_entries);  in nfsd_reply_cache_free_locked()
125 		nfsd_stats_drc_mem_usage_sub(nn, sizeof(*rp));  in nfsd_reply_cache_free_locked()
132 			struct nfsd_net *nn)  in nfsd_reply_cache_free()  argument
135 	nfsd_reply_cache_free_locked(b, rp, nn);  in nfsd_reply_cache_free()
151 static int nfsd_reply_cache_stats_init(struct nfsd_net *nn)  in nfsd_reply_cache_stats_init()  argument
153 	return nfsd_percpu_counters_init(nn->counter, NFSD_NET_COUNTERS_NUM);  in nfsd_reply_cache_stats_init()
156 static void nfsd_reply_cache_stats_destroy(struct nfsd_net *nn)  in nfsd_reply_cache_stats_destroy()  argument
158 	nfsd_percpu_counters_destroy(nn->counter, NFSD_NET_COUNTERS_NUM);  in nfsd_reply_cache_stats_destroy()
161 int nfsd_reply_cache_init(struct nfsd_net *nn)  in nfsd_reply_cache_init()  argument
167 	nn->max_drc_entries = nfsd_cache_size_limit();  in nfsd_reply_cache_init()
168 	atomic_set(&nn->num_drc_entries, 0);  in nfsd_reply_cache_init()
169 	hashsize = nfsd_hashsize(nn->max_drc_entries);  in nfsd_reply_cache_init()
170 	nn->maskbits = ilog2(hashsize);  in nfsd_reply_cache_init()
172 	status = nfsd_reply_cache_stats_init(nn);  in nfsd_reply_cache_init()
176 	nn->nfsd_reply_cache_shrinker.scan_objects = nfsd_reply_cache_scan;  in nfsd_reply_cache_init()
177 	nn->nfsd_reply_cache_shrinker.count_objects = nfsd_reply_cache_count;  in nfsd_reply_cache_init()
178 	nn->nfsd_reply_cache_shrinker.seeks = 1;  in nfsd_reply_cache_init()
179 	status = register_shrinker(&nn->nfsd_reply_cache_shrinker,  in nfsd_reply_cache_init()
180 				   "nfsd-reply:%s", nn->nfsd_name);  in nfsd_reply_cache_init()
184 	nn->drc_hashtbl = kvzalloc(array_size(hashsize,  in nfsd_reply_cache_init()
185 				sizeof(*nn->drc_hashtbl)), GFP_KERNEL);  in nfsd_reply_cache_init()
186 	if (!nn->drc_hashtbl)  in nfsd_reply_cache_init()
190 		INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head);  in nfsd_reply_cache_init()
191 		spin_lock_init(&nn->drc_hashtbl[i].cache_lock);  in nfsd_reply_cache_init()
193 	nn->drc_hashsize = hashsize;  in nfsd_reply_cache_init()
197 	unregister_shrinker(&nn->nfsd_reply_cache_shrinker);  in nfsd_reply_cache_init()
199 	nfsd_reply_cache_stats_destroy(nn);  in nfsd_reply_cache_init()
205 void nfsd_reply_cache_shutdown(struct nfsd_net *nn)  in nfsd_reply_cache_shutdown()  argument
210 	unregister_shrinker(&nn->nfsd_reply_cache_shrinker);  in nfsd_reply_cache_shutdown()
212 	for (i = 0; i < nn->drc_hashsize; i++) {  in nfsd_reply_cache_shutdown()
213 		struct list_head *head = &nn->drc_hashtbl[i].lru_head;  in nfsd_reply_cache_shutdown()
216 			nfsd_reply_cache_free_locked(&nn->drc_hashtbl[i],  in nfsd_reply_cache_shutdown()
217 									rp, nn);  in nfsd_reply_cache_shutdown()
220 	nfsd_reply_cache_stats_destroy(nn);  in nfsd_reply_cache_shutdown()
222 	kvfree(nn->drc_hashtbl);  in nfsd_reply_cache_shutdown()
223 	nn->drc_hashtbl = NULL;  in nfsd_reply_cache_shutdown()
224 	nn->drc_hashsize = 0;  in nfsd_reply_cache_shutdown()
240 nfsd_cache_bucket_find(__be32 xid, struct nfsd_net *nn)  in nfsd_cache_bucket_find()  argument
242 	unsigned int hash = hash_32((__force u32)xid, nn->maskbits);  in nfsd_cache_bucket_find()
244 	return &nn->drc_hashtbl[hash];  in nfsd_cache_bucket_find()
247 static long prune_bucket(struct nfsd_drc_bucket *b, struct nfsd_net *nn,  in prune_bucket()  argument
260 		if (atomic_read(&nn->num_drc_entries) <= nn->max_drc_entries &&  in prune_bucket()
263 		nfsd_reply_cache_free_locked(b, rp, nn);  in prune_bucket()
270 static long nfsd_prune_bucket(struct nfsd_drc_bucket *b, struct nfsd_net *nn)  in nfsd_prune_bucket()  argument
272 	return prune_bucket(b, nn, 3);  in nfsd_prune_bucket()
280 prune_cache_entries(struct nfsd_net *nn)  in prune_cache_entries()  argument
285 	for (i = 0; i < nn->drc_hashsize; i++) {  in prune_cache_entries()
286 		struct nfsd_drc_bucket *b = &nn->drc_hashtbl[i];  in prune_cache_entries()
291 		freed += prune_bucket(b, nn, 0);  in prune_cache_entries()
300 	struct nfsd_net *nn = container_of(shrink,  in nfsd_reply_cache_count()  local
303 	return atomic_read(&nn->num_drc_entries);  in nfsd_reply_cache_count()
309 	struct nfsd_net *nn = container_of(shrink,  in nfsd_reply_cache_scan()  local
312 	return prune_cache_entries(nn);  in nfsd_reply_cache_scan()
349 			const struct svc_cacherep *rp, struct nfsd_net *nn)  in nfsd_cache_key_cmp()  argument
353 		nfsd_stats_payload_misses_inc(nn);  in nfsd_cache_key_cmp()
354 		trace_nfsd_drc_mismatch(nn, key, rp);  in nfsd_cache_key_cmp()
367 			struct nfsd_net *nn)  in nfsd_cache_insert()  argument
380 		cmp = nfsd_cache_key_cmp(key, rp, nn);  in nfsd_cache_insert()
394 	if (entries > nn->longest_chain) {  in nfsd_cache_insert()
395 		nn->longest_chain = entries;  in nfsd_cache_insert()
396 		nn->longest_chain_cachesize = atomic_read(&nn->num_drc_entries);  in nfsd_cache_insert()
397 	} else if (entries == nn->longest_chain) {  in nfsd_cache_insert()
399 		nn->longest_chain_cachesize = min_t(unsigned int,  in nfsd_cache_insert()
400 				nn->longest_chain_cachesize,  in nfsd_cache_insert()
401 				atomic_read(&nn->num_drc_entries));  in nfsd_cache_insert()
425 	struct nfsd_net		*nn;  in nfsd_cache_lookup()  local
444 	nn = net_generic(SVC_NET(rqstp), nfsd_net_id);  in nfsd_cache_lookup()
445 	rp = nfsd_reply_cache_alloc(rqstp, csum, nn);  in nfsd_cache_lookup()
449 	b = nfsd_cache_bucket_find(rqstp->rq_xid, nn);  in nfsd_cache_lookup()
451 	found = nfsd_cache_insert(b, rp, nn);  in nfsd_cache_lookup()
459 	atomic_inc(&nn->num_drc_entries);  in nfsd_cache_lookup()
460 	nfsd_stats_drc_mem_usage_add(nn, sizeof(*rp));  in nfsd_cache_lookup()
462 	nfsd_prune_bucket(b, nn);  in nfsd_cache_lookup()
471 	nfsd_reply_cache_free_locked(NULL, rp, nn);  in nfsd_cache_lookup()
504 	trace_nfsd_drc_found(nn, rqstp, rtn);  in nfsd_cache_lookup()
530 	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);  in nfsd_cache_update()  local
540 	b = nfsd_cache_bucket_find(rp->c_key.k_xid, nn);  in nfsd_cache_update()
547 		nfsd_reply_cache_free(b, rp, nn);  in nfsd_cache_update()
562 			nfsd_reply_cache_free(b, rp, nn);  in nfsd_cache_update()
569 		nfsd_reply_cache_free(b, rp, nn);  in nfsd_cache_update()
573 	nfsd_stats_drc_mem_usage_add(nn, bufsize);  in nfsd_cache_update()
609 	struct nfsd_net *nn = net_generic(file_inode(m->file)->i_sb->s_fs_info,  in nfsd_reply_cache_stats_show()  local
612 	seq_printf(m, "max entries:           %u\n", nn->max_drc_entries);  in nfsd_reply_cache_stats_show()
614 		   atomic_read(&nn->num_drc_entries));  in nfsd_reply_cache_stats_show()
615 	seq_printf(m, "hash buckets:          %u\n", 1 << nn->maskbits);  in nfsd_reply_cache_stats_show()
617 		   percpu_counter_sum_positive(&nn->counter[NFSD_NET_DRC_MEM_USAGE]));  in nfsd_reply_cache_stats_show()
625 		   percpu_counter_sum_positive(&nn->counter[NFSD_NET_PAYLOAD_MISSES]));  in nfsd_reply_cache_stats_show()
626 	seq_printf(m, "longest chain len:     %u\n", nn->longest_chain);  in nfsd_reply_cache_stats_show()
627 	seq_printf(m, "cachesize at longest:  %u\n", nn->longest_chain_cachesize);  in nfsd_reply_cache_stats_show()