3264 	unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
3268 		READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]),	\
3269 		READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]),	\
3317 	return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;  in get_nr_gens()
3707 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in read_ctrl_pos()  local
3708 	int hist = lru_hist_from_seq(lrugen->min_seq[type]);  in read_ctrl_pos()
3710 	pos->refaulted = lrugen->avg_refaulted[type][tier] +  in read_ctrl_pos()
3711 			 atomic_long_read(&lrugen->refaulted[hist][type][tier]);  in read_ctrl_pos()
3712 	pos->total = lrugen->avg_total[type][tier] +  in read_ctrl_pos()
3713 		     atomic_long_read(&lrugen->evicted[hist][type][tier]);  in read_ctrl_pos()
3715 		pos->total += lrugen->protected[hist][type][tier - 1];  in read_ctrl_pos()
3722 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in reset_ctrl_pos()  local
3724 	unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;  in reset_ctrl_pos()
3737 			sum = lrugen->avg_refaulted[type][tier] +  in reset_ctrl_pos()
3738 			      atomic_long_read(&lrugen->refaulted[hist][type][tier]);  in reset_ctrl_pos()
3739 			WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);  in reset_ctrl_pos()
3741 			sum = lrugen->avg_total[type][tier] +  in reset_ctrl_pos()
3742 			      atomic_long_read(&lrugen->evicted[hist][type][tier]);  in reset_ctrl_pos()
3744 				sum += lrugen->protected[hist][type][tier - 1];  in reset_ctrl_pos()
3745 			WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);  in reset_ctrl_pos()
3749 			atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);  in reset_ctrl_pos()
3750 			atomic_long_set(&lrugen->evicted[hist][type][tier], 0);  in reset_ctrl_pos()
3752 				WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);  in reset_ctrl_pos()
3799 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in folio_inc_gen()  local
3800 	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);  in folio_inc_gen()
3844 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in reset_batch_size()  local
3856 		WRITE_ONCE(lrugen->nr_pages[gen][type][zone],  in reset_batch_size()
3857 			   lrugen->nr_pages[gen][type][zone] + delta);  in reset_batch_size()
4363 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in inc_min_seq()  local
4364 	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);  in inc_min_seq()
4371 		struct list_head *head = &lrugen->folios[old_gen][type][zone];  in inc_min_seq()
4382 			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);  in inc_min_seq()
4390 	WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);  in inc_min_seq()
4399 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in try_to_inc_min_seq()  local
4406 		while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {  in try_to_inc_min_seq()
4410 				if (!list_empty(&lrugen->folios[gen][type][zone]))  in try_to_inc_min_seq()
4423 		min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);  in try_to_inc_min_seq()
4427 		if (min_seq[type] == lrugen->min_seq[type])  in try_to_inc_min_seq()
4431 		WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);  in try_to_inc_min_seq()
4442 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in inc_max_seq()  local
4468 	prev = lru_gen_from_seq(lrugen->max_seq - 1);  in inc_max_seq()
4469 	next = lru_gen_from_seq(lrugen->max_seq + 1);  in inc_max_seq()
4474 			long delta = lrugen->nr_pages[prev][type][zone] -  in inc_max_seq()
4475 				     lrugen->nr_pages[next][type][zone];  in inc_max_seq()
4488 	WRITE_ONCE(lrugen->timestamps[next], jiffies);  in inc_max_seq()
4490 	smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);  in inc_max_seq()
4501 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in try_to_inc_max_seq()  local
4503 	VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));  in try_to_inc_max_seq()
4554 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lruvec_is_sizable()  local
4566 				total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);  in lruvec_is_sizable()
4584 	birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);  in lruvec_is_reclaimable()
4768 	return READ_ONCE(lruvec->lrugen.seg);  in lru_gen_memcg_seg()
4781 	VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));  in lru_gen_rotate_memcg()
4784 	new = old = lruvec->lrugen.gen;  in lru_gen_rotate_memcg()
4798 	WRITE_ONCE(lruvec->lrugen.seg, seg);  in lru_gen_rotate_memcg()
4799 	WRITE_ONCE(lruvec->lrugen.gen, new);  in lru_gen_rotate_memcg()
4801 	hlist_nulls_del_rcu(&lruvec->lrugen.list);  in lru_gen_rotate_memcg()
4804 		hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);  in lru_gen_rotate_memcg()
4806 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);  in lru_gen_rotate_memcg()
4829 		VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));  in lru_gen_online_memcg()
4833 		lruvec->lrugen.gen = gen;  in lru_gen_online_memcg()
4835 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);  in lru_gen_online_memcg()
4864 		if (hlist_nulls_unhashed(&lruvec->lrugen.list))  in lru_gen_release_memcg()
4867 		gen = lruvec->lrugen.gen;  in lru_gen_release_memcg()
4869 		hlist_nulls_del_init_rcu(&lruvec->lrugen.list);  in lru_gen_release_memcg()
4911 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in sort_folio()  local
4935 	if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {  in sort_folio()
4936 		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);  in sort_folio()
4942 		int hist = lru_hist_from_seq(lrugen->min_seq[type]);  in sort_folio()
4945 		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);  in sort_folio()
4947 		WRITE_ONCE(lrugen->protected[hist][type][tier - 1],  in sort_folio()
4948 			   lrugen->protected[hist][type][tier - 1] + delta);  in sort_folio()
4955 		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);  in sort_folio()
4963 		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);  in sort_folio()
5014 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in scan_folios()  local
5022 	gen = lru_gen_from_seq(lrugen->min_seq[type]);  in scan_folios()
5028 		struct list_head *head = &lrugen->folios[gen][type][zone];  in scan_folios()
5269 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in should_run_aging()  local
5288 				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);  in should_run_aging()
5471 	struct lru_gen_folio *lrugen;  in shrink_many()  local
5483 	hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {  in shrink_many()
5492 		if (gen != READ_ONCE(lrugen->gen))  in shrink_many()
5495 		lruvec = container_of(lrugen, struct lruvec, lrugen);  in shrink_many()
5640 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in state_is_valid()  local
5642 	if (lrugen->enabled) {  in state_is_valid()
5653 			if (!list_empty(&lrugen->folios[gen][type][zone]))  in state_is_valid()
5698 		struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];  in drain_evictable()
5752 			lruvec->lrugen.enabled = enabled;  in lru_gen_change_state()
5915 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_seq_show_full()  local
5925 				n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);  in lru_gen_seq_show_full()
5926 				n[1] = READ_ONCE(lrugen->avg_total[type][tier]);  in lru_gen_seq_show_full()
5929 				n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);  in lru_gen_seq_show_full()
5930 				n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);  in lru_gen_seq_show_full()
5932 					n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]);  in lru_gen_seq_show_full()
5965 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_seq_show()  local
5993 		unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);  in lru_gen_seq_show()
6002 				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);  in lru_gen_seq_show()
6219 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_init_lruvec()  local
6221 	lrugen->max_seq = MIN_NR_GENS + 1;  in lru_gen_init_lruvec()
6222 	lrugen->enabled = lru_gen_enabled();  in lru_gen_init_lruvec()
6225 		lrugen->timestamps[i] = jiffies;  in lru_gen_init_lruvec()
6228 		INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);  in lru_gen_init_lruvec()
6263 		VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,  in lru_gen_exit_memcg()
6264 					   sizeof(lruvec->lrugen.nr_pages)));  in lru_gen_exit_memcg()
6266 		lruvec->lrugen.list.next = LIST_POISON1;  in lru_gen_exit_memcg()