1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* memcontrol.c - Memory Controller
3  *
4  * Copyright IBM Corporation, 2007
5  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6  *
7  * Copyright 2007 OpenVZ SWsoft Inc
8  * Author: Pavel Emelianov <xemul@openvz.org>
9  *
10  * Memory thresholds
11  * Copyright (C) 2009 Nokia Corporation
12  * Author: Kirill A. Shutemov
13  *
14  * Kernel Memory Controller
15  * Copyright (C) 2012 Parallels Inc. and Google Inc.
16  * Authors: Glauber Costa and Suleiman Souhlal
17  *
18  * Native page reclaim
19  * Charge lifetime sanitation
20  * Lockless page tracking & accounting
21  * Unified hierarchy configuration model
22  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
23  *
24  * Per memcg lru locking
25  * Copyright (C) 2020 Alibaba, Inc, Alex Shi
26  */
27 
28 #include <linux/page_counter.h>
29 #include <linux/memcontrol.h>
30 #include <linux/cgroup.h>
31 #include <linux/pagewalk.h>
32 #include <linux/sched/mm.h>
33 #include <linux/shmem_fs.h>
34 #include <linux/hugetlb.h>
35 #include <linux/pagemap.h>
36 #include <linux/vm_event_item.h>
37 #include <linux/smp.h>
38 #include <linux/page-flags.h>
39 #include <linux/backing-dev.h>
40 #include <linux/bit_spinlock.h>
41 #include <linux/rcupdate.h>
42 #include <linux/limits.h>
43 #include <linux/export.h>
44 #include <linux/mutex.h>
45 #include <linux/rbtree.h>
46 #include <linux/slab.h>
47 #include <linux/swap.h>
48 #include <linux/swapops.h>
49 #include <linux/spinlock.h>
50 #include <linux/eventfd.h>
51 #include <linux/poll.h>
52 #include <linux/sort.h>
53 #include <linux/fs.h>
54 #include <linux/seq_file.h>
55 #include <linux/vmpressure.h>
56 #include <linux/memremap.h>
57 #include <linux/mm_inline.h>
58 #include <linux/swap_cgroup.h>
59 #include <linux/cpu.h>
60 #include <linux/oom.h>
61 #include <linux/lockdep.h>
62 #include <linux/file.h>
63 #include <linux/resume_user_mode.h>
64 #include <linux/psi.h>
65 #include <linux/seq_buf.h>
66 #include "internal.h"
67 #include <net/sock.h>
68 #include <net/ip.h>
69 #include "slab.h"
70 #include "swap.h"
71 
72 #include <linux/uaccess.h>
73 
74 #include <trace/events/vmscan.h>
75 
76 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
77 EXPORT_SYMBOL(memory_cgrp_subsys);
78 
79 struct mem_cgroup *root_mem_cgroup __read_mostly;
80 
81 /* Active memory cgroup to use from an interrupt context */
82 DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
83 EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
84 
85 /* Socket memory accounting disabled? */
86 static bool cgroup_memory_nosocket __ro_after_init;
87 
88 /* Kernel memory accounting disabled? */
89 static bool cgroup_memory_nokmem __ro_after_init;
90 
91 #ifdef CONFIG_CGROUP_WRITEBACK
92 static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
93 #endif
94 
95 /* Whether legacy memory+swap accounting is active */
do_memsw_account(void)96 static bool do_memsw_account(void)
97 {
98 	return !cgroup_subsys_on_dfl(memory_cgrp_subsys);
99 }
100 
101 #define THRESHOLDS_EVENTS_TARGET 128
102 #define SOFTLIMIT_EVENTS_TARGET 1024
103 
104 /*
105  * Cgroups above their limits are maintained in a RB-Tree, independent of
106  * their hierarchy representation
107  */
108 
109 struct mem_cgroup_tree_per_node {
110 	struct rb_root rb_root;
111 	struct rb_node *rb_rightmost;
112 	spinlock_t lock;
113 };
114 
115 struct mem_cgroup_tree {
116 	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
117 };
118 
119 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
120 
121 /* for OOM */
122 struct mem_cgroup_eventfd_list {
123 	struct list_head list;
124 	struct eventfd_ctx *eventfd;
125 };
126 
127 /*
128  * cgroup_event represents events which userspace want to receive.
129  */
130 struct mem_cgroup_event {
131 	/*
132 	 * memcg which the event belongs to.
133 	 */
134 	struct mem_cgroup *memcg;
135 	/*
136 	 * eventfd to signal userspace about the event.
137 	 */
138 	struct eventfd_ctx *eventfd;
139 	/*
140 	 * Each of these stored in a list by the cgroup.
141 	 */
142 	struct list_head list;
143 	/*
144 	 * register_event() callback will be used to add new userspace
145 	 * waiter for changes related to this event.  Use eventfd_signal()
146 	 * on eventfd to send notification to userspace.
147 	 */
148 	int (*register_event)(struct mem_cgroup *memcg,
149 			      struct eventfd_ctx *eventfd, const char *args);
150 	/*
151 	 * unregister_event() callback will be called when userspace closes
152 	 * the eventfd or on cgroup removing.  This callback must be set,
153 	 * if you want provide notification functionality.
154 	 */
155 	void (*unregister_event)(struct mem_cgroup *memcg,
156 				 struct eventfd_ctx *eventfd);
157 	/*
158 	 * All fields below needed to unregister event when
159 	 * userspace closes eventfd.
160 	 */
161 	poll_table pt;
162 	wait_queue_head_t *wqh;
163 	wait_queue_entry_t wait;
164 	struct work_struct remove;
165 };
166 
167 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
168 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
169 
170 /* Stuffs for move charges at task migration. */
171 /*
172  * Types of charges to be moved.
173  */
174 #define MOVE_ANON	0x1U
175 #define MOVE_FILE	0x2U
176 #define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
177 
178 /* "mc" and its members are protected by cgroup_mutex */
179 static struct move_charge_struct {
180 	spinlock_t	  lock; /* for from, to */
181 	struct mm_struct  *mm;
182 	struct mem_cgroup *from;
183 	struct mem_cgroup *to;
184 	unsigned long flags;
185 	unsigned long precharge;
186 	unsigned long moved_charge;
187 	unsigned long moved_swap;
188 	struct task_struct *moving_task;	/* a task moving charges */
189 	wait_queue_head_t waitq;		/* a waitq for other context */
190 } mc = {
191 	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
192 	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
193 };
194 
195 /*
196  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
197  * limit reclaim to prevent infinite loops, if they ever occur.
198  */
199 #define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
200 #define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
201 
202 /* for encoding cft->private value on file */
203 enum res_type {
204 	_MEM,
205 	_MEMSWAP,
206 	_KMEM,
207 	_TCP,
208 };
209 
210 #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
211 #define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
212 #define MEMFILE_ATTR(val)	((val) & 0xffff)
213 
214 /*
215  * Iteration constructs for visiting all cgroups (under a tree).  If
216  * loops are exited prematurely (break), mem_cgroup_iter_break() must
217  * be used for reference counting.
218  */
219 #define for_each_mem_cgroup_tree(iter, root)		\
220 	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
221 	     iter != NULL;				\
222 	     iter = mem_cgroup_iter(root, iter, NULL))
223 
224 #define for_each_mem_cgroup(iter)			\
225 	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
226 	     iter != NULL;				\
227 	     iter = mem_cgroup_iter(NULL, iter, NULL))
228 
task_is_dying(void)229 static inline bool task_is_dying(void)
230 {
231 	return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
232 		(current->flags & PF_EXITING);
233 }
234 
235 /* Some nice accessors for the vmpressure. */
memcg_to_vmpressure(struct mem_cgroup * memcg)236 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
237 {
238 	if (!memcg)
239 		memcg = root_mem_cgroup;
240 	return &memcg->vmpressure;
241 }
242 
vmpressure_to_memcg(struct vmpressure * vmpr)243 struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr)
244 {
245 	return container_of(vmpr, struct mem_cgroup, vmpressure);
246 }
247 
248 #ifdef CONFIG_MEMCG_KMEM
249 static DEFINE_SPINLOCK(objcg_lock);
250 
mem_cgroup_kmem_disabled(void)251 bool mem_cgroup_kmem_disabled(void)
252 {
253 	return cgroup_memory_nokmem;
254 }
255 
256 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
257 				      unsigned int nr_pages);
258 
obj_cgroup_release(struct percpu_ref * ref)259 static void obj_cgroup_release(struct percpu_ref *ref)
260 {
261 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
262 	unsigned int nr_bytes;
263 	unsigned int nr_pages;
264 	unsigned long flags;
265 
266 	/*
267 	 * At this point all allocated objects are freed, and
268 	 * objcg->nr_charged_bytes can't have an arbitrary byte value.
269 	 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
270 	 *
271 	 * The following sequence can lead to it:
272 	 * 1) CPU0: objcg == stock->cached_objcg
273 	 * 2) CPU1: we do a small allocation (e.g. 92 bytes),
274 	 *          PAGE_SIZE bytes are charged
275 	 * 3) CPU1: a process from another memcg is allocating something,
276 	 *          the stock if flushed,
277 	 *          objcg->nr_charged_bytes = PAGE_SIZE - 92
278 	 * 5) CPU0: we do release this object,
279 	 *          92 bytes are added to stock->nr_bytes
280 	 * 6) CPU0: stock is flushed,
281 	 *          92 bytes are added to objcg->nr_charged_bytes
282 	 *
283 	 * In the result, nr_charged_bytes == PAGE_SIZE.
284 	 * This page will be uncharged in obj_cgroup_release().
285 	 */
286 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);
287 	WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
288 	nr_pages = nr_bytes >> PAGE_SHIFT;
289 
290 	if (nr_pages)
291 		obj_cgroup_uncharge_pages(objcg, nr_pages);
292 
293 	spin_lock_irqsave(&objcg_lock, flags);
294 	list_del(&objcg->list);
295 	spin_unlock_irqrestore(&objcg_lock, flags);
296 
297 	percpu_ref_exit(ref);
298 	kfree_rcu(objcg, rcu);
299 }
300 
obj_cgroup_alloc(void)301 static struct obj_cgroup *obj_cgroup_alloc(void)
302 {
303 	struct obj_cgroup *objcg;
304 	int ret;
305 
306 	objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
307 	if (!objcg)
308 		return NULL;
309 
310 	ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
311 			      GFP_KERNEL);
312 	if (ret) {
313 		kfree(objcg);
314 		return NULL;
315 	}
316 	INIT_LIST_HEAD(&objcg->list);
317 	return objcg;
318 }
319 
memcg_reparent_objcgs(struct mem_cgroup * memcg,struct mem_cgroup * parent)320 static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
321 				  struct mem_cgroup *parent)
322 {
323 	struct obj_cgroup *objcg, *iter;
324 
325 	objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
326 
327 	spin_lock_irq(&objcg_lock);
328 
329 	/* 1) Ready to reparent active objcg. */
330 	list_add(&objcg->list, &memcg->objcg_list);
331 	/* 2) Reparent active objcg and already reparented objcgs to parent. */
332 	list_for_each_entry(iter, &memcg->objcg_list, list)
333 		WRITE_ONCE(iter->memcg, parent);
334 	/* 3) Move already reparented objcgs to the parent's list */
335 	list_splice(&memcg->objcg_list, &parent->objcg_list);
336 
337 	spin_unlock_irq(&objcg_lock);
338 
339 	percpu_ref_kill(&objcg->refcnt);
340 }
341 
342 /*
343  * A lot of the calls to the cache allocation functions are expected to be
344  * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
345  * conditional to this static branch, we'll have to allow modules that does
346  * kmem_cache_alloc and the such to see this symbol as well
347  */
348 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
349 EXPORT_SYMBOL(memcg_kmem_enabled_key);
350 #endif
351 
352 /**
353  * mem_cgroup_css_from_page - css of the memcg associated with a page
354  * @page: page of interest
355  *
356  * If memcg is bound to the default hierarchy, css of the memcg associated
357  * with @page is returned.  The returned css remains associated with @page
358  * until it is released.
359  *
360  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
361  * is returned.
362  */
mem_cgroup_css_from_page(struct page * page)363 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
364 {
365 	struct mem_cgroup *memcg;
366 
367 	memcg = page_memcg(page);
368 
369 	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
370 		memcg = root_mem_cgroup;
371 
372 	return &memcg->css;
373 }
374 
375 /**
376  * page_cgroup_ino - return inode number of the memcg a page is charged to
377  * @page: the page
378  *
379  * Look up the closest online ancestor of the memory cgroup @page is charged to
380  * and return its inode number or 0 if @page is not charged to any cgroup. It
381  * is safe to call this function without holding a reference to @page.
382  *
383  * Note, this function is inherently racy, because there is nothing to prevent
384  * the cgroup inode from getting torn down and potentially reallocated a moment
385  * after page_cgroup_ino() returns, so it only should be used by callers that
386  * do not care (such as procfs interfaces).
387  */
page_cgroup_ino(struct page * page)388 ino_t page_cgroup_ino(struct page *page)
389 {
390 	struct mem_cgroup *memcg;
391 	unsigned long ino = 0;
392 
393 	rcu_read_lock();
394 	memcg = page_memcg_check(page);
395 
396 	while (memcg && !(memcg->css.flags & CSS_ONLINE))
397 		memcg = parent_mem_cgroup(memcg);
398 	if (memcg)
399 		ino = cgroup_ino(memcg->css.cgroup);
400 	rcu_read_unlock();
401 	return ino;
402 }
403 
__mem_cgroup_insert_exceeded(struct mem_cgroup_per_node * mz,struct mem_cgroup_tree_per_node * mctz,unsigned long new_usage_in_excess)404 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
405 					 struct mem_cgroup_tree_per_node *mctz,
406 					 unsigned long new_usage_in_excess)
407 {
408 	struct rb_node **p = &mctz->rb_root.rb_node;
409 	struct rb_node *parent = NULL;
410 	struct mem_cgroup_per_node *mz_node;
411 	bool rightmost = true;
412 
413 	if (mz->on_tree)
414 		return;
415 
416 	mz->usage_in_excess = new_usage_in_excess;
417 	if (!mz->usage_in_excess)
418 		return;
419 	while (*p) {
420 		parent = *p;
421 		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
422 					tree_node);
423 		if (mz->usage_in_excess < mz_node->usage_in_excess) {
424 			p = &(*p)->rb_left;
425 			rightmost = false;
426 		} else {
427 			p = &(*p)->rb_right;
428 		}
429 	}
430 
431 	if (rightmost)
432 		mctz->rb_rightmost = &mz->tree_node;
433 
434 	rb_link_node(&mz->tree_node, parent, p);
435 	rb_insert_color(&mz->tree_node, &mctz->rb_root);
436 	mz->on_tree = true;
437 }
438 
__mem_cgroup_remove_exceeded(struct mem_cgroup_per_node * mz,struct mem_cgroup_tree_per_node * mctz)439 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
440 					 struct mem_cgroup_tree_per_node *mctz)
441 {
442 	if (!mz->on_tree)
443 		return;
444 
445 	if (&mz->tree_node == mctz->rb_rightmost)
446 		mctz->rb_rightmost = rb_prev(&mz->tree_node);
447 
448 	rb_erase(&mz->tree_node, &mctz->rb_root);
449 	mz->on_tree = false;
450 }
451 
mem_cgroup_remove_exceeded(struct mem_cgroup_per_node * mz,struct mem_cgroup_tree_per_node * mctz)452 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
453 				       struct mem_cgroup_tree_per_node *mctz)
454 {
455 	unsigned long flags;
456 
457 	spin_lock_irqsave(&mctz->lock, flags);
458 	__mem_cgroup_remove_exceeded(mz, mctz);
459 	spin_unlock_irqrestore(&mctz->lock, flags);
460 }
461 
soft_limit_excess(struct mem_cgroup * memcg)462 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
463 {
464 	unsigned long nr_pages = page_counter_read(&memcg->memory);
465 	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
466 	unsigned long excess = 0;
467 
468 	if (nr_pages > soft_limit)
469 		excess = nr_pages - soft_limit;
470 
471 	return excess;
472 }
473 
mem_cgroup_update_tree(struct mem_cgroup * memcg,int nid)474 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
475 {
476 	unsigned long excess;
477 	struct mem_cgroup_per_node *mz;
478 	struct mem_cgroup_tree_per_node *mctz;
479 
480 	mctz = soft_limit_tree.rb_tree_per_node[nid];
481 	if (!mctz)
482 		return;
483 	/*
484 	 * Necessary to update all ancestors when hierarchy is used.
485 	 * because their event counter is not touched.
486 	 */
487 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
488 		mz = memcg->nodeinfo[nid];
489 		excess = soft_limit_excess(memcg);
490 		/*
491 		 * We have to update the tree if mz is on RB-tree or
492 		 * mem is over its softlimit.
493 		 */
494 		if (excess || mz->on_tree) {
495 			unsigned long flags;
496 
497 			spin_lock_irqsave(&mctz->lock, flags);
498 			/* if on-tree, remove it */
499 			if (mz->on_tree)
500 				__mem_cgroup_remove_exceeded(mz, mctz);
501 			/*
502 			 * Insert again. mz->usage_in_excess will be updated.
503 			 * If excess is 0, no tree ops.
504 			 */
505 			__mem_cgroup_insert_exceeded(mz, mctz, excess);
506 			spin_unlock_irqrestore(&mctz->lock, flags);
507 		}
508 	}
509 }
510 
mem_cgroup_remove_from_trees(struct mem_cgroup * memcg)511 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
512 {
513 	struct mem_cgroup_tree_per_node *mctz;
514 	struct mem_cgroup_per_node *mz;
515 	int nid;
516 
517 	for_each_node(nid) {
518 		mz = memcg->nodeinfo[nid];
519 		mctz = soft_limit_tree.rb_tree_per_node[nid];
520 		if (mctz)
521 			mem_cgroup_remove_exceeded(mz, mctz);
522 	}
523 }
524 
525 static struct mem_cgroup_per_node *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node * mctz)526 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
527 {
528 	struct mem_cgroup_per_node *mz;
529 
530 retry:
531 	mz = NULL;
532 	if (!mctz->rb_rightmost)
533 		goto done;		/* Nothing to reclaim from */
534 
535 	mz = rb_entry(mctz->rb_rightmost,
536 		      struct mem_cgroup_per_node, tree_node);
537 	/*
538 	 * Remove the node now but someone else can add it back,
539 	 * we will to add it back at the end of reclaim to its correct
540 	 * position in the tree.
541 	 */
542 	__mem_cgroup_remove_exceeded(mz, mctz);
543 	if (!soft_limit_excess(mz->memcg) ||
544 	    !css_tryget(&mz->memcg->css))
545 		goto retry;
546 done:
547 	return mz;
548 }
549 
550 static struct mem_cgroup_per_node *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node * mctz)551 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
552 {
553 	struct mem_cgroup_per_node *mz;
554 
555 	spin_lock_irq(&mctz->lock);
556 	mz = __mem_cgroup_largest_soft_limit_node(mctz);
557 	spin_unlock_irq(&mctz->lock);
558 	return mz;
559 }
560 
561 /*
562  * memcg and lruvec stats flushing
563  *
564  * Many codepaths leading to stats update or read are performance sensitive and
565  * adding stats flushing in such codepaths is not desirable. So, to optimize the
566  * flushing the kernel does:
567  *
568  * 1) Periodically and asynchronously flush the stats every 2 seconds to not let
569  *    rstat update tree grow unbounded.
570  *
571  * 2) Flush the stats synchronously on reader side only when there are more than
572  *    (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization
573  *    will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but
574  *    only for 2 seconds due to (1).
575  */
576 static void flush_memcg_stats_dwork(struct work_struct *w);
577 static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);
578 static DEFINE_SPINLOCK(stats_flush_lock);
579 static DEFINE_PER_CPU(unsigned int, stats_updates);
580 static atomic_t stats_flush_threshold = ATOMIC_INIT(0);
581 static u64 flush_next_time;
582 
583 #define FLUSH_TIME (2UL*HZ)
584 
585 /*
586  * Accessors to ensure that preemption is disabled on PREEMPT_RT because it can
587  * not rely on this as part of an acquired spinlock_t lock. These functions are
588  * never used in hardirq context on PREEMPT_RT and therefore disabling preemtion
589  * is sufficient.
590  */
memcg_stats_lock(void)591 static void memcg_stats_lock(void)
592 {
593 	preempt_disable_nested();
594 	VM_WARN_ON_IRQS_ENABLED();
595 }
596 
__memcg_stats_lock(void)597 static void __memcg_stats_lock(void)
598 {
599 	preempt_disable_nested();
600 }
601 
memcg_stats_unlock(void)602 static void memcg_stats_unlock(void)
603 {
604 	preempt_enable_nested();
605 }
606 
memcg_rstat_updated(struct mem_cgroup * memcg,int val)607 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
608 {
609 	unsigned int x;
610 
611 	cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
612 
613 	x = __this_cpu_add_return(stats_updates, abs(val));
614 	if (x > MEMCG_CHARGE_BATCH) {
615 		/*
616 		 * If stats_flush_threshold exceeds the threshold
617 		 * (>num_online_cpus()), cgroup stats update will be triggered
618 		 * in __mem_cgroup_flush_stats(). Increasing this var further
619 		 * is redundant and simply adds overhead in atomic update.
620 		 */
621 		if (atomic_read(&stats_flush_threshold) <= num_online_cpus())
622 			atomic_add(x / MEMCG_CHARGE_BATCH, &stats_flush_threshold);
623 		__this_cpu_write(stats_updates, 0);
624 	}
625 }
626 
__mem_cgroup_flush_stats(void)627 static void __mem_cgroup_flush_stats(void)
628 {
629 	unsigned long flag;
630 
631 	if (!spin_trylock_irqsave(&stats_flush_lock, flag))
632 		return;
633 
634 	flush_next_time = jiffies_64 + 2*FLUSH_TIME;
635 	cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup);
636 	atomic_set(&stats_flush_threshold, 0);
637 	spin_unlock_irqrestore(&stats_flush_lock, flag);
638 }
639 
mem_cgroup_flush_stats(void)640 void mem_cgroup_flush_stats(void)
641 {
642 	if (atomic_read(&stats_flush_threshold) > num_online_cpus())
643 		__mem_cgroup_flush_stats();
644 }
645 
mem_cgroup_flush_stats_delayed(void)646 void mem_cgroup_flush_stats_delayed(void)
647 {
648 	if (time_after64(jiffies_64, flush_next_time))
649 		mem_cgroup_flush_stats();
650 }
651 
flush_memcg_stats_dwork(struct work_struct * w)652 static void flush_memcg_stats_dwork(struct work_struct *w)
653 {
654 	__mem_cgroup_flush_stats();
655 	queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME);
656 }
657 
658 /* Subset of vm_event_item to report for memcg event stats */
659 static const unsigned int memcg_vm_event_stat[] = {
660 	PGPGIN,
661 	PGPGOUT,
662 	PGSCAN_KSWAPD,
663 	PGSCAN_DIRECT,
664 	PGSTEAL_KSWAPD,
665 	PGSTEAL_DIRECT,
666 	PGFAULT,
667 	PGMAJFAULT,
668 	PGREFILL,
669 	PGACTIVATE,
670 	PGDEACTIVATE,
671 	PGLAZYFREE,
672 	PGLAZYFREED,
673 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
674 	ZSWPIN,
675 	ZSWPOUT,
676 #endif
677 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
678 	THP_FAULT_ALLOC,
679 	THP_COLLAPSE_ALLOC,
680 #endif
681 };
682 
683 #define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat)
684 static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly;
685 
init_memcg_events(void)686 static void init_memcg_events(void)
687 {
688 	int i;
689 
690 	for (i = 0; i < NR_MEMCG_EVENTS; ++i)
691 		mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1;
692 }
693 
memcg_events_index(enum vm_event_item idx)694 static inline int memcg_events_index(enum vm_event_item idx)
695 {
696 	return mem_cgroup_events_index[idx] - 1;
697 }
698 
699 struct memcg_vmstats_percpu {
700 	/* Local (CPU and cgroup) page state & events */
701 	long			state[MEMCG_NR_STAT];
702 	unsigned long		events[NR_MEMCG_EVENTS];
703 
704 	/* Delta calculation for lockless upward propagation */
705 	long			state_prev[MEMCG_NR_STAT];
706 	unsigned long		events_prev[NR_MEMCG_EVENTS];
707 
708 	/* Cgroup1: threshold notifications & softlimit tree updates */
709 	unsigned long		nr_page_events;
710 	unsigned long		targets[MEM_CGROUP_NTARGETS];
711 };
712 
713 struct memcg_vmstats {
714 	/* Aggregated (CPU and subtree) page state & events */
715 	long			state[MEMCG_NR_STAT];
716 	unsigned long		events[NR_MEMCG_EVENTS];
717 
718 	/* Pending child counts during tree propagation */
719 	long			state_pending[MEMCG_NR_STAT];
720 	unsigned long		events_pending[NR_MEMCG_EVENTS];
721 };
722 
memcg_page_state(struct mem_cgroup * memcg,int idx)723 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
724 {
725 	long x = READ_ONCE(memcg->vmstats->state[idx]);
726 #ifdef CONFIG_SMP
727 	if (x < 0)
728 		x = 0;
729 #endif
730 	return x;
731 }
732 
733 /**
734  * __mod_memcg_state - update cgroup memory statistics
735  * @memcg: the memory cgroup
736  * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
737  * @val: delta to add to the counter, can be negative
738  */
__mod_memcg_state(struct mem_cgroup * memcg,int idx,int val)739 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
740 {
741 	if (mem_cgroup_disabled())
742 		return;
743 
744 	__this_cpu_add(memcg->vmstats_percpu->state[idx], val);
745 	memcg_rstat_updated(memcg, val);
746 }
747 
748 /* idx can be of type enum memcg_stat_item or node_stat_item. */
memcg_page_state_local(struct mem_cgroup * memcg,int idx)749 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
750 {
751 	long x = 0;
752 	int cpu;
753 
754 	for_each_possible_cpu(cpu)
755 		x += per_cpu(memcg->vmstats_percpu->state[idx], cpu);
756 #ifdef CONFIG_SMP
757 	if (x < 0)
758 		x = 0;
759 #endif
760 	return x;
761 }
762 
__mod_memcg_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)763 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
764 			      int val)
765 {
766 	struct mem_cgroup_per_node *pn;
767 	struct mem_cgroup *memcg;
768 
769 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
770 	memcg = pn->memcg;
771 
772 	/*
773 	 * The caller from rmap relay on disabled preemption becase they never
774 	 * update their counter from in-interrupt context. For these two
775 	 * counters we check that the update is never performed from an
776 	 * interrupt context while other caller need to have disabled interrupt.
777 	 */
778 	__memcg_stats_lock();
779 	if (IS_ENABLED(CONFIG_DEBUG_VM)) {
780 		switch (idx) {
781 		case NR_ANON_MAPPED:
782 		case NR_FILE_MAPPED:
783 		case NR_ANON_THPS:
784 		case NR_SHMEM_PMDMAPPED:
785 		case NR_FILE_PMDMAPPED:
786 			WARN_ON_ONCE(!in_task());
787 			break;
788 		default:
789 			VM_WARN_ON_IRQS_ENABLED();
790 		}
791 	}
792 
793 	/* Update memcg */
794 	__this_cpu_add(memcg->vmstats_percpu->state[idx], val);
795 
796 	/* Update lruvec */
797 	__this_cpu_add(pn->lruvec_stats_percpu->state[idx], val);
798 
799 	memcg_rstat_updated(memcg, val);
800 	memcg_stats_unlock();
801 }
802 
803 /**
804  * __mod_lruvec_state - update lruvec memory statistics
805  * @lruvec: the lruvec
806  * @idx: the stat item
807  * @val: delta to add to the counter, can be negative
808  *
809  * The lruvec is the intersection of the NUMA node and a cgroup. This
810  * function updates the all three counters that are affected by a
811  * change of state at this level: per-node, per-cgroup, per-lruvec.
812  */
__mod_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)813 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
814 			int val)
815 {
816 	/* Update node */
817 	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
818 
819 	/* Update memcg and lruvec */
820 	if (!mem_cgroup_disabled())
821 		__mod_memcg_lruvec_state(lruvec, idx, val);
822 }
823 
__mod_lruvec_page_state(struct page * page,enum node_stat_item idx,int val)824 void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx,
825 			     int val)
826 {
827 	struct page *head = compound_head(page); /* rmap on tail pages */
828 	struct mem_cgroup *memcg;
829 	pg_data_t *pgdat = page_pgdat(page);
830 	struct lruvec *lruvec;
831 
832 	rcu_read_lock();
833 	memcg = page_memcg(head);
834 	/* Untracked pages have no memcg, no lruvec. Update only the node */
835 	if (!memcg) {
836 		rcu_read_unlock();
837 		__mod_node_page_state(pgdat, idx, val);
838 		return;
839 	}
840 
841 	lruvec = mem_cgroup_lruvec(memcg, pgdat);
842 	__mod_lruvec_state(lruvec, idx, val);
843 	rcu_read_unlock();
844 }
845 EXPORT_SYMBOL(__mod_lruvec_page_state);
846 
__mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)847 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val)
848 {
849 	pg_data_t *pgdat = page_pgdat(virt_to_page(p));
850 	struct mem_cgroup *memcg;
851 	struct lruvec *lruvec;
852 
853 	rcu_read_lock();
854 	memcg = mem_cgroup_from_slab_obj(p);
855 
856 	/*
857 	 * Untracked pages have no memcg, no lruvec. Update only the
858 	 * node. If we reparent the slab objects to the root memcg,
859 	 * when we free the slab object, we need to update the per-memcg
860 	 * vmstats to keep it correct for the root memcg.
861 	 */
862 	if (!memcg) {
863 		__mod_node_page_state(pgdat, idx, val);
864 	} else {
865 		lruvec = mem_cgroup_lruvec(memcg, pgdat);
866 		__mod_lruvec_state(lruvec, idx, val);
867 	}
868 	rcu_read_unlock();
869 }
870 
871 /**
872  * __count_memcg_events - account VM events in a cgroup
873  * @memcg: the memory cgroup
874  * @idx: the event item
875  * @count: the number of events that occurred
876  */
__count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)877 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
878 			  unsigned long count)
879 {
880 	int index = memcg_events_index(idx);
881 
882 	if (mem_cgroup_disabled() || index < 0)
883 		return;
884 
885 	memcg_stats_lock();
886 	__this_cpu_add(memcg->vmstats_percpu->events[index], count);
887 	memcg_rstat_updated(memcg, count);
888 	memcg_stats_unlock();
889 }
890 
memcg_events(struct mem_cgroup * memcg,int event)891 static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
892 {
893 	int index = memcg_events_index(event);
894 
895 	if (index < 0)
896 		return 0;
897 	return READ_ONCE(memcg->vmstats->events[index]);
898 }
899 
memcg_events_local(struct mem_cgroup * memcg,int event)900 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
901 {
902 	long x = 0;
903 	int cpu;
904 	int index = memcg_events_index(event);
905 
906 	if (index < 0)
907 		return 0;
908 
909 	for_each_possible_cpu(cpu)
910 		x += per_cpu(memcg->vmstats_percpu->events[index], cpu);
911 	return x;
912 }
913 
mem_cgroup_charge_statistics(struct mem_cgroup * memcg,int nr_pages)914 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
915 					 int nr_pages)
916 {
917 	/* pagein of a big page is an event. So, ignore page size */
918 	if (nr_pages > 0)
919 		__count_memcg_events(memcg, PGPGIN, 1);
920 	else {
921 		__count_memcg_events(memcg, PGPGOUT, 1);
922 		nr_pages = -nr_pages; /* for event */
923 	}
924 
925 	__this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
926 }
927 
mem_cgroup_event_ratelimit(struct mem_cgroup * memcg,enum mem_cgroup_events_target target)928 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
929 				       enum mem_cgroup_events_target target)
930 {
931 	unsigned long val, next;
932 
933 	val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
934 	next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
935 	/* from time_after() in jiffies.h */
936 	if ((long)(next - val) < 0) {
937 		switch (target) {
938 		case MEM_CGROUP_TARGET_THRESH:
939 			next = val + THRESHOLDS_EVENTS_TARGET;
940 			break;
941 		case MEM_CGROUP_TARGET_SOFTLIMIT:
942 			next = val + SOFTLIMIT_EVENTS_TARGET;
943 			break;
944 		default:
945 			break;
946 		}
947 		__this_cpu_write(memcg->vmstats_percpu->targets[target], next);
948 		return true;
949 	}
950 	return false;
951 }
952 
953 /*
954  * Check events in order.
955  *
956  */
memcg_check_events(struct mem_cgroup * memcg,int nid)957 static void memcg_check_events(struct mem_cgroup *memcg, int nid)
958 {
959 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
960 		return;
961 
962 	/* threshold event is triggered in finer grain than soft limit */
963 	if (unlikely(mem_cgroup_event_ratelimit(memcg,
964 						MEM_CGROUP_TARGET_THRESH))) {
965 		bool do_softlimit;
966 
967 		do_softlimit = mem_cgroup_event_ratelimit(memcg,
968 						MEM_CGROUP_TARGET_SOFTLIMIT);
969 		mem_cgroup_threshold(memcg);
970 		if (unlikely(do_softlimit))
971 			mem_cgroup_update_tree(memcg, nid);
972 	}
973 }
974 
mem_cgroup_from_task(struct task_struct * p)975 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
976 {
977 	/*
978 	 * mm_update_next_owner() may clear mm->owner to NULL
979 	 * if it races with swapoff, page migration, etc.
980 	 * So this can be called with p == NULL.
981 	 */
982 	if (unlikely(!p))
983 		return NULL;
984 
985 	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
986 }
987 EXPORT_SYMBOL(mem_cgroup_from_task);
988 
active_memcg(void)989 static __always_inline struct mem_cgroup *active_memcg(void)
990 {
991 	if (!in_task())
992 		return this_cpu_read(int_active_memcg);
993 	else
994 		return current->active_memcg;
995 }
996 
997 /**
998  * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
999  * @mm: mm from which memcg should be extracted. It can be NULL.
1000  *
1001  * Obtain a reference on mm->memcg and returns it if successful. If mm
1002  * is NULL, then the memcg is chosen as follows:
1003  * 1) The active memcg, if set.
1004  * 2) current->mm->memcg, if available
1005  * 3) root memcg
1006  * If mem_cgroup is disabled, NULL is returned.
1007  */
get_mem_cgroup_from_mm(struct mm_struct * mm)1008 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1009 {
1010 	struct mem_cgroup *memcg;
1011 
1012 	if (mem_cgroup_disabled())
1013 		return NULL;
1014 
1015 	/*
1016 	 * Page cache insertions can happen without an
1017 	 * actual mm context, e.g. during disk probing
1018 	 * on boot, loopback IO, acct() writes etc.
1019 	 *
1020 	 * No need to css_get on root memcg as the reference
1021 	 * counting is disabled on the root level in the
1022 	 * cgroup core. See CSS_NO_REF.
1023 	 */
1024 	if (unlikely(!mm)) {
1025 		memcg = active_memcg();
1026 		if (unlikely(memcg)) {
1027 			/* remote memcg must hold a ref */
1028 			css_get(&memcg->css);
1029 			return memcg;
1030 		}
1031 		mm = current->mm;
1032 		if (unlikely(!mm))
1033 			return root_mem_cgroup;
1034 	}
1035 
1036 	rcu_read_lock();
1037 	do {
1038 		memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1039 		if (unlikely(!memcg))
1040 			memcg = root_mem_cgroup;
1041 	} while (!css_tryget(&memcg->css));
1042 	rcu_read_unlock();
1043 	return memcg;
1044 }
1045 EXPORT_SYMBOL(get_mem_cgroup_from_mm);
1046 
memcg_kmem_bypass(void)1047 static __always_inline bool memcg_kmem_bypass(void)
1048 {
1049 	/* Allow remote memcg charging from any context. */
1050 	if (unlikely(active_memcg()))
1051 		return false;
1052 
1053 	/* Memcg to charge can't be determined. */
1054 	if (!in_task() || !current->mm || (current->flags & PF_KTHREAD))
1055 		return true;
1056 
1057 	return false;
1058 }
1059 
1060 /**
1061  * mem_cgroup_iter - iterate over memory cgroup hierarchy
1062  * @root: hierarchy root
1063  * @prev: previously returned memcg, NULL on first invocation
1064  * @reclaim: cookie for shared reclaim walks, NULL for full walks
1065  *
1066  * Returns references to children of the hierarchy below @root, or
1067  * @root itself, or %NULL after a full round-trip.
1068  *
1069  * Caller must pass the return value in @prev on subsequent
1070  * invocations for reference counting, or use mem_cgroup_iter_break()
1071  * to cancel a hierarchy walk before the round-trip is complete.
1072  *
1073  * Reclaimers can specify a node in @reclaim to divide up the memcgs
1074  * in the hierarchy among all concurrent reclaimers operating on the
1075  * same node.
1076  */
mem_cgroup_iter(struct mem_cgroup * root,struct mem_cgroup * prev,struct mem_cgroup_reclaim_cookie * reclaim)1077 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1078 				   struct mem_cgroup *prev,
1079 				   struct mem_cgroup_reclaim_cookie *reclaim)
1080 {
1081 	struct mem_cgroup_reclaim_iter *iter;
1082 	struct cgroup_subsys_state *css = NULL;
1083 	struct mem_cgroup *memcg = NULL;
1084 	struct mem_cgroup *pos = NULL;
1085 
1086 	if (mem_cgroup_disabled())
1087 		return NULL;
1088 
1089 	if (!root)
1090 		root = root_mem_cgroup;
1091 
1092 	rcu_read_lock();
1093 
1094 	if (reclaim) {
1095 		struct mem_cgroup_per_node *mz;
1096 
1097 		mz = root->nodeinfo[reclaim->pgdat->node_id];
1098 		iter = &mz->iter;
1099 
1100 		/*
1101 		 * On start, join the current reclaim iteration cycle.
1102 		 * Exit when a concurrent walker completes it.
1103 		 */
1104 		if (!prev)
1105 			reclaim->generation = iter->generation;
1106 		else if (reclaim->generation != iter->generation)
1107 			goto out_unlock;
1108 
1109 		while (1) {
1110 			pos = READ_ONCE(iter->position);
1111 			if (!pos || css_tryget(&pos->css))
1112 				break;
1113 			/*
1114 			 * css reference reached zero, so iter->position will
1115 			 * be cleared by ->css_released. However, we should not
1116 			 * rely on this happening soon, because ->css_released
1117 			 * is called from a work queue, and by busy-waiting we
1118 			 * might block it. So we clear iter->position right
1119 			 * away.
1120 			 */
1121 			(void)cmpxchg(&iter->position, pos, NULL);
1122 		}
1123 	} else if (prev) {
1124 		pos = prev;
1125 	}
1126 
1127 	if (pos)
1128 		css = &pos->css;
1129 
1130 	for (;;) {
1131 		css = css_next_descendant_pre(css, &root->css);
1132 		if (!css) {
1133 			/*
1134 			 * Reclaimers share the hierarchy walk, and a
1135 			 * new one might jump in right at the end of
1136 			 * the hierarchy - make sure they see at least
1137 			 * one group and restart from the beginning.
1138 			 */
1139 			if (!prev)
1140 				continue;
1141 			break;
1142 		}
1143 
1144 		/*
1145 		 * Verify the css and acquire a reference.  The root
1146 		 * is provided by the caller, so we know it's alive
1147 		 * and kicking, and don't take an extra reference.
1148 		 */
1149 		if (css == &root->css || css_tryget(css)) {
1150 			memcg = mem_cgroup_from_css(css);
1151 			break;
1152 		}
1153 	}
1154 
1155 	if (reclaim) {
1156 		/*
1157 		 * The position could have already been updated by a competing
1158 		 * thread, so check that the value hasn't changed since we read
1159 		 * it to avoid reclaiming from the same cgroup twice.
1160 		 */
1161 		(void)cmpxchg(&iter->position, pos, memcg);
1162 
1163 		if (pos)
1164 			css_put(&pos->css);
1165 
1166 		if (!memcg)
1167 			iter->generation++;
1168 	}
1169 
1170 out_unlock:
1171 	rcu_read_unlock();
1172 	if (prev && prev != root)
1173 		css_put(&prev->css);
1174 
1175 	return memcg;
1176 }
1177 
1178 /**
1179  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1180  * @root: hierarchy root
1181  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1182  */
mem_cgroup_iter_break(struct mem_cgroup * root,struct mem_cgroup * prev)1183 void mem_cgroup_iter_break(struct mem_cgroup *root,
1184 			   struct mem_cgroup *prev)
1185 {
1186 	if (!root)
1187 		root = root_mem_cgroup;
1188 	if (prev && prev != root)
1189 		css_put(&prev->css);
1190 }
1191 
__invalidate_reclaim_iterators(struct mem_cgroup * from,struct mem_cgroup * dead_memcg)1192 static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1193 					struct mem_cgroup *dead_memcg)
1194 {
1195 	struct mem_cgroup_reclaim_iter *iter;
1196 	struct mem_cgroup_per_node *mz;
1197 	int nid;
1198 
1199 	for_each_node(nid) {
1200 		mz = from->nodeinfo[nid];
1201 		iter = &mz->iter;
1202 		cmpxchg(&iter->position, dead_memcg, NULL);
1203 	}
1204 }
1205 
invalidate_reclaim_iterators(struct mem_cgroup * dead_memcg)1206 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1207 {
1208 	struct mem_cgroup *memcg = dead_memcg;
1209 	struct mem_cgroup *last;
1210 
1211 	do {
1212 		__invalidate_reclaim_iterators(memcg, dead_memcg);
1213 		last = memcg;
1214 	} while ((memcg = parent_mem_cgroup(memcg)));
1215 
1216 	/*
1217 	 * When cgroup1 non-hierarchy mode is used,
1218 	 * parent_mem_cgroup() does not walk all the way up to the
1219 	 * cgroup root (root_mem_cgroup). So we have to handle
1220 	 * dead_memcg from cgroup root separately.
1221 	 */
1222 	if (last != root_mem_cgroup)
1223 		__invalidate_reclaim_iterators(root_mem_cgroup,
1224 						dead_memcg);
1225 }
1226 
1227 /**
1228  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1229  * @memcg: hierarchy root
1230  * @fn: function to call for each task
1231  * @arg: argument passed to @fn
1232  *
1233  * This function iterates over tasks attached to @memcg or to any of its
1234  * descendants and calls @fn for each task. If @fn returns a non-zero
1235  * value, the function breaks the iteration loop and returns the value.
1236  * Otherwise, it will iterate over all tasks and return 0.
1237  *
1238  * This function must not be called for the root memory cgroup.
1239  */
mem_cgroup_scan_tasks(struct mem_cgroup * memcg,int (* fn)(struct task_struct *,void *),void * arg)1240 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1241 			  int (*fn)(struct task_struct *, void *), void *arg)
1242 {
1243 	struct mem_cgroup *iter;
1244 	int ret = 0;
1245 
1246 	BUG_ON(memcg == root_mem_cgroup);
1247 
1248 	for_each_mem_cgroup_tree(iter, memcg) {
1249 		struct css_task_iter it;
1250 		struct task_struct *task;
1251 
1252 		css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
1253 		while (!ret && (task = css_task_iter_next(&it)))
1254 			ret = fn(task, arg);
1255 		css_task_iter_end(&it);
1256 		if (ret) {
1257 			mem_cgroup_iter_break(memcg, iter);
1258 			break;
1259 		}
1260 	}
1261 	return ret;
1262 }
1263 
1264 #ifdef CONFIG_DEBUG_VM
lruvec_memcg_debug(struct lruvec * lruvec,struct folio * folio)1265 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
1266 {
1267 	struct mem_cgroup *memcg;
1268 
1269 	if (mem_cgroup_disabled())
1270 		return;
1271 
1272 	memcg = folio_memcg(folio);
1273 
1274 	if (!memcg)
1275 		VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != root_mem_cgroup, folio);
1276 	else
1277 		VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio);
1278 }
1279 #endif
1280 
1281 /**
1282  * folio_lruvec_lock - Lock the lruvec for a folio.
1283  * @folio: Pointer to the folio.
1284  *
1285  * These functions are safe to use under any of the following conditions:
1286  * - folio locked
1287  * - folio_test_lru false
1288  * - folio_memcg_lock()
1289  * - folio frozen (refcount of 0)
1290  *
1291  * Return: The lruvec this folio is on with its lock held.
1292  */
folio_lruvec_lock(struct folio * folio)1293 struct lruvec *folio_lruvec_lock(struct folio *folio)
1294 {
1295 	struct lruvec *lruvec = folio_lruvec(folio);
1296 
1297 	spin_lock(&lruvec->lru_lock);
1298 	lruvec_memcg_debug(lruvec, folio);
1299 
1300 	return lruvec;
1301 }
1302 
1303 /**
1304  * folio_lruvec_lock_irq - Lock the lruvec for a folio.
1305  * @folio: Pointer to the folio.
1306  *
1307  * These functions are safe to use under any of the following conditions:
1308  * - folio locked
1309  * - folio_test_lru false
1310  * - folio_memcg_lock()
1311  * - folio frozen (refcount of 0)
1312  *
1313  * Return: The lruvec this folio is on with its lock held and interrupts
1314  * disabled.
1315  */
folio_lruvec_lock_irq(struct folio * folio)1316 struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
1317 {
1318 	struct lruvec *lruvec = folio_lruvec(folio);
1319 
1320 	spin_lock_irq(&lruvec->lru_lock);
1321 	lruvec_memcg_debug(lruvec, folio);
1322 
1323 	return lruvec;
1324 }
1325 
1326 /**
1327  * folio_lruvec_lock_irqsave - Lock the lruvec for a folio.
1328  * @folio: Pointer to the folio.
1329  * @flags: Pointer to irqsave flags.
1330  *
1331  * These functions are safe to use under any of the following conditions:
1332  * - folio locked
1333  * - folio_test_lru false
1334  * - folio_memcg_lock()
1335  * - folio frozen (refcount of 0)
1336  *
1337  * Return: The lruvec this folio is on with its lock held and interrupts
1338  * disabled.
1339  */
folio_lruvec_lock_irqsave(struct folio * folio,unsigned long * flags)1340 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
1341 		unsigned long *flags)
1342 {
1343 	struct lruvec *lruvec = folio_lruvec(folio);
1344 
1345 	spin_lock_irqsave(&lruvec->lru_lock, *flags);
1346 	lruvec_memcg_debug(lruvec, folio);
1347 
1348 	return lruvec;
1349 }
1350 
1351 /**
1352  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1353  * @lruvec: mem_cgroup per zone lru vector
1354  * @lru: index of lru list the page is sitting on
1355  * @zid: zone id of the accounted pages
1356  * @nr_pages: positive when adding or negative when removing
1357  *
1358  * This function must be called under lru_lock, just before a page is added
1359  * to or just after a page is removed from an lru list.
1360  */
mem_cgroup_update_lru_size(struct lruvec * lruvec,enum lru_list lru,int zid,int nr_pages)1361 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1362 				int zid, int nr_pages)
1363 {
1364 	struct mem_cgroup_per_node *mz;
1365 	unsigned long *lru_size;
1366 	long size;
1367 
1368 	if (mem_cgroup_disabled())
1369 		return;
1370 
1371 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1372 	lru_size = &mz->lru_zone_size[zid][lru];
1373 
1374 	if (nr_pages < 0)
1375 		*lru_size += nr_pages;
1376 
1377 	size = *lru_size;
1378 	if (WARN_ONCE(size < 0,
1379 		"%s(%p, %d, %d): lru_size %ld\n",
1380 		__func__, lruvec, lru, nr_pages, size)) {
1381 		VM_BUG_ON(1);
1382 		*lru_size = 0;
1383 	}
1384 
1385 	if (nr_pages > 0)
1386 		*lru_size += nr_pages;
1387 }
1388 
1389 /**
1390  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1391  * @memcg: the memory cgroup
1392  *
1393  * Returns the maximum amount of memory @mem can be charged with, in
1394  * pages.
1395  */
mem_cgroup_margin(struct mem_cgroup * memcg)1396 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1397 {
1398 	unsigned long margin = 0;
1399 	unsigned long count;
1400 	unsigned long limit;
1401 
1402 	count = page_counter_read(&memcg->memory);
1403 	limit = READ_ONCE(memcg->memory.max);
1404 	if (count < limit)
1405 		margin = limit - count;
1406 
1407 	if (do_memsw_account()) {
1408 		count = page_counter_read(&memcg->memsw);
1409 		limit = READ_ONCE(memcg->memsw.max);
1410 		if (count < limit)
1411 			margin = min(margin, limit - count);
1412 		else
1413 			margin = 0;
1414 	}
1415 
1416 	return margin;
1417 }
1418 
1419 /*
1420  * A routine for checking "mem" is under move_account() or not.
1421  *
1422  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1423  * moving cgroups. This is for waiting at high-memory pressure
1424  * caused by "move".
1425  */
mem_cgroup_under_move(struct mem_cgroup * memcg)1426 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1427 {
1428 	struct mem_cgroup *from;
1429 	struct mem_cgroup *to;
1430 	bool ret = false;
1431 	/*
1432 	 * Unlike task_move routines, we access mc.to, mc.from not under
1433 	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1434 	 */
1435 	spin_lock(&mc.lock);
1436 	from = mc.from;
1437 	to = mc.to;
1438 	if (!from)
1439 		goto unlock;
1440 
1441 	ret = mem_cgroup_is_descendant(from, memcg) ||
1442 		mem_cgroup_is_descendant(to, memcg);
1443 unlock:
1444 	spin_unlock(&mc.lock);
1445 	return ret;
1446 }
1447 
mem_cgroup_wait_acct_move(struct mem_cgroup * memcg)1448 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1449 {
1450 	if (mc.moving_task && current != mc.moving_task) {
1451 		if (mem_cgroup_under_move(memcg)) {
1452 			DEFINE_WAIT(wait);
1453 			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1454 			/* moving charge context might have finished. */
1455 			if (mc.moving_task)
1456 				schedule();
1457 			finish_wait(&mc.waitq, &wait);
1458 			return true;
1459 		}
1460 	}
1461 	return false;
1462 }
1463 
1464 struct memory_stat {
1465 	const char *name;
1466 	unsigned int idx;
1467 };
1468 
1469 static const struct memory_stat memory_stats[] = {
1470 	{ "anon",			NR_ANON_MAPPED			},
1471 	{ "file",			NR_FILE_PAGES			},
1472 	{ "kernel",			MEMCG_KMEM			},
1473 	{ "kernel_stack",		NR_KERNEL_STACK_KB		},
1474 	{ "pagetables",			NR_PAGETABLE			},
1475 	{ "sec_pagetables",		NR_SECONDARY_PAGETABLE		},
1476 	{ "percpu",			MEMCG_PERCPU_B			},
1477 	{ "sock",			MEMCG_SOCK			},
1478 	{ "vmalloc",			MEMCG_VMALLOC			},
1479 	{ "shmem",			NR_SHMEM			},
1480 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
1481 	{ "zswap",			MEMCG_ZSWAP_B			},
1482 	{ "zswapped",			MEMCG_ZSWAPPED			},
1483 #endif
1484 	{ "file_mapped",		NR_FILE_MAPPED			},
1485 	{ "file_dirty",			NR_FILE_DIRTY			},
1486 	{ "file_writeback",		NR_WRITEBACK			},
1487 #ifdef CONFIG_SWAP
1488 	{ "swapcached",			NR_SWAPCACHE			},
1489 #endif
1490 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1491 	{ "anon_thp",			NR_ANON_THPS			},
1492 	{ "file_thp",			NR_FILE_THPS			},
1493 	{ "shmem_thp",			NR_SHMEM_THPS			},
1494 #endif
1495 	{ "inactive_anon",		NR_INACTIVE_ANON		},
1496 	{ "active_anon",		NR_ACTIVE_ANON			},
1497 	{ "inactive_file",		NR_INACTIVE_FILE		},
1498 	{ "active_file",		NR_ACTIVE_FILE			},
1499 	{ "unevictable",		NR_UNEVICTABLE			},
1500 	{ "slab_reclaimable",		NR_SLAB_RECLAIMABLE_B		},
1501 	{ "slab_unreclaimable",		NR_SLAB_UNRECLAIMABLE_B		},
1502 
1503 	/* The memory events */
1504 	{ "workingset_refault_anon",	WORKINGSET_REFAULT_ANON		},
1505 	{ "workingset_refault_file",	WORKINGSET_REFAULT_FILE		},
1506 	{ "workingset_activate_anon",	WORKINGSET_ACTIVATE_ANON	},
1507 	{ "workingset_activate_file",	WORKINGSET_ACTIVATE_FILE	},
1508 	{ "workingset_restore_anon",	WORKINGSET_RESTORE_ANON		},
1509 	{ "workingset_restore_file",	WORKINGSET_RESTORE_FILE		},
1510 	{ "workingset_nodereclaim",	WORKINGSET_NODERECLAIM		},
1511 };
1512 
1513 /* Translate stat items to the correct unit for memory.stat output */
memcg_page_state_unit(int item)1514 static int memcg_page_state_unit(int item)
1515 {
1516 	switch (item) {
1517 	case MEMCG_PERCPU_B:
1518 	case MEMCG_ZSWAP_B:
1519 	case NR_SLAB_RECLAIMABLE_B:
1520 	case NR_SLAB_UNRECLAIMABLE_B:
1521 	case WORKINGSET_REFAULT_ANON:
1522 	case WORKINGSET_REFAULT_FILE:
1523 	case WORKINGSET_ACTIVATE_ANON:
1524 	case WORKINGSET_ACTIVATE_FILE:
1525 	case WORKINGSET_RESTORE_ANON:
1526 	case WORKINGSET_RESTORE_FILE:
1527 	case WORKINGSET_NODERECLAIM:
1528 		return 1;
1529 	case NR_KERNEL_STACK_KB:
1530 		return SZ_1K;
1531 	default:
1532 		return PAGE_SIZE;
1533 	}
1534 }
1535 
memcg_page_state_output(struct mem_cgroup * memcg,int item)1536 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1537 						    int item)
1538 {
1539 	return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
1540 }
1541 
memory_stat_format(struct mem_cgroup * memcg,char * buf,int bufsize)1542 static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize)
1543 {
1544 	struct seq_buf s;
1545 	int i;
1546 
1547 	seq_buf_init(&s, buf, bufsize);
1548 
1549 	/*
1550 	 * Provide statistics on the state of the memory subsystem as
1551 	 * well as cumulative event counters that show past behavior.
1552 	 *
1553 	 * This list is ordered following a combination of these gradients:
1554 	 * 1) generic big picture -> specifics and details
1555 	 * 2) reflecting userspace activity -> reflecting kernel heuristics
1556 	 *
1557 	 * Current memory state:
1558 	 */
1559 	mem_cgroup_flush_stats();
1560 
1561 	for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
1562 		u64 size;
1563 
1564 		size = memcg_page_state_output(memcg, memory_stats[i].idx);
1565 		seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size);
1566 
1567 		if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
1568 			size += memcg_page_state_output(memcg,
1569 							NR_SLAB_RECLAIMABLE_B);
1570 			seq_buf_printf(&s, "slab %llu\n", size);
1571 		}
1572 	}
1573 
1574 	/* Accumulated memory events */
1575 	seq_buf_printf(&s, "pgscan %lu\n",
1576 		       memcg_events(memcg, PGSCAN_KSWAPD) +
1577 		       memcg_events(memcg, PGSCAN_DIRECT));
1578 	seq_buf_printf(&s, "pgsteal %lu\n",
1579 		       memcg_events(memcg, PGSTEAL_KSWAPD) +
1580 		       memcg_events(memcg, PGSTEAL_DIRECT));
1581 
1582 	for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {
1583 		if (memcg_vm_event_stat[i] == PGPGIN ||
1584 		    memcg_vm_event_stat[i] == PGPGOUT)
1585 			continue;
1586 
1587 		seq_buf_printf(&s, "%s %lu\n",
1588 			       vm_event_name(memcg_vm_event_stat[i]),
1589 			       memcg_events(memcg, memcg_vm_event_stat[i]));
1590 	}
1591 
1592 	/* The above should easily fit into one page */
1593 	WARN_ON_ONCE(seq_buf_has_overflowed(&s));
1594 }
1595 
1596 #define K(x) ((x) << (PAGE_SHIFT-10))
1597 /**
1598  * mem_cgroup_print_oom_context: Print OOM information relevant to
1599  * memory controller.
1600  * @memcg: The memory cgroup that went over limit
1601  * @p: Task that is going to be killed
1602  *
1603  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1604  * enabled
1605  */
mem_cgroup_print_oom_context(struct mem_cgroup * memcg,struct task_struct * p)1606 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1607 {
1608 	rcu_read_lock();
1609 
1610 	if (memcg) {
1611 		pr_cont(",oom_memcg=");
1612 		pr_cont_cgroup_path(memcg->css.cgroup);
1613 	} else
1614 		pr_cont(",global_oom");
1615 	if (p) {
1616 		pr_cont(",task_memcg=");
1617 		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1618 	}
1619 	rcu_read_unlock();
1620 }
1621 
1622 /**
1623  * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1624  * memory controller.
1625  * @memcg: The memory cgroup that went over limit
1626  */
mem_cgroup_print_oom_meminfo(struct mem_cgroup * memcg)1627 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1628 {
1629 	/* Use static buffer, for the caller is holding oom_lock. */
1630 	static char buf[PAGE_SIZE];
1631 
1632 	lockdep_assert_held(&oom_lock);
1633 
1634 	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1635 		K((u64)page_counter_read(&memcg->memory)),
1636 		K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1637 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1638 		pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1639 			K((u64)page_counter_read(&memcg->swap)),
1640 			K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1641 	else {
1642 		pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1643 			K((u64)page_counter_read(&memcg->memsw)),
1644 			K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1645 		pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1646 			K((u64)page_counter_read(&memcg->kmem)),
1647 			K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1648 	}
1649 
1650 	pr_info("Memory cgroup stats for ");
1651 	pr_cont_cgroup_path(memcg->css.cgroup);
1652 	pr_cont(":");
1653 	memory_stat_format(memcg, buf, sizeof(buf));
1654 	pr_info("%s", buf);
1655 }
1656 
1657 /*
1658  * Return the memory (and swap, if configured) limit for a memcg.
1659  */
mem_cgroup_get_max(struct mem_cgroup * memcg)1660 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1661 {
1662 	unsigned long max = READ_ONCE(memcg->memory.max);
1663 
1664 	if (do_memsw_account()) {
1665 		if (mem_cgroup_swappiness(memcg)) {
1666 			/* Calculate swap excess capacity from memsw limit */
1667 			unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
1668 
1669 			max += min(swap, (unsigned long)total_swap_pages);
1670 		}
1671 	} else {
1672 		if (mem_cgroup_swappiness(memcg))
1673 			max += min(READ_ONCE(memcg->swap.max),
1674 				   (unsigned long)total_swap_pages);
1675 	}
1676 	return max;
1677 }
1678 
mem_cgroup_size(struct mem_cgroup * memcg)1679 unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1680 {
1681 	return page_counter_read(&memcg->memory);
1682 }
1683 
mem_cgroup_out_of_memory(struct mem_cgroup * memcg,gfp_t gfp_mask,int order)1684 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1685 				     int order)
1686 {
1687 	struct oom_control oc = {
1688 		.zonelist = NULL,
1689 		.nodemask = NULL,
1690 		.memcg = memcg,
1691 		.gfp_mask = gfp_mask,
1692 		.order = order,
1693 	};
1694 	bool ret = true;
1695 
1696 	if (mutex_lock_killable(&oom_lock))
1697 		return true;
1698 
1699 	if (mem_cgroup_margin(memcg) >= (1 << order))
1700 		goto unlock;
1701 
1702 	/*
1703 	 * A few threads which were not waiting at mutex_lock_killable() can
1704 	 * fail to bail out. Therefore, check again after holding oom_lock.
1705 	 */
1706 	ret = task_is_dying() || out_of_memory(&oc);
1707 
1708 unlock:
1709 	mutex_unlock(&oom_lock);
1710 	return ret;
1711 }
1712 
mem_cgroup_soft_reclaim(struct mem_cgroup * root_memcg,pg_data_t * pgdat,gfp_t gfp_mask,unsigned long * total_scanned)1713 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1714 				   pg_data_t *pgdat,
1715 				   gfp_t gfp_mask,
1716 				   unsigned long *total_scanned)
1717 {
1718 	struct mem_cgroup *victim = NULL;
1719 	int total = 0;
1720 	int loop = 0;
1721 	unsigned long excess;
1722 	unsigned long nr_scanned;
1723 	struct mem_cgroup_reclaim_cookie reclaim = {
1724 		.pgdat = pgdat,
1725 	};
1726 
1727 	excess = soft_limit_excess(root_memcg);
1728 
1729 	while (1) {
1730 		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1731 		if (!victim) {
1732 			loop++;
1733 			if (loop >= 2) {
1734 				/*
1735 				 * If we have not been able to reclaim
1736 				 * anything, it might because there are
1737 				 * no reclaimable pages under this hierarchy
1738 				 */
1739 				if (!total)
1740 					break;
1741 				/*
1742 				 * We want to do more targeted reclaim.
1743 				 * excess >> 2 is not to excessive so as to
1744 				 * reclaim too much, nor too less that we keep
1745 				 * coming back to reclaim from this cgroup
1746 				 */
1747 				if (total >= (excess >> 2) ||
1748 					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1749 					break;
1750 			}
1751 			continue;
1752 		}
1753 		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1754 					pgdat, &nr_scanned);
1755 		*total_scanned += nr_scanned;
1756 		if (!soft_limit_excess(root_memcg))
1757 			break;
1758 	}
1759 	mem_cgroup_iter_break(root_memcg, victim);
1760 	return total;
1761 }
1762 
1763 #ifdef CONFIG_LOCKDEP
1764 static struct lockdep_map memcg_oom_lock_dep_map = {
1765 	.name = "memcg_oom_lock",
1766 };
1767 #endif
1768 
1769 static DEFINE_SPINLOCK(memcg_oom_lock);
1770 
1771 /*
1772  * Check OOM-Killer is already running under our hierarchy.
1773  * If someone is running, return false.
1774  */
mem_cgroup_oom_trylock(struct mem_cgroup * memcg)1775 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1776 {
1777 	struct mem_cgroup *iter, *failed = NULL;
1778 
1779 	spin_lock(&memcg_oom_lock);
1780 
1781 	for_each_mem_cgroup_tree(iter, memcg) {
1782 		if (iter->oom_lock) {
1783 			/*
1784 			 * this subtree of our hierarchy is already locked
1785 			 * so we cannot give a lock.
1786 			 */
1787 			failed = iter;
1788 			mem_cgroup_iter_break(memcg, iter);
1789 			break;
1790 		} else
1791 			iter->oom_lock = true;
1792 	}
1793 
1794 	if (failed) {
1795 		/*
1796 		 * OK, we failed to lock the whole subtree so we have
1797 		 * to clean up what we set up to the failing subtree
1798 		 */
1799 		for_each_mem_cgroup_tree(iter, memcg) {
1800 			if (iter == failed) {
1801 				mem_cgroup_iter_break(memcg, iter);
1802 				break;
1803 			}
1804 			iter->oom_lock = false;
1805 		}
1806 	} else
1807 		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1808 
1809 	spin_unlock(&memcg_oom_lock);
1810 
1811 	return !failed;
1812 }
1813 
mem_cgroup_oom_unlock(struct mem_cgroup * memcg)1814 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1815 {
1816 	struct mem_cgroup *iter;
1817 
1818 	spin_lock(&memcg_oom_lock);
1819 	mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
1820 	for_each_mem_cgroup_tree(iter, memcg)
1821 		iter->oom_lock = false;
1822 	spin_unlock(&memcg_oom_lock);
1823 }
1824 
mem_cgroup_mark_under_oom(struct mem_cgroup * memcg)1825 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1826 {
1827 	struct mem_cgroup *iter;
1828 
1829 	spin_lock(&memcg_oom_lock);
1830 	for_each_mem_cgroup_tree(iter, memcg)
1831 		iter->under_oom++;
1832 	spin_unlock(&memcg_oom_lock);
1833 }
1834 
mem_cgroup_unmark_under_oom(struct mem_cgroup * memcg)1835 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1836 {
1837 	struct mem_cgroup *iter;
1838 
1839 	/*
1840 	 * Be careful about under_oom underflows because a child memcg
1841 	 * could have been added after mem_cgroup_mark_under_oom.
1842 	 */
1843 	spin_lock(&memcg_oom_lock);
1844 	for_each_mem_cgroup_tree(iter, memcg)
1845 		if (iter->under_oom > 0)
1846 			iter->under_oom--;
1847 	spin_unlock(&memcg_oom_lock);
1848 }
1849 
1850 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1851 
1852 struct oom_wait_info {
1853 	struct mem_cgroup *memcg;
1854 	wait_queue_entry_t	wait;
1855 };
1856 
memcg_oom_wake_function(wait_queue_entry_t * wait,unsigned mode,int sync,void * arg)1857 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1858 	unsigned mode, int sync, void *arg)
1859 {
1860 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1861 	struct mem_cgroup *oom_wait_memcg;
1862 	struct oom_wait_info *oom_wait_info;
1863 
1864 	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1865 	oom_wait_memcg = oom_wait_info->memcg;
1866 
1867 	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1868 	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1869 		return 0;
1870 	return autoremove_wake_function(wait, mode, sync, arg);
1871 }
1872 
memcg_oom_recover(struct mem_cgroup * memcg)1873 static void memcg_oom_recover(struct mem_cgroup *memcg)
1874 {
1875 	/*
1876 	 * For the following lockless ->under_oom test, the only required
1877 	 * guarantee is that it must see the state asserted by an OOM when
1878 	 * this function is called as a result of userland actions
1879 	 * triggered by the notification of the OOM.  This is trivially
1880 	 * achieved by invoking mem_cgroup_mark_under_oom() before
1881 	 * triggering notification.
1882 	 */
1883 	if (memcg && memcg->under_oom)
1884 		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1885 }
1886 
1887 /*
1888  * Returns true if successfully killed one or more processes. Though in some
1889  * corner cases it can return true even without killing any process.
1890  */
mem_cgroup_oom(struct mem_cgroup * memcg,gfp_t mask,int order)1891 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1892 {
1893 	bool locked, ret;
1894 
1895 	if (order > PAGE_ALLOC_COSTLY_ORDER)
1896 		return false;
1897 
1898 	memcg_memory_event(memcg, MEMCG_OOM);
1899 
1900 	/*
1901 	 * We are in the middle of the charge context here, so we
1902 	 * don't want to block when potentially sitting on a callstack
1903 	 * that holds all kinds of filesystem and mm locks.
1904 	 *
1905 	 * cgroup1 allows disabling the OOM killer and waiting for outside
1906 	 * handling until the charge can succeed; remember the context and put
1907 	 * the task to sleep at the end of the page fault when all locks are
1908 	 * released.
1909 	 *
1910 	 * On the other hand, in-kernel OOM killer allows for an async victim
1911 	 * memory reclaim (oom_reaper) and that means that we are not solely
1912 	 * relying on the oom victim to make a forward progress and we can
1913 	 * invoke the oom killer here.
1914 	 *
1915 	 * Please note that mem_cgroup_out_of_memory might fail to find a
1916 	 * victim and then we have to bail out from the charge path.
1917 	 */
1918 	if (memcg->oom_kill_disable) {
1919 		if (current->in_user_fault) {
1920 			css_get(&memcg->css);
1921 			current->memcg_in_oom = memcg;
1922 			current->memcg_oom_gfp_mask = mask;
1923 			current->memcg_oom_order = order;
1924 		}
1925 		return false;
1926 	}
1927 
1928 	mem_cgroup_mark_under_oom(memcg);
1929 
1930 	locked = mem_cgroup_oom_trylock(memcg);
1931 
1932 	if (locked)
1933 		mem_cgroup_oom_notify(memcg);
1934 
1935 	mem_cgroup_unmark_under_oom(memcg);
1936 	ret = mem_cgroup_out_of_memory(memcg, mask, order);
1937 
1938 	if (locked)
1939 		mem_cgroup_oom_unlock(memcg);
1940 
1941 	return ret;
1942 }
1943 
1944 /**
1945  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1946  * @handle: actually kill/wait or just clean up the OOM state
1947  *
1948  * This has to be called at the end of a page fault if the memcg OOM
1949  * handler was enabled.
1950  *
1951  * Memcg supports userspace OOM handling where failed allocations must
1952  * sleep on a waitqueue until the userspace task resolves the
1953  * situation.  Sleeping directly in the charge context with all kinds
1954  * of locks held is not a good idea, instead we remember an OOM state
1955  * in the task and mem_cgroup_oom_synchronize() has to be called at
1956  * the end of the page fault to complete the OOM handling.
1957  *
1958  * Returns %true if an ongoing memcg OOM situation was detected and
1959  * completed, %false otherwise.
1960  */
mem_cgroup_oom_synchronize(bool handle)1961 bool mem_cgroup_oom_synchronize(bool handle)
1962 {
1963 	struct mem_cgroup *memcg = current->memcg_in_oom;
1964 	struct oom_wait_info owait;
1965 	bool locked;
1966 
1967 	/* OOM is global, do not handle */
1968 	if (!memcg)
1969 		return false;
1970 
1971 	if (!handle)
1972 		goto cleanup;
1973 
1974 	owait.memcg = memcg;
1975 	owait.wait.flags = 0;
1976 	owait.wait.func = memcg_oom_wake_function;
1977 	owait.wait.private = current;
1978 	INIT_LIST_HEAD(&owait.wait.entry);
1979 
1980 	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1981 	mem_cgroup_mark_under_oom(memcg);
1982 
1983 	locked = mem_cgroup_oom_trylock(memcg);
1984 
1985 	if (locked)
1986 		mem_cgroup_oom_notify(memcg);
1987 
1988 	if (locked && !memcg->oom_kill_disable) {
1989 		mem_cgroup_unmark_under_oom(memcg);
1990 		finish_wait(&memcg_oom_waitq, &owait.wait);
1991 		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1992 					 current->memcg_oom_order);
1993 	} else {
1994 		schedule();
1995 		mem_cgroup_unmark_under_oom(memcg);
1996 		finish_wait(&memcg_oom_waitq, &owait.wait);
1997 	}
1998 
1999 	if (locked) {
2000 		mem_cgroup_oom_unlock(memcg);
2001 		/*
2002 		 * There is no guarantee that an OOM-lock contender
2003 		 * sees the wakeups triggered by the OOM kill
2004 		 * uncharges.  Wake any sleepers explicitly.
2005 		 */
2006 		memcg_oom_recover(memcg);
2007 	}
2008 cleanup:
2009 	current->memcg_in_oom = NULL;
2010 	css_put(&memcg->css);
2011 	return true;
2012 }
2013 
2014 /**
2015  * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
2016  * @victim: task to be killed by the OOM killer
2017  * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2018  *
2019  * Returns a pointer to a memory cgroup, which has to be cleaned up
2020  * by killing all belonging OOM-killable tasks.
2021  *
2022  * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2023  */
mem_cgroup_get_oom_group(struct task_struct * victim,struct mem_cgroup * oom_domain)2024 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
2025 					    struct mem_cgroup *oom_domain)
2026 {
2027 	struct mem_cgroup *oom_group = NULL;
2028 	struct mem_cgroup *memcg;
2029 
2030 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2031 		return NULL;
2032 
2033 	if (!oom_domain)
2034 		oom_domain = root_mem_cgroup;
2035 
2036 	rcu_read_lock();
2037 
2038 	memcg = mem_cgroup_from_task(victim);
2039 	if (memcg == root_mem_cgroup)
2040 		goto out;
2041 
2042 	/*
2043 	 * If the victim task has been asynchronously moved to a different
2044 	 * memory cgroup, we might end up killing tasks outside oom_domain.
2045 	 * In this case it's better to ignore memory.group.oom.
2046 	 */
2047 	if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2048 		goto out;
2049 
2050 	/*
2051 	 * Traverse the memory cgroup hierarchy from the victim task's
2052 	 * cgroup up to the OOMing cgroup (or root) to find the
2053 	 * highest-level memory cgroup with oom.group set.
2054 	 */
2055 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2056 		if (memcg->oom_group)
2057 			oom_group = memcg;
2058 
2059 		if (memcg == oom_domain)
2060 			break;
2061 	}
2062 
2063 	if (oom_group)
2064 		css_get(&oom_group->css);
2065 out:
2066 	rcu_read_unlock();
2067 
2068 	return oom_group;
2069 }
2070 
mem_cgroup_print_oom_group(struct mem_cgroup * memcg)2071 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2072 {
2073 	pr_info("Tasks in ");
2074 	pr_cont_cgroup_path(memcg->css.cgroup);
2075 	pr_cont(" are going to be killed due to memory.oom.group set\n");
2076 }
2077 
2078 /**
2079  * folio_memcg_lock - Bind a folio to its memcg.
2080  * @folio: The folio.
2081  *
2082  * This function prevents unlocked LRU folios from being moved to
2083  * another cgroup.
2084  *
2085  * It ensures lifetime of the bound memcg.  The caller is responsible
2086  * for the lifetime of the folio.
2087  */
folio_memcg_lock(struct folio * folio)2088 void folio_memcg_lock(struct folio *folio)
2089 {
2090 	struct mem_cgroup *memcg;
2091 	unsigned long flags;
2092 
2093 	/*
2094 	 * The RCU lock is held throughout the transaction.  The fast
2095 	 * path can get away without acquiring the memcg->move_lock
2096 	 * because page moving starts with an RCU grace period.
2097          */
2098 	rcu_read_lock();
2099 
2100 	if (mem_cgroup_disabled())
2101 		return;
2102 again:
2103 	memcg = folio_memcg(folio);
2104 	if (unlikely(!memcg))
2105 		return;
2106 
2107 #ifdef CONFIG_PROVE_LOCKING
2108 	local_irq_save(flags);
2109 	might_lock(&memcg->move_lock);
2110 	local_irq_restore(flags);
2111 #endif
2112 
2113 	if (atomic_read(&memcg->moving_account) <= 0)
2114 		return;
2115 
2116 	spin_lock_irqsave(&memcg->move_lock, flags);
2117 	if (memcg != folio_memcg(folio)) {
2118 		spin_unlock_irqrestore(&memcg->move_lock, flags);
2119 		goto again;
2120 	}
2121 
2122 	/*
2123 	 * When charge migration first begins, we can have multiple
2124 	 * critical sections holding the fast-path RCU lock and one
2125 	 * holding the slowpath move_lock. Track the task who has the
2126 	 * move_lock for unlock_page_memcg().
2127 	 */
2128 	memcg->move_lock_task = current;
2129 	memcg->move_lock_flags = flags;
2130 }
2131 
lock_page_memcg(struct page * page)2132 void lock_page_memcg(struct page *page)
2133 {
2134 	folio_memcg_lock(page_folio(page));
2135 }
2136 
__folio_memcg_unlock(struct mem_cgroup * memcg)2137 static void __folio_memcg_unlock(struct mem_cgroup *memcg)
2138 {
2139 	if (memcg && memcg->move_lock_task == current) {
2140 		unsigned long flags = memcg->move_lock_flags;
2141 
2142 		memcg->move_lock_task = NULL;
2143 		memcg->move_lock_flags = 0;
2144 
2145 		spin_unlock_irqrestore(&memcg->move_lock, flags);
2146 	}
2147 
2148 	rcu_read_unlock();
2149 }
2150 
2151 /**
2152  * folio_memcg_unlock - Release the binding between a folio and its memcg.
2153  * @folio: The folio.
2154  *
2155  * This releases the binding created by folio_memcg_lock().  This does
2156  * not change the accounting of this folio to its memcg, but it does
2157  * permit others to change it.
2158  */
folio_memcg_unlock(struct folio * folio)2159 void folio_memcg_unlock(struct folio *folio)
2160 {
2161 	__folio_memcg_unlock(folio_memcg(folio));
2162 }
2163 
unlock_page_memcg(struct page * page)2164 void unlock_page_memcg(struct page *page)
2165 {
2166 	folio_memcg_unlock(page_folio(page));
2167 }
2168 
2169 struct memcg_stock_pcp {
2170 	local_lock_t stock_lock;
2171 	struct mem_cgroup *cached; /* this never be root cgroup */
2172 	unsigned int nr_pages;
2173 
2174 #ifdef CONFIG_MEMCG_KMEM
2175 	struct obj_cgroup *cached_objcg;
2176 	struct pglist_data *cached_pgdat;
2177 	unsigned int nr_bytes;
2178 	int nr_slab_reclaimable_b;
2179 	int nr_slab_unreclaimable_b;
2180 #endif
2181 
2182 	struct work_struct work;
2183 	unsigned long flags;
2184 #define FLUSHING_CACHED_CHARGE	0
2185 };
2186 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock) = {
2187 	.stock_lock = INIT_LOCAL_LOCK(stock_lock),
2188 };
2189 static DEFINE_MUTEX(percpu_charge_mutex);
2190 
2191 #ifdef CONFIG_MEMCG_KMEM
2192 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock);
2193 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2194 				     struct mem_cgroup *root_memcg);
2195 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages);
2196 
2197 #else
drain_obj_stock(struct memcg_stock_pcp * stock)2198 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock)
2199 {
2200 	return NULL;
2201 }
obj_stock_flush_required(struct memcg_stock_pcp * stock,struct mem_cgroup * root_memcg)2202 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2203 				     struct mem_cgroup *root_memcg)
2204 {
2205 	return false;
2206 }
memcg_account_kmem(struct mem_cgroup * memcg,int nr_pages)2207 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
2208 {
2209 }
2210 #endif
2211 
2212 /**
2213  * consume_stock: Try to consume stocked charge on this cpu.
2214  * @memcg: memcg to consume from.
2215  * @nr_pages: how many pages to charge.
2216  *
2217  * The charges will only happen if @memcg matches the current cpu's memcg
2218  * stock, and at least @nr_pages are available in that stock.  Failure to
2219  * service an allocation will refill the stock.
2220  *
2221  * returns true if successful, false otherwise.
2222  */
consume_stock(struct mem_cgroup * memcg,unsigned int nr_pages)2223 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2224 {
2225 	struct memcg_stock_pcp *stock;
2226 	unsigned long flags;
2227 	bool ret = false;
2228 
2229 	if (nr_pages > MEMCG_CHARGE_BATCH)
2230 		return ret;
2231 
2232 	local_lock_irqsave(&memcg_stock.stock_lock, flags);
2233 
2234 	stock = this_cpu_ptr(&memcg_stock);
2235 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2236 		stock->nr_pages -= nr_pages;
2237 		ret = true;
2238 	}
2239 
2240 	local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
2241 
2242 	return ret;
2243 }
2244 
2245 /*
2246  * Returns stocks cached in percpu and reset cached information.
2247  */
drain_stock(struct memcg_stock_pcp * stock)2248 static void drain_stock(struct memcg_stock_pcp *stock)
2249 {
2250 	struct mem_cgroup *old = stock->cached;
2251 
2252 	if (!old)
2253 		return;
2254 
2255 	if (stock->nr_pages) {
2256 		page_counter_uncharge(&old->memory, stock->nr_pages);
2257 		if (do_memsw_account())
2258 			page_counter_uncharge(&old->memsw, stock->nr_pages);
2259 		stock->nr_pages = 0;
2260 	}
2261 
2262 	css_put(&old->css);
2263 	stock->cached = NULL;
2264 }
2265 
drain_local_stock(struct work_struct * dummy)2266 static void drain_local_stock(struct work_struct *dummy)
2267 {
2268 	struct memcg_stock_pcp *stock;
2269 	struct obj_cgroup *old = NULL;
2270 	unsigned long flags;
2271 
2272 	/*
2273 	 * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs.
2274 	 * drain_stock races is that we always operate on local CPU stock
2275 	 * here with IRQ disabled
2276 	 */
2277 	local_lock_irqsave(&memcg_stock.stock_lock, flags);
2278 
2279 	stock = this_cpu_ptr(&memcg_stock);
2280 	old = drain_obj_stock(stock);
2281 	drain_stock(stock);
2282 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2283 
2284 	local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
2285 	if (old)
2286 		obj_cgroup_put(old);
2287 }
2288 
2289 /*
2290  * Cache charges(val) to local per_cpu area.
2291  * This will be consumed by consume_stock() function, later.
2292  */
__refill_stock(struct mem_cgroup * memcg,unsigned int nr_pages)2293 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2294 {
2295 	struct memcg_stock_pcp *stock;
2296 
2297 	stock = this_cpu_ptr(&memcg_stock);
2298 	if (stock->cached != memcg) { /* reset if necessary */
2299 		drain_stock(stock);
2300 		css_get(&memcg->css);
2301 		stock->cached = memcg;
2302 	}
2303 	stock->nr_pages += nr_pages;
2304 
2305 	if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2306 		drain_stock(stock);
2307 }
2308 
refill_stock(struct mem_cgroup * memcg,unsigned int nr_pages)2309 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2310 {
2311 	unsigned long flags;
2312 
2313 	local_lock_irqsave(&memcg_stock.stock_lock, flags);
2314 	__refill_stock(memcg, nr_pages);
2315 	local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
2316 }
2317 
2318 /*
2319  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2320  * of the hierarchy under it.
2321  */
drain_all_stock(struct mem_cgroup * root_memcg)2322 static void drain_all_stock(struct mem_cgroup *root_memcg)
2323 {
2324 	int cpu, curcpu;
2325 
2326 	/* If someone's already draining, avoid adding running more workers. */
2327 	if (!mutex_trylock(&percpu_charge_mutex))
2328 		return;
2329 	/*
2330 	 * Notify other cpus that system-wide "drain" is running
2331 	 * We do not care about races with the cpu hotplug because cpu down
2332 	 * as well as workers from this path always operate on the local
2333 	 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2334 	 */
2335 	migrate_disable();
2336 	curcpu = smp_processor_id();
2337 	for_each_online_cpu(cpu) {
2338 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2339 		struct mem_cgroup *memcg;
2340 		bool flush = false;
2341 
2342 		rcu_read_lock();
2343 		memcg = stock->cached;
2344 		if (memcg && stock->nr_pages &&
2345 		    mem_cgroup_is_descendant(memcg, root_memcg))
2346 			flush = true;
2347 		else if (obj_stock_flush_required(stock, root_memcg))
2348 			flush = true;
2349 		rcu_read_unlock();
2350 
2351 		if (flush &&
2352 		    !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2353 			if (cpu == curcpu)
2354 				drain_local_stock(&stock->work);
2355 			else
2356 				schedule_work_on(cpu, &stock->work);
2357 		}
2358 	}
2359 	migrate_enable();
2360 	mutex_unlock(&percpu_charge_mutex);
2361 }
2362 
memcg_hotplug_cpu_dead(unsigned int cpu)2363 static int memcg_hotplug_cpu_dead(unsigned int cpu)
2364 {
2365 	struct memcg_stock_pcp *stock;
2366 
2367 	stock = &per_cpu(memcg_stock, cpu);
2368 	drain_stock(stock);
2369 
2370 	return 0;
2371 }
2372 
reclaim_high(struct mem_cgroup * memcg,unsigned int nr_pages,gfp_t gfp_mask)2373 static unsigned long reclaim_high(struct mem_cgroup *memcg,
2374 				  unsigned int nr_pages,
2375 				  gfp_t gfp_mask)
2376 {
2377 	unsigned long nr_reclaimed = 0;
2378 
2379 	do {
2380 		unsigned long pflags;
2381 
2382 		if (page_counter_read(&memcg->memory) <=
2383 		    READ_ONCE(memcg->memory.high))
2384 			continue;
2385 
2386 		memcg_memory_event(memcg, MEMCG_HIGH);
2387 
2388 		psi_memstall_enter(&pflags);
2389 		nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2390 							gfp_mask,
2391 							MEMCG_RECLAIM_MAY_SWAP);
2392 		psi_memstall_leave(&pflags);
2393 	} while ((memcg = parent_mem_cgroup(memcg)) &&
2394 		 !mem_cgroup_is_root(memcg));
2395 
2396 	return nr_reclaimed;
2397 }
2398 
high_work_func(struct work_struct * work)2399 static void high_work_func(struct work_struct *work)
2400 {
2401 	struct mem_cgroup *memcg;
2402 
2403 	memcg = container_of(work, struct mem_cgroup, high_work);
2404 	reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2405 }
2406 
2407 /*
2408  * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
2409  * enough to still cause a significant slowdown in most cases, while still
2410  * allowing diagnostics and tracing to proceed without becoming stuck.
2411  */
2412 #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2413 
2414 /*
2415  * When calculating the delay, we use these either side of the exponentiation to
2416  * maintain precision and scale to a reasonable number of jiffies (see the table
2417  * below.
2418  *
2419  * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2420  *   overage ratio to a delay.
2421  * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
2422  *   proposed penalty in order to reduce to a reasonable number of jiffies, and
2423  *   to produce a reasonable delay curve.
2424  *
2425  * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
2426  * reasonable delay curve compared to precision-adjusted overage, not
2427  * penalising heavily at first, but still making sure that growth beyond the
2428  * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2429  * example, with a high of 100 megabytes:
2430  *
2431  *  +-------+------------------------+
2432  *  | usage | time to allocate in ms |
2433  *  +-------+------------------------+
2434  *  | 100M  |                      0 |
2435  *  | 101M  |                      6 |
2436  *  | 102M  |                     25 |
2437  *  | 103M  |                     57 |
2438  *  | 104M  |                    102 |
2439  *  | 105M  |                    159 |
2440  *  | 106M  |                    230 |
2441  *  | 107M  |                    313 |
2442  *  | 108M  |                    409 |
2443  *  | 109M  |                    518 |
2444  *  | 110M  |                    639 |
2445  *  | 111M  |                    774 |
2446  *  | 112M  |                    921 |
2447  *  | 113M  |                   1081 |
2448  *  | 114M  |                   1254 |
2449  *  | 115M  |                   1439 |
2450  *  | 116M  |                   1638 |
2451  *  | 117M  |                   1849 |
2452  *  | 118M  |                   2000 |
2453  *  | 119M  |                   2000 |
2454  *  | 120M  |                   2000 |
2455  *  +-------+------------------------+
2456  */
2457  #define MEMCG_DELAY_PRECISION_SHIFT 20
2458  #define MEMCG_DELAY_SCALING_SHIFT 14
2459 
calculate_overage(unsigned long usage,unsigned long high)2460 static u64 calculate_overage(unsigned long usage, unsigned long high)
2461 {
2462 	u64 overage;
2463 
2464 	if (usage <= high)
2465 		return 0;
2466 
2467 	/*
2468 	 * Prevent division by 0 in overage calculation by acting as if
2469 	 * it was a threshold of 1 page
2470 	 */
2471 	high = max(high, 1UL);
2472 
2473 	overage = usage - high;
2474 	overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2475 	return div64_u64(overage, high);
2476 }
2477 
mem_find_max_overage(struct mem_cgroup * memcg)2478 static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2479 {
2480 	u64 overage, max_overage = 0;
2481 
2482 	do {
2483 		overage = calculate_overage(page_counter_read(&memcg->memory),
2484 					    READ_ONCE(memcg->memory.high));
2485 		max_overage = max(overage, max_overage);
2486 	} while ((memcg = parent_mem_cgroup(memcg)) &&
2487 		 !mem_cgroup_is_root(memcg));
2488 
2489 	return max_overage;
2490 }
2491 
swap_find_max_overage(struct mem_cgroup * memcg)2492 static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2493 {
2494 	u64 overage, max_overage = 0;
2495 
2496 	do {
2497 		overage = calculate_overage(page_counter_read(&memcg->swap),
2498 					    READ_ONCE(memcg->swap.high));
2499 		if (overage)
2500 			memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2501 		max_overage = max(overage, max_overage);
2502 	} while ((memcg = parent_mem_cgroup(memcg)) &&
2503 		 !mem_cgroup_is_root(memcg));
2504 
2505 	return max_overage;
2506 }
2507 
2508 /*
2509  * Get the number of jiffies that we should penalise a mischievous cgroup which
2510  * is exceeding its memory.high by checking both it and its ancestors.
2511  */
calculate_high_delay(struct mem_cgroup * memcg,unsigned int nr_pages,u64 max_overage)2512 static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2513 					  unsigned int nr_pages,
2514 					  u64 max_overage)
2515 {
2516 	unsigned long penalty_jiffies;
2517 
2518 	if (!max_overage)
2519 		return 0;
2520 
2521 	/*
2522 	 * We use overage compared to memory.high to calculate the number of
2523 	 * jiffies to sleep (penalty_jiffies). Ideally this value should be
2524 	 * fairly lenient on small overages, and increasingly harsh when the
2525 	 * memcg in question makes it clear that it has no intention of stopping
2526 	 * its crazy behaviour, so we exponentially increase the delay based on
2527 	 * overage amount.
2528 	 */
2529 	penalty_jiffies = max_overage * max_overage * HZ;
2530 	penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2531 	penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
2532 
2533 	/*
2534 	 * Factor in the task's own contribution to the overage, such that four
2535 	 * N-sized allocations are throttled approximately the same as one
2536 	 * 4N-sized allocation.
2537 	 *
2538 	 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
2539 	 * larger the current charge patch is than that.
2540 	 */
2541 	return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
2542 }
2543 
2544 /*
2545  * Scheduled by try_charge() to be executed from the userland return path
2546  * and reclaims memory over the high limit.
2547  */
mem_cgroup_handle_over_high(void)2548 void mem_cgroup_handle_over_high(void)
2549 {
2550 	unsigned long penalty_jiffies;
2551 	unsigned long pflags;
2552 	unsigned long nr_reclaimed;
2553 	unsigned int nr_pages = current->memcg_nr_pages_over_high;
2554 	int nr_retries = MAX_RECLAIM_RETRIES;
2555 	struct mem_cgroup *memcg;
2556 	bool in_retry = false;
2557 
2558 	if (likely(!nr_pages))
2559 		return;
2560 
2561 	memcg = get_mem_cgroup_from_mm(current->mm);
2562 	current->memcg_nr_pages_over_high = 0;
2563 
2564 retry_reclaim:
2565 	/*
2566 	 * The allocating task should reclaim at least the batch size, but for
2567 	 * subsequent retries we only want to do what's necessary to prevent oom
2568 	 * or breaching resource isolation.
2569 	 *
2570 	 * This is distinct from memory.max or page allocator behaviour because
2571 	 * memory.high is currently batched, whereas memory.max and the page
2572 	 * allocator run every time an allocation is made.
2573 	 */
2574 	nr_reclaimed = reclaim_high(memcg,
2575 				    in_retry ? SWAP_CLUSTER_MAX : nr_pages,
2576 				    GFP_KERNEL);
2577 
2578 	/*
2579 	 * memory.high is breached and reclaim is unable to keep up. Throttle
2580 	 * allocators proactively to slow down excessive growth.
2581 	 */
2582 	penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2583 					       mem_find_max_overage(memcg));
2584 
2585 	penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2586 						swap_find_max_overage(memcg));
2587 
2588 	/*
2589 	 * Clamp the max delay per usermode return so as to still keep the
2590 	 * application moving forwards and also permit diagnostics, albeit
2591 	 * extremely slowly.
2592 	 */
2593 	penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2594 
2595 	/*
2596 	 * Don't sleep if the amount of jiffies this memcg owes us is so low
2597 	 * that it's not even worth doing, in an attempt to be nice to those who
2598 	 * go only a small amount over their memory.high value and maybe haven't
2599 	 * been aggressively reclaimed enough yet.
2600 	 */
2601 	if (penalty_jiffies <= HZ / 100)
2602 		goto out;
2603 
2604 	/*
2605 	 * If reclaim is making forward progress but we're still over
2606 	 * memory.high, we want to encourage that rather than doing allocator
2607 	 * throttling.
2608 	 */
2609 	if (nr_reclaimed || nr_retries--) {
2610 		in_retry = true;
2611 		goto retry_reclaim;
2612 	}
2613 
2614 	/*
2615 	 * If we exit early, we're guaranteed to die (since
2616 	 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
2617 	 * need to account for any ill-begotten jiffies to pay them off later.
2618 	 */
2619 	psi_memstall_enter(&pflags);
2620 	schedule_timeout_killable(penalty_jiffies);
2621 	psi_memstall_leave(&pflags);
2622 
2623 out:
2624 	css_put(&memcg->css);
2625 }
2626 
try_charge_memcg(struct mem_cgroup * memcg,gfp_t gfp_mask,unsigned int nr_pages)2627 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
2628 			unsigned int nr_pages)
2629 {
2630 	unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2631 	int nr_retries = MAX_RECLAIM_RETRIES;
2632 	struct mem_cgroup *mem_over_limit;
2633 	struct page_counter *counter;
2634 	unsigned long nr_reclaimed;
2635 	bool passed_oom = false;
2636 	unsigned int reclaim_options = MEMCG_RECLAIM_MAY_SWAP;
2637 	bool drained = false;
2638 	bool raised_max_event = false;
2639 	unsigned long pflags;
2640 
2641 retry:
2642 	if (consume_stock(memcg, nr_pages))
2643 		return 0;
2644 
2645 	if (!do_memsw_account() ||
2646 	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2647 		if (page_counter_try_charge(&memcg->memory, batch, &counter))
2648 			goto done_restock;
2649 		if (do_memsw_account())
2650 			page_counter_uncharge(&memcg->memsw, batch);
2651 		mem_over_limit = mem_cgroup_from_counter(counter, memory);
2652 	} else {
2653 		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2654 		reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP;
2655 	}
2656 
2657 	if (batch > nr_pages) {
2658 		batch = nr_pages;
2659 		goto retry;
2660 	}
2661 
2662 	/*
2663 	 * Prevent unbounded recursion when reclaim operations need to
2664 	 * allocate memory. This might exceed the limits temporarily,
2665 	 * but we prefer facilitating memory reclaim and getting back
2666 	 * under the limit over triggering OOM kills in these cases.
2667 	 */
2668 	if (unlikely(current->flags & PF_MEMALLOC))
2669 		goto force;
2670 
2671 	if (unlikely(task_in_memcg_oom(current)))
2672 		goto nomem;
2673 
2674 	if (!gfpflags_allow_blocking(gfp_mask))
2675 		goto nomem;
2676 
2677 	memcg_memory_event(mem_over_limit, MEMCG_MAX);
2678 	raised_max_event = true;
2679 
2680 	psi_memstall_enter(&pflags);
2681 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2682 						    gfp_mask, reclaim_options);
2683 	psi_memstall_leave(&pflags);
2684 
2685 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2686 		goto retry;
2687 
2688 	if (!drained) {
2689 		drain_all_stock(mem_over_limit);
2690 		drained = true;
2691 		goto retry;
2692 	}
2693 
2694 	if (gfp_mask & __GFP_NORETRY)
2695 		goto nomem;
2696 	/*
2697 	 * Even though the limit is exceeded at this point, reclaim
2698 	 * may have been able to free some pages.  Retry the charge
2699 	 * before killing the task.
2700 	 *
2701 	 * Only for regular pages, though: huge pages are rather
2702 	 * unlikely to succeed so close to the limit, and we fall back
2703 	 * to regular pages anyway in case of failure.
2704 	 */
2705 	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2706 		goto retry;
2707 	/*
2708 	 * At task move, charge accounts can be doubly counted. So, it's
2709 	 * better to wait until the end of task_move if something is going on.
2710 	 */
2711 	if (mem_cgroup_wait_acct_move(mem_over_limit))
2712 		goto retry;
2713 
2714 	if (nr_retries--)
2715 		goto retry;
2716 
2717 	if (gfp_mask & __GFP_RETRY_MAYFAIL)
2718 		goto nomem;
2719 
2720 	/* Avoid endless loop for tasks bypassed by the oom killer */
2721 	if (passed_oom && task_is_dying())
2722 		goto nomem;
2723 
2724 	/*
2725 	 * keep retrying as long as the memcg oom killer is able to make
2726 	 * a forward progress or bypass the charge if the oom killer
2727 	 * couldn't make any progress.
2728 	 */
2729 	if (mem_cgroup_oom(mem_over_limit, gfp_mask,
2730 			   get_order(nr_pages * PAGE_SIZE))) {
2731 		passed_oom = true;
2732 		nr_retries = MAX_RECLAIM_RETRIES;
2733 		goto retry;
2734 	}
2735 nomem:
2736 	/*
2737 	 * Memcg doesn't have a dedicated reserve for atomic
2738 	 * allocations. But like the global atomic pool, we need to
2739 	 * put the burden of reclaim on regular allocation requests
2740 	 * and let these go through as privileged allocations.
2741 	 */
2742 	if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
2743 		return -ENOMEM;
2744 force:
2745 	/*
2746 	 * If the allocation has to be enforced, don't forget to raise
2747 	 * a MEMCG_MAX event.
2748 	 */
2749 	if (!raised_max_event)
2750 		memcg_memory_event(mem_over_limit, MEMCG_MAX);
2751 
2752 	/*
2753 	 * The allocation either can't fail or will lead to more memory
2754 	 * being freed very soon.  Allow memory usage go over the limit
2755 	 * temporarily by force charging it.
2756 	 */
2757 	page_counter_charge(&memcg->memory, nr_pages);
2758 	if (do_memsw_account())
2759 		page_counter_charge(&memcg->memsw, nr_pages);
2760 
2761 	return 0;
2762 
2763 done_restock:
2764 	if (batch > nr_pages)
2765 		refill_stock(memcg, batch - nr_pages);
2766 
2767 	/*
2768 	 * If the hierarchy is above the normal consumption range, schedule
2769 	 * reclaim on returning to userland.  We can perform reclaim here
2770 	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2771 	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
2772 	 * not recorded as it most likely matches current's and won't
2773 	 * change in the meantime.  As high limit is checked again before
2774 	 * reclaim, the cost of mismatch is negligible.
2775 	 */
2776 	do {
2777 		bool mem_high, swap_high;
2778 
2779 		mem_high = page_counter_read(&memcg->memory) >
2780 			READ_ONCE(memcg->memory.high);
2781 		swap_high = page_counter_read(&memcg->swap) >
2782 			READ_ONCE(memcg->swap.high);
2783 
2784 		/* Don't bother a random interrupted task */
2785 		if (!in_task()) {
2786 			if (mem_high) {
2787 				schedule_work(&memcg->high_work);
2788 				break;
2789 			}
2790 			continue;
2791 		}
2792 
2793 		if (mem_high || swap_high) {
2794 			/*
2795 			 * The allocating tasks in this cgroup will need to do
2796 			 * reclaim or be throttled to prevent further growth
2797 			 * of the memory or swap footprints.
2798 			 *
2799 			 * Target some best-effort fairness between the tasks,
2800 			 * and distribute reclaim work and delay penalties
2801 			 * based on how much each task is actually allocating.
2802 			 */
2803 			current->memcg_nr_pages_over_high += batch;
2804 			set_notify_resume(current);
2805 			break;
2806 		}
2807 	} while ((memcg = parent_mem_cgroup(memcg)));
2808 
2809 	if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH &&
2810 	    !(current->flags & PF_MEMALLOC) &&
2811 	    gfpflags_allow_blocking(gfp_mask)) {
2812 		mem_cgroup_handle_over_high();
2813 	}
2814 	return 0;
2815 }
2816 
try_charge(struct mem_cgroup * memcg,gfp_t gfp_mask,unsigned int nr_pages)2817 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2818 			     unsigned int nr_pages)
2819 {
2820 	if (mem_cgroup_is_root(memcg))
2821 		return 0;
2822 
2823 	return try_charge_memcg(memcg, gfp_mask, nr_pages);
2824 }
2825 
cancel_charge(struct mem_cgroup * memcg,unsigned int nr_pages)2826 static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2827 {
2828 	if (mem_cgroup_is_root(memcg))
2829 		return;
2830 
2831 	page_counter_uncharge(&memcg->memory, nr_pages);
2832 	if (do_memsw_account())
2833 		page_counter_uncharge(&memcg->memsw, nr_pages);
2834 }
2835 
commit_charge(struct folio * folio,struct mem_cgroup * memcg)2836 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
2837 {
2838 	VM_BUG_ON_FOLIO(folio_memcg(folio), folio);
2839 	/*
2840 	 * Any of the following ensures page's memcg stability:
2841 	 *
2842 	 * - the page lock
2843 	 * - LRU isolation
2844 	 * - lock_page_memcg()
2845 	 * - exclusive reference
2846 	 * - mem_cgroup_trylock_pages()
2847 	 */
2848 	folio->memcg_data = (unsigned long)memcg;
2849 }
2850 
2851 #ifdef CONFIG_MEMCG_KMEM
2852 /*
2853  * The allocated objcg pointers array is not accounted directly.
2854  * Moreover, it should not come from DMA buffer and is not readily
2855  * reclaimable. So those GFP bits should be masked off.
2856  */
2857 #define OBJCGS_CLEAR_MASK	(__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT)
2858 
2859 /*
2860  * mod_objcg_mlstate() may be called with irq enabled, so
2861  * mod_memcg_lruvec_state() should be used.
2862  */
mod_objcg_mlstate(struct obj_cgroup * objcg,struct pglist_data * pgdat,enum node_stat_item idx,int nr)2863 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
2864 				     struct pglist_data *pgdat,
2865 				     enum node_stat_item idx, int nr)
2866 {
2867 	struct mem_cgroup *memcg;
2868 	struct lruvec *lruvec;
2869 
2870 	rcu_read_lock();
2871 	memcg = obj_cgroup_memcg(objcg);
2872 	lruvec = mem_cgroup_lruvec(memcg, pgdat);
2873 	mod_memcg_lruvec_state(lruvec, idx, nr);
2874 	rcu_read_unlock();
2875 }
2876 
memcg_alloc_slab_cgroups(struct slab * slab,struct kmem_cache * s,gfp_t gfp,bool new_slab)2877 int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s,
2878 				 gfp_t gfp, bool new_slab)
2879 {
2880 	unsigned int objects = objs_per_slab(s, slab);
2881 	unsigned long memcg_data;
2882 	void *vec;
2883 
2884 	gfp &= ~OBJCGS_CLEAR_MASK;
2885 	vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2886 			   slab_nid(slab));
2887 	if (!vec)
2888 		return -ENOMEM;
2889 
2890 	memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
2891 	if (new_slab) {
2892 		/*
2893 		 * If the slab is brand new and nobody can yet access its
2894 		 * memcg_data, no synchronization is required and memcg_data can
2895 		 * be simply assigned.
2896 		 */
2897 		slab->memcg_data = memcg_data;
2898 	} else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) {
2899 		/*
2900 		 * If the slab is already in use, somebody can allocate and
2901 		 * assign obj_cgroups in parallel. In this case the existing
2902 		 * objcg vector should be reused.
2903 		 */
2904 		kfree(vec);
2905 		return 0;
2906 	}
2907 
2908 	kmemleak_not_leak(vec);
2909 	return 0;
2910 }
2911 
2912 static __always_inline
mem_cgroup_from_obj_folio(struct folio * folio,void * p)2913 struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p)
2914 {
2915 	/*
2916 	 * Slab objects are accounted individually, not per-page.
2917 	 * Memcg membership data for each individual object is saved in
2918 	 * slab->memcg_data.
2919 	 */
2920 	if (folio_test_slab(folio)) {
2921 		struct obj_cgroup **objcgs;
2922 		struct slab *slab;
2923 		unsigned int off;
2924 
2925 		slab = folio_slab(folio);
2926 		objcgs = slab_objcgs(slab);
2927 		if (!objcgs)
2928 			return NULL;
2929 
2930 		off = obj_to_index(slab->slab_cache, slab, p);
2931 		if (objcgs[off])
2932 			return obj_cgroup_memcg(objcgs[off]);
2933 
2934 		return NULL;
2935 	}
2936 
2937 	/*
2938 	 * page_memcg_check() is used here, because in theory we can encounter
2939 	 * a folio where the slab flag has been cleared already, but
2940 	 * slab->memcg_data has not been freed yet
2941 	 * page_memcg_check(page) will guarantee that a proper memory
2942 	 * cgroup pointer or NULL will be returned.
2943 	 */
2944 	return page_memcg_check(folio_page(folio, 0));
2945 }
2946 
2947 /*
2948  * Returns a pointer to the memory cgroup to which the kernel object is charged.
2949  *
2950  * A passed kernel object can be a slab object, vmalloc object or a generic
2951  * kernel page, so different mechanisms for getting the memory cgroup pointer
2952  * should be used.
2953  *
2954  * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller
2955  * can not know for sure how the kernel object is implemented.
2956  * mem_cgroup_from_obj() can be safely used in such cases.
2957  *
2958  * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2959  * cgroup_mutex, etc.
2960  */
mem_cgroup_from_obj(void * p)2961 struct mem_cgroup *mem_cgroup_from_obj(void *p)
2962 {
2963 	struct folio *folio;
2964 
2965 	if (mem_cgroup_disabled())
2966 		return NULL;
2967 
2968 	if (unlikely(is_vmalloc_addr(p)))
2969 		folio = page_folio(vmalloc_to_page(p));
2970 	else
2971 		folio = virt_to_folio(p);
2972 
2973 	return mem_cgroup_from_obj_folio(folio, p);
2974 }
2975 
2976 /*
2977  * Returns a pointer to the memory cgroup to which the kernel object is charged.
2978  * Similar to mem_cgroup_from_obj(), but faster and not suitable for objects,
2979  * allocated using vmalloc().
2980  *
2981  * A passed kernel object must be a slab object or a generic kernel page.
2982  *
2983  * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2984  * cgroup_mutex, etc.
2985  */
mem_cgroup_from_slab_obj(void * p)2986 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)
2987 {
2988 	if (mem_cgroup_disabled())
2989 		return NULL;
2990 
2991 	return mem_cgroup_from_obj_folio(virt_to_folio(p), p);
2992 }
2993 
__get_obj_cgroup_from_memcg(struct mem_cgroup * memcg)2994 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
2995 {
2996 	struct obj_cgroup *objcg = NULL;
2997 
2998 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
2999 		objcg = rcu_dereference(memcg->objcg);
3000 		if (objcg && obj_cgroup_tryget(objcg))
3001 			break;
3002 		objcg = NULL;
3003 	}
3004 	return objcg;
3005 }
3006 
get_obj_cgroup_from_current(void)3007 __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
3008 {
3009 	struct obj_cgroup *objcg = NULL;
3010 	struct mem_cgroup *memcg;
3011 
3012 	if (memcg_kmem_bypass())
3013 		return NULL;
3014 
3015 	rcu_read_lock();
3016 	if (unlikely(active_memcg()))
3017 		memcg = active_memcg();
3018 	else
3019 		memcg = mem_cgroup_from_task(current);
3020 	objcg = __get_obj_cgroup_from_memcg(memcg);
3021 	rcu_read_unlock();
3022 	return objcg;
3023 }
3024 
get_obj_cgroup_from_page(struct page * page)3025 struct obj_cgroup *get_obj_cgroup_from_page(struct page *page)
3026 {
3027 	struct obj_cgroup *objcg;
3028 
3029 	if (!memcg_kmem_enabled())
3030 		return NULL;
3031 
3032 	if (PageMemcgKmem(page)) {
3033 		objcg = __folio_objcg(page_folio(page));
3034 		obj_cgroup_get(objcg);
3035 	} else {
3036 		struct mem_cgroup *memcg;
3037 
3038 		rcu_read_lock();
3039 		memcg = __folio_memcg(page_folio(page));
3040 		if (memcg)
3041 			objcg = __get_obj_cgroup_from_memcg(memcg);
3042 		else
3043 			objcg = NULL;
3044 		rcu_read_unlock();
3045 	}
3046 	return objcg;
3047 }
3048 
memcg_account_kmem(struct mem_cgroup * memcg,int nr_pages)3049 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
3050 {
3051 	mod_memcg_state(memcg, MEMCG_KMEM, nr_pages);
3052 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
3053 		if (nr_pages > 0)
3054 			page_counter_charge(&memcg->kmem, nr_pages);
3055 		else
3056 			page_counter_uncharge(&memcg->kmem, -nr_pages);
3057 	}
3058 }
3059 
3060 
3061 /*
3062  * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg
3063  * @objcg: object cgroup to uncharge
3064  * @nr_pages: number of pages to uncharge
3065  */
obj_cgroup_uncharge_pages(struct obj_cgroup * objcg,unsigned int nr_pages)3066 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
3067 				      unsigned int nr_pages)
3068 {
3069 	struct mem_cgroup *memcg;
3070 
3071 	memcg = get_mem_cgroup_from_objcg(objcg);
3072 
3073 	memcg_account_kmem(memcg, -nr_pages);
3074 	refill_stock(memcg, nr_pages);
3075 
3076 	css_put(&memcg->css);
3077 }
3078 
3079 /*
3080  * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg
3081  * @objcg: object cgroup to charge
3082  * @gfp: reclaim mode
3083  * @nr_pages: number of pages to charge
3084  *
3085  * Returns 0 on success, an error code on failure.
3086  */
obj_cgroup_charge_pages(struct obj_cgroup * objcg,gfp_t gfp,unsigned int nr_pages)3087 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
3088 				   unsigned int nr_pages)
3089 {
3090 	struct mem_cgroup *memcg;
3091 	int ret;
3092 
3093 	memcg = get_mem_cgroup_from_objcg(objcg);
3094 
3095 	ret = try_charge_memcg(memcg, gfp, nr_pages);
3096 	if (ret)
3097 		goto out;
3098 
3099 	memcg_account_kmem(memcg, nr_pages);
3100 out:
3101 	css_put(&memcg->css);
3102 
3103 	return ret;
3104 }
3105 
3106 /**
3107  * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
3108  * @page: page to charge
3109  * @gfp: reclaim mode
3110  * @order: allocation order
3111  *
3112  * Returns 0 on success, an error code on failure.
3113  */
__memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)3114 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
3115 {
3116 	struct obj_cgroup *objcg;
3117 	int ret = 0;
3118 
3119 	objcg = get_obj_cgroup_from_current();
3120 	if (objcg) {
3121 		ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
3122 		if (!ret) {
3123 			page->memcg_data = (unsigned long)objcg |
3124 				MEMCG_DATA_KMEM;
3125 			return 0;
3126 		}
3127 		obj_cgroup_put(objcg);
3128 	}
3129 	return ret;
3130 }
3131 
3132 /**
3133  * __memcg_kmem_uncharge_page: uncharge a kmem page
3134  * @page: page to uncharge
3135  * @order: allocation order
3136  */
__memcg_kmem_uncharge_page(struct page * page,int order)3137 void __memcg_kmem_uncharge_page(struct page *page, int order)
3138 {
3139 	struct folio *folio = page_folio(page);
3140 	struct obj_cgroup *objcg;
3141 	unsigned int nr_pages = 1 << order;
3142 
3143 	if (!folio_memcg_kmem(folio))
3144 		return;
3145 
3146 	objcg = __folio_objcg(folio);
3147 	obj_cgroup_uncharge_pages(objcg, nr_pages);
3148 	folio->memcg_data = 0;
3149 	obj_cgroup_put(objcg);
3150 }
3151 
mod_objcg_state(struct obj_cgroup * objcg,struct pglist_data * pgdat,enum node_stat_item idx,int nr)3152 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
3153 		     enum node_stat_item idx, int nr)
3154 {
3155 	struct memcg_stock_pcp *stock;
3156 	struct obj_cgroup *old = NULL;
3157 	unsigned long flags;
3158 	int *bytes;
3159 
3160 	local_lock_irqsave(&memcg_stock.stock_lock, flags);
3161 	stock = this_cpu_ptr(&memcg_stock);
3162 
3163 	/*
3164 	 * Save vmstat data in stock and skip vmstat array update unless
3165 	 * accumulating over a page of vmstat data or when pgdat or idx
3166 	 * changes.
3167 	 */
3168 	if (stock->cached_objcg != objcg) {
3169 		old = drain_obj_stock(stock);
3170 		obj_cgroup_get(objcg);
3171 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3172 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3173 		stock->cached_objcg = objcg;
3174 		stock->cached_pgdat = pgdat;
3175 	} else if (stock->cached_pgdat != pgdat) {
3176 		/* Flush the existing cached vmstat data */
3177 		struct pglist_data *oldpg = stock->cached_pgdat;
3178 
3179 		if (stock->nr_slab_reclaimable_b) {
3180 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,
3181 					  stock->nr_slab_reclaimable_b);
3182 			stock->nr_slab_reclaimable_b = 0;
3183 		}
3184 		if (stock->nr_slab_unreclaimable_b) {
3185 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,
3186 					  stock->nr_slab_unreclaimable_b);
3187 			stock->nr_slab_unreclaimable_b = 0;
3188 		}
3189 		stock->cached_pgdat = pgdat;
3190 	}
3191 
3192 	bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b
3193 					       : &stock->nr_slab_unreclaimable_b;
3194 	/*
3195 	 * Even for large object >= PAGE_SIZE, the vmstat data will still be
3196 	 * cached locally at least once before pushing it out.
3197 	 */
3198 	if (!*bytes) {
3199 		*bytes = nr;
3200 		nr = 0;
3201 	} else {
3202 		*bytes += nr;
3203 		if (abs(*bytes) > PAGE_SIZE) {
3204 			nr = *bytes;
3205 			*bytes = 0;
3206 		} else {
3207 			nr = 0;
3208 		}
3209 	}
3210 	if (nr)
3211 		mod_objcg_mlstate(objcg, pgdat, idx, nr);
3212 
3213 	local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
3214 	if (old)
3215 		obj_cgroup_put(old);
3216 }
3217 
consume_obj_stock(struct obj_cgroup * objcg,unsigned int nr_bytes)3218 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3219 {
3220 	struct memcg_stock_pcp *stock;
3221 	unsigned long flags;
3222 	bool ret = false;
3223 
3224 	local_lock_irqsave(&memcg_stock.stock_lock, flags);
3225 
3226 	stock = this_cpu_ptr(&memcg_stock);
3227 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3228 		stock->nr_bytes -= nr_bytes;
3229 		ret = true;
3230 	}
3231 
3232 	local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
3233 
3234 	return ret;
3235 }
3236 
drain_obj_stock(struct memcg_stock_pcp * stock)3237 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock)
3238 {
3239 	struct obj_cgroup *old = stock->cached_objcg;
3240 
3241 	if (!old)
3242 		return NULL;
3243 
3244 	if (stock->nr_bytes) {
3245 		unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3246 		unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
3247 
3248 		if (nr_pages) {
3249 			struct mem_cgroup *memcg;
3250 
3251 			memcg = get_mem_cgroup_from_objcg(old);
3252 
3253 			memcg_account_kmem(memcg, -nr_pages);
3254 			__refill_stock(memcg, nr_pages);
3255 
3256 			css_put(&memcg->css);
3257 		}
3258 
3259 		/*
3260 		 * The leftover is flushed to the centralized per-memcg value.
3261 		 * On the next attempt to refill obj stock it will be moved
3262 		 * to a per-cpu stock (probably, on an other CPU), see
3263 		 * refill_obj_stock().
3264 		 *
3265 		 * How often it's flushed is a trade-off between the memory
3266 		 * limit enforcement accuracy and potential CPU contention,
3267 		 * so it might be changed in the future.
3268 		 */
3269 		atomic_add(nr_bytes, &old->nr_charged_bytes);
3270 		stock->nr_bytes = 0;
3271 	}
3272 
3273 	/*
3274 	 * Flush the vmstat data in current stock
3275 	 */
3276 	if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) {
3277 		if (stock->nr_slab_reclaimable_b) {
3278 			mod_objcg_mlstate(old, stock->cached_pgdat,
3279 					  NR_SLAB_RECLAIMABLE_B,
3280 					  stock->nr_slab_reclaimable_b);
3281 			stock->nr_slab_reclaimable_b = 0;
3282 		}
3283 		if (stock->nr_slab_unreclaimable_b) {
3284 			mod_objcg_mlstate(old, stock->cached_pgdat,
3285 					  NR_SLAB_UNRECLAIMABLE_B,
3286 					  stock->nr_slab_unreclaimable_b);
3287 			stock->nr_slab_unreclaimable_b = 0;
3288 		}
3289 		stock->cached_pgdat = NULL;
3290 	}
3291 
3292 	stock->cached_objcg = NULL;
3293 	/*
3294 	 * The `old' objects needs to be released by the caller via
3295 	 * obj_cgroup_put() outside of memcg_stock_pcp::stock_lock.
3296 	 */
3297 	return old;
3298 }
3299 
obj_stock_flush_required(struct memcg_stock_pcp * stock,struct mem_cgroup * root_memcg)3300 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3301 				     struct mem_cgroup *root_memcg)
3302 {
3303 	struct mem_cgroup *memcg;
3304 
3305 	if (stock->cached_objcg) {
3306 		memcg = obj_cgroup_memcg(stock->cached_objcg);
3307 		if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3308 			return true;
3309 	}
3310 
3311 	return false;
3312 }
3313 
refill_obj_stock(struct obj_cgroup * objcg,unsigned int nr_bytes,bool allow_uncharge)3314 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,
3315 			     bool allow_uncharge)
3316 {
3317 	struct memcg_stock_pcp *stock;
3318 	struct obj_cgroup *old = NULL;
3319 	unsigned long flags;
3320 	unsigned int nr_pages = 0;
3321 
3322 	local_lock_irqsave(&memcg_stock.stock_lock, flags);
3323 
3324 	stock = this_cpu_ptr(&memcg_stock);
3325 	if (stock->cached_objcg != objcg) { /* reset if necessary */
3326 		old = drain_obj_stock(stock);
3327 		obj_cgroup_get(objcg);
3328 		stock->cached_objcg = objcg;
3329 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3330 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3331 		allow_uncharge = true;	/* Allow uncharge when objcg changes */
3332 	}
3333 	stock->nr_bytes += nr_bytes;
3334 
3335 	if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) {
3336 		nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3337 		stock->nr_bytes &= (PAGE_SIZE - 1);
3338 	}
3339 
3340 	local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
3341 	if (old)
3342 		obj_cgroup_put(old);
3343 
3344 	if (nr_pages)
3345 		obj_cgroup_uncharge_pages(objcg, nr_pages);
3346 }
3347 
obj_cgroup_charge(struct obj_cgroup * objcg,gfp_t gfp,size_t size)3348 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3349 {
3350 	unsigned int nr_pages, nr_bytes;
3351 	int ret;
3352 
3353 	if (consume_obj_stock(objcg, size))
3354 		return 0;
3355 
3356 	/*
3357 	 * In theory, objcg->nr_charged_bytes can have enough
3358 	 * pre-charged bytes to satisfy the allocation. However,
3359 	 * flushing objcg->nr_charged_bytes requires two atomic
3360 	 * operations, and objcg->nr_charged_bytes can't be big.
3361 	 * The shared objcg->nr_charged_bytes can also become a
3362 	 * performance bottleneck if all tasks of the same memcg are
3363 	 * trying to update it. So it's better to ignore it and try
3364 	 * grab some new pages. The stock's nr_bytes will be flushed to
3365 	 * objcg->nr_charged_bytes later on when objcg changes.
3366 	 *
3367 	 * The stock's nr_bytes may contain enough pre-charged bytes
3368 	 * to allow one less page from being charged, but we can't rely
3369 	 * on the pre-charged bytes not being changed outside of
3370 	 * consume_obj_stock() or refill_obj_stock(). So ignore those
3371 	 * pre-charged bytes as well when charging pages. To avoid a
3372 	 * page uncharge right after a page charge, we set the
3373 	 * allow_uncharge flag to false when calling refill_obj_stock()
3374 	 * to temporarily allow the pre-charged bytes to exceed the page
3375 	 * size limit. The maximum reachable value of the pre-charged
3376 	 * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data
3377 	 * race.
3378 	 */
3379 	nr_pages = size >> PAGE_SHIFT;
3380 	nr_bytes = size & (PAGE_SIZE - 1);
3381 
3382 	if (nr_bytes)
3383 		nr_pages += 1;
3384 
3385 	ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);
3386 	if (!ret && nr_bytes)
3387 		refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);
3388 
3389 	return ret;
3390 }
3391 
obj_cgroup_uncharge(struct obj_cgroup * objcg,size_t size)3392 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3393 {
3394 	refill_obj_stock(objcg, size, true);
3395 }
3396 
3397 #endif /* CONFIG_MEMCG_KMEM */
3398 
3399 /*
3400  * Because page_memcg(head) is not set on tails, set it now.
3401  */
split_page_memcg(struct page * head,unsigned int nr)3402 void split_page_memcg(struct page *head, unsigned int nr)
3403 {
3404 	struct folio *folio = page_folio(head);
3405 	struct mem_cgroup *memcg = folio_memcg(folio);
3406 	int i;
3407 
3408 	if (mem_cgroup_disabled() || !memcg)
3409 		return;
3410 
3411 	for (i = 1; i < nr; i++)
3412 		folio_page(folio, i)->memcg_data = folio->memcg_data;
3413 
3414 	if (folio_memcg_kmem(folio))
3415 		obj_cgroup_get_many(__folio_objcg(folio), nr - 1);
3416 	else
3417 		css_get_many(&memcg->css, nr - 1);
3418 }
3419 
3420 #ifdef CONFIG_SWAP
3421 /**
3422  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3423  * @entry: swap entry to be moved
3424  * @from:  mem_cgroup which the entry is moved from
3425  * @to:  mem_cgroup which the entry is moved to
3426  *
3427  * It succeeds only when the swap_cgroup's record for this entry is the same
3428  * as the mem_cgroup's id of @from.
3429  *
3430  * Returns 0 on success, -EINVAL on failure.
3431  *
3432  * The caller must have charged to @to, IOW, called page_counter_charge() about
3433  * both res and memsw, and called css_get().
3434  */
mem_cgroup_move_swap_account(swp_entry_t entry,struct mem_cgroup * from,struct mem_cgroup * to)3435 static int mem_cgroup_move_swap_account(swp_entry_t entry,
3436 				struct mem_cgroup *from, struct mem_cgroup *to)
3437 {
3438 	unsigned short old_id, new_id;
3439 
3440 	old_id = mem_cgroup_id(from);
3441 	new_id = mem_cgroup_id(to);
3442 
3443 	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
3444 		mod_memcg_state(from, MEMCG_SWAP, -1);
3445 		mod_memcg_state(to, MEMCG_SWAP, 1);
3446 		return 0;
3447 	}
3448 	return -EINVAL;
3449 }
3450 #else
mem_cgroup_move_swap_account(swp_entry_t entry,struct mem_cgroup * from,struct mem_cgroup * to)3451 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3452 				struct mem_cgroup *from, struct mem_cgroup *to)
3453 {
3454 	return -EINVAL;
3455 }
3456 #endif
3457 
3458 static DEFINE_MUTEX(memcg_max_mutex);
3459 
mem_cgroup_resize_max(struct mem_cgroup * memcg,unsigned long max,bool memsw)3460 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3461 				 unsigned long max, bool memsw)
3462 {
3463 	bool enlarge = false;
3464 	bool drained = false;
3465 	int ret;
3466 	bool limits_invariant;
3467 	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3468 
3469 	do {
3470 		if (signal_pending(current)) {
3471 			ret = -EINTR;
3472 			break;
3473 		}
3474 
3475 		mutex_lock(&memcg_max_mutex);
3476 		/*
3477 		 * Make sure that the new limit (memsw or memory limit) doesn't
3478 		 * break our basic invariant rule memory.max <= memsw.max.
3479 		 */
3480 		limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3481 					   max <= memcg->memsw.max;
3482 		if (!limits_invariant) {
3483 			mutex_unlock(&memcg_max_mutex);
3484 			ret = -EINVAL;
3485 			break;
3486 		}
3487 		if (max > counter->max)
3488 			enlarge = true;
3489 		ret = page_counter_set_max(counter, max);
3490 		mutex_unlock(&memcg_max_mutex);
3491 
3492 		if (!ret)
3493 			break;
3494 
3495 		if (!drained) {
3496 			drain_all_stock(memcg);
3497 			drained = true;
3498 			continue;
3499 		}
3500 
3501 		if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
3502 					memsw ? 0 : MEMCG_RECLAIM_MAY_SWAP)) {
3503 			ret = -EBUSY;
3504 			break;
3505 		}
3506 	} while (true);
3507 
3508 	if (!ret && enlarge)
3509 		memcg_oom_recover(memcg);
3510 
3511 	return ret;
3512 }
3513 
mem_cgroup_soft_limit_reclaim(pg_data_t * pgdat,int order,gfp_t gfp_mask,unsigned long * total_scanned)3514 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
3515 					    gfp_t gfp_mask,
3516 					    unsigned long *total_scanned)
3517 {
3518 	unsigned long nr_reclaimed = 0;
3519 	struct mem_cgroup_per_node *mz, *next_mz = NULL;
3520 	unsigned long reclaimed;
3521 	int loop = 0;
3522 	struct mem_cgroup_tree_per_node *mctz;
3523 	unsigned long excess;
3524 
3525 	if (order > 0)
3526 		return 0;
3527 
3528 	mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id];
3529 
3530 	/*
3531 	 * Do not even bother to check the largest node if the root
3532 	 * is empty. Do it lockless to prevent lock bouncing. Races
3533 	 * are acceptable as soft limit is best effort anyway.
3534 	 */
3535 	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
3536 		return 0;
3537 
3538 	/*
3539 	 * This loop can run a while, specially if mem_cgroup's continuously
3540 	 * keep exceeding their soft limit and putting the system under
3541 	 * pressure
3542 	 */
3543 	do {
3544 		if (next_mz)
3545 			mz = next_mz;
3546 		else
3547 			mz = mem_cgroup_largest_soft_limit_node(mctz);
3548 		if (!mz)
3549 			break;
3550 
3551 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3552 						    gfp_mask, total_scanned);
3553 		nr_reclaimed += reclaimed;
3554 		spin_lock_irq(&mctz->lock);
3555 
3556 		/*
3557 		 * If we failed to reclaim anything from this memory cgroup
3558 		 * it is time to move on to the next cgroup
3559 		 */
3560 		next_mz = NULL;
3561 		if (!reclaimed)
3562 			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3563 
3564 		excess = soft_limit_excess(mz->memcg);
3565 		/*
3566 		 * One school of thought says that we should not add
3567 		 * back the node to the tree if reclaim returns 0.
3568 		 * But our reclaim could return 0, simply because due
3569 		 * to priority we are exposing a smaller subset of
3570 		 * memory to reclaim from. Consider this as a longer
3571 		 * term TODO.
3572 		 */
3573 		/* If excess == 0, no tree ops */
3574 		__mem_cgroup_insert_exceeded(mz, mctz, excess);
3575 		spin_unlock_irq(&mctz->lock);
3576 		css_put(&mz->memcg->css);
3577 		loop++;
3578 		/*
3579 		 * Could not reclaim anything and there are no more
3580 		 * mem cgroups to try or we seem to be looping without
3581 		 * reclaiming anything.
3582 		 */
3583 		if (!nr_reclaimed &&
3584 			(next_mz == NULL ||
3585 			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3586 			break;
3587 	} while (!nr_reclaimed);
3588 	if (next_mz)
3589 		css_put(&next_mz->memcg->css);
3590 	return nr_reclaimed;
3591 }
3592 
3593 /*
3594  * Reclaims as many pages from the given memcg as possible.
3595  *
3596  * Caller is responsible for holding css reference for memcg.
3597  */
mem_cgroup_force_empty(struct mem_cgroup * memcg)3598 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3599 {
3600 	int nr_retries = MAX_RECLAIM_RETRIES;
3601 
3602 	/* we call try-to-free pages for make this cgroup empty */
3603 	lru_add_drain_all();
3604 
3605 	drain_all_stock(memcg);
3606 
3607 	/* try to free all pages in this cgroup */
3608 	while (nr_retries && page_counter_read(&memcg->memory)) {
3609 		if (signal_pending(current))
3610 			return -EINTR;
3611 
3612 		if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
3613 						  MEMCG_RECLAIM_MAY_SWAP))
3614 			nr_retries--;
3615 	}
3616 
3617 	return 0;
3618 }
3619 
mem_cgroup_force_empty_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)3620 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3621 					    char *buf, size_t nbytes,
3622 					    loff_t off)
3623 {
3624 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3625 
3626 	if (mem_cgroup_is_root(memcg))
3627 		return -EINVAL;
3628 	return mem_cgroup_force_empty(memcg) ?: nbytes;
3629 }
3630 
mem_cgroup_hierarchy_read(struct cgroup_subsys_state * css,struct cftype * cft)3631 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3632 				     struct cftype *cft)
3633 {
3634 	return 1;
3635 }
3636 
mem_cgroup_hierarchy_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)3637 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3638 				      struct cftype *cft, u64 val)
3639 {
3640 	if (val == 1)
3641 		return 0;
3642 
3643 	pr_warn_once("Non-hierarchical mode is deprecated. "
3644 		     "Please report your usecase to linux-mm@kvack.org if you "
3645 		     "depend on this functionality.\n");
3646 
3647 	return -EINVAL;
3648 }
3649 
mem_cgroup_usage(struct mem_cgroup * memcg,bool swap)3650 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3651 {
3652 	unsigned long val;
3653 
3654 	if (mem_cgroup_is_root(memcg)) {
3655 		mem_cgroup_flush_stats();
3656 		val = memcg_page_state(memcg, NR_FILE_PAGES) +
3657 			memcg_page_state(memcg, NR_ANON_MAPPED);
3658 		if (swap)
3659 			val += memcg_page_state(memcg, MEMCG_SWAP);
3660 	} else {
3661 		if (!swap)
3662 			val = page_counter_read(&memcg->memory);
3663 		else
3664 			val = page_counter_read(&memcg->memsw);
3665 	}
3666 	return val;
3667 }
3668 
3669 enum {
3670 	RES_USAGE,
3671 	RES_LIMIT,
3672 	RES_MAX_USAGE,
3673 	RES_FAILCNT,
3674 	RES_SOFT_LIMIT,
3675 };
3676 
mem_cgroup_read_u64(struct cgroup_subsys_state * css,struct cftype * cft)3677 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3678 			       struct cftype *cft)
3679 {
3680 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3681 	struct page_counter *counter;
3682 
3683 	switch (MEMFILE_TYPE(cft->private)) {
3684 	case _MEM:
3685 		counter = &memcg->memory;
3686 		break;
3687 	case _MEMSWAP:
3688 		counter = &memcg->memsw;
3689 		break;
3690 	case _KMEM:
3691 		counter = &memcg->kmem;
3692 		break;
3693 	case _TCP:
3694 		counter = &memcg->tcpmem;
3695 		break;
3696 	default:
3697 		BUG();
3698 	}
3699 
3700 	switch (MEMFILE_ATTR(cft->private)) {
3701 	case RES_USAGE:
3702 		if (counter == &memcg->memory)
3703 			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3704 		if (counter == &memcg->memsw)
3705 			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3706 		return (u64)page_counter_read(counter) * PAGE_SIZE;
3707 	case RES_LIMIT:
3708 		return (u64)counter->max * PAGE_SIZE;
3709 	case RES_MAX_USAGE:
3710 		return (u64)counter->watermark * PAGE_SIZE;
3711 	case RES_FAILCNT:
3712 		return counter->failcnt;
3713 	case RES_SOFT_LIMIT:
3714 		return (u64)memcg->soft_limit * PAGE_SIZE;
3715 	default:
3716 		BUG();
3717 	}
3718 }
3719 
3720 #ifdef CONFIG_MEMCG_KMEM
memcg_online_kmem(struct mem_cgroup * memcg)3721 static int memcg_online_kmem(struct mem_cgroup *memcg)
3722 {
3723 	struct obj_cgroup *objcg;
3724 
3725 	if (mem_cgroup_kmem_disabled())
3726 		return 0;
3727 
3728 	if (unlikely(mem_cgroup_is_root(memcg)))
3729 		return 0;
3730 
3731 	objcg = obj_cgroup_alloc();
3732 	if (!objcg)
3733 		return -ENOMEM;
3734 
3735 	objcg->memcg = memcg;
3736 	rcu_assign_pointer(memcg->objcg, objcg);
3737 
3738 	static_branch_enable(&memcg_kmem_enabled_key);
3739 
3740 	memcg->kmemcg_id = memcg->id.id;
3741 
3742 	return 0;
3743 }
3744 
memcg_offline_kmem(struct mem_cgroup * memcg)3745 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3746 {
3747 	struct mem_cgroup *parent;
3748 
3749 	if (mem_cgroup_kmem_disabled())
3750 		return;
3751 
3752 	if (unlikely(mem_cgroup_is_root(memcg)))
3753 		return;
3754 
3755 	parent = parent_mem_cgroup(memcg);
3756 	if (!parent)
3757 		parent = root_mem_cgroup;
3758 
3759 	memcg_reparent_objcgs(memcg, parent);
3760 
3761 	/*
3762 	 * After we have finished memcg_reparent_objcgs(), all list_lrus
3763 	 * corresponding to this cgroup are guaranteed to remain empty.
3764 	 * The ordering is imposed by list_lru_node->lock taken by
3765 	 * memcg_reparent_list_lrus().
3766 	 */
3767 	memcg_reparent_list_lrus(memcg, parent);
3768 }
3769 #else
memcg_online_kmem(struct mem_cgroup * memcg)3770 static int memcg_online_kmem(struct mem_cgroup *memcg)
3771 {
3772 	return 0;
3773 }
memcg_offline_kmem(struct mem_cgroup * memcg)3774 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3775 {
3776 }
3777 #endif /* CONFIG_MEMCG_KMEM */
3778 
memcg_update_tcp_max(struct mem_cgroup * memcg,unsigned long max)3779 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3780 {
3781 	int ret;
3782 
3783 	mutex_lock(&memcg_max_mutex);
3784 
3785 	ret = page_counter_set_max(&memcg->tcpmem, max);
3786 	if (ret)
3787 		goto out;
3788 
3789 	if (!memcg->tcpmem_active) {
3790 		/*
3791 		 * The active flag needs to be written after the static_key
3792 		 * update. This is what guarantees that the socket activation
3793 		 * function is the last one to run. See mem_cgroup_sk_alloc()
3794 		 * for details, and note that we don't mark any socket as
3795 		 * belonging to this memcg until that flag is up.
3796 		 *
3797 		 * We need to do this, because static_keys will span multiple
3798 		 * sites, but we can't control their order. If we mark a socket
3799 		 * as accounted, but the accounting functions are not patched in
3800 		 * yet, we'll lose accounting.
3801 		 *
3802 		 * We never race with the readers in mem_cgroup_sk_alloc(),
3803 		 * because when this value change, the code to process it is not
3804 		 * patched in yet.
3805 		 */
3806 		static_branch_inc(&memcg_sockets_enabled_key);
3807 		memcg->tcpmem_active = true;
3808 	}
3809 out:
3810 	mutex_unlock(&memcg_max_mutex);
3811 	return ret;
3812 }
3813 
3814 /*
3815  * The user of this function is...
3816  * RES_LIMIT.
3817  */
mem_cgroup_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)3818 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3819 				char *buf, size_t nbytes, loff_t off)
3820 {
3821 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3822 	unsigned long nr_pages;
3823 	int ret;
3824 
3825 	buf = strstrip(buf);
3826 	ret = page_counter_memparse(buf, "-1", &nr_pages);
3827 	if (ret)
3828 		return ret;
3829 
3830 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3831 	case RES_LIMIT:
3832 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3833 			ret = -EINVAL;
3834 			break;
3835 		}
3836 		switch (MEMFILE_TYPE(of_cft(of)->private)) {
3837 		case _MEM:
3838 			ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3839 			break;
3840 		case _MEMSWAP:
3841 			ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3842 			break;
3843 		case _KMEM:
3844 			/* kmem.limit_in_bytes is deprecated. */
3845 			ret = -EOPNOTSUPP;
3846 			break;
3847 		case _TCP:
3848 			ret = memcg_update_tcp_max(memcg, nr_pages);
3849 			break;
3850 		}
3851 		break;
3852 	case RES_SOFT_LIMIT:
3853 		if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
3854 			ret = -EOPNOTSUPP;
3855 		} else {
3856 			memcg->soft_limit = nr_pages;
3857 			ret = 0;
3858 		}
3859 		break;
3860 	}
3861 	return ret ?: nbytes;
3862 }
3863 
mem_cgroup_reset(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)3864 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3865 				size_t nbytes, loff_t off)
3866 {
3867 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3868 	struct page_counter *counter;
3869 
3870 	switch (MEMFILE_TYPE(of_cft(of)->private)) {
3871 	case _MEM:
3872 		counter = &memcg->memory;
3873 		break;
3874 	case _MEMSWAP:
3875 		counter = &memcg->memsw;
3876 		break;
3877 	case _KMEM:
3878 		counter = &memcg->kmem;
3879 		break;
3880 	case _TCP:
3881 		counter = &memcg->tcpmem;
3882 		break;
3883 	default:
3884 		BUG();
3885 	}
3886 
3887 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3888 	case RES_MAX_USAGE:
3889 		page_counter_reset_watermark(counter);
3890 		break;
3891 	case RES_FAILCNT:
3892 		counter->failcnt = 0;
3893 		break;
3894 	default:
3895 		BUG();
3896 	}
3897 
3898 	return nbytes;
3899 }
3900 
mem_cgroup_move_charge_read(struct cgroup_subsys_state * css,struct cftype * cft)3901 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3902 					struct cftype *cft)
3903 {
3904 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3905 }
3906 
3907 #ifdef CONFIG_MMU
mem_cgroup_move_charge_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)3908 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3909 					struct cftype *cft, u64 val)
3910 {
3911 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3912 
3913 	if (val & ~MOVE_MASK)
3914 		return -EINVAL;
3915 
3916 	/*
3917 	 * No kind of locking is needed in here, because ->can_attach() will
3918 	 * check this value once in the beginning of the process, and then carry
3919 	 * on with stale data. This means that changes to this value will only
3920 	 * affect task migrations starting after the change.
3921 	 */
3922 	memcg->move_charge_at_immigrate = val;
3923 	return 0;
3924 }
3925 #else
mem_cgroup_move_charge_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)3926 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3927 					struct cftype *cft, u64 val)
3928 {
3929 	return -ENOSYS;
3930 }
3931 #endif
3932 
3933 #ifdef CONFIG_NUMA
3934 
3935 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3936 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3937 #define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)
3938 
mem_cgroup_node_nr_lru_pages(struct mem_cgroup * memcg,int nid,unsigned int lru_mask,bool tree)3939 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
3940 				int nid, unsigned int lru_mask, bool tree)
3941 {
3942 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
3943 	unsigned long nr = 0;
3944 	enum lru_list lru;
3945 
3946 	VM_BUG_ON((unsigned)nid >= nr_node_ids);
3947 
3948 	for_each_lru(lru) {
3949 		if (!(BIT(lru) & lru_mask))
3950 			continue;
3951 		if (tree)
3952 			nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3953 		else
3954 			nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
3955 	}
3956 	return nr;
3957 }
3958 
mem_cgroup_nr_lru_pages(struct mem_cgroup * memcg,unsigned int lru_mask,bool tree)3959 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
3960 					     unsigned int lru_mask,
3961 					     bool tree)
3962 {
3963 	unsigned long nr = 0;
3964 	enum lru_list lru;
3965 
3966 	for_each_lru(lru) {
3967 		if (!(BIT(lru) & lru_mask))
3968 			continue;
3969 		if (tree)
3970 			nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
3971 		else
3972 			nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
3973 	}
3974 	return nr;
3975 }
3976 
memcg_numa_stat_show(struct seq_file * m,void * v)3977 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3978 {
3979 	struct numa_stat {
3980 		const char *name;
3981 		unsigned int lru_mask;
3982 	};
3983 
3984 	static const struct numa_stat stats[] = {
3985 		{ "total", LRU_ALL },
3986 		{ "file", LRU_ALL_FILE },
3987 		{ "anon", LRU_ALL_ANON },
3988 		{ "unevictable", BIT(LRU_UNEVICTABLE) },
3989 	};
3990 	const struct numa_stat *stat;
3991 	int nid;
3992 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3993 
3994 	mem_cgroup_flush_stats();
3995 
3996 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3997 		seq_printf(m, "%s=%lu", stat->name,
3998 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3999 						   false));
4000 		for_each_node_state(nid, N_MEMORY)
4001 			seq_printf(m, " N%d=%lu", nid,
4002 				   mem_cgroup_node_nr_lru_pages(memcg, nid,
4003 							stat->lru_mask, false));
4004 		seq_putc(m, '\n');
4005 	}
4006 
4007 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
4008 
4009 		seq_printf(m, "hierarchical_%s=%lu", stat->name,
4010 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4011 						   true));
4012 		for_each_node_state(nid, N_MEMORY)
4013 			seq_printf(m, " N%d=%lu", nid,
4014 				   mem_cgroup_node_nr_lru_pages(memcg, nid,
4015 							stat->lru_mask, true));
4016 		seq_putc(m, '\n');
4017 	}
4018 
4019 	return 0;
4020 }
4021 #endif /* CONFIG_NUMA */
4022 
4023 static const unsigned int memcg1_stats[] = {
4024 	NR_FILE_PAGES,
4025 	NR_ANON_MAPPED,
4026 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4027 	NR_ANON_THPS,
4028 #endif
4029 	NR_SHMEM,
4030 	NR_FILE_MAPPED,
4031 	NR_FILE_DIRTY,
4032 	NR_WRITEBACK,
4033 	WORKINGSET_REFAULT_ANON,
4034 	WORKINGSET_REFAULT_FILE,
4035 	MEMCG_SWAP,
4036 };
4037 
4038 static const char *const memcg1_stat_names[] = {
4039 	"cache",
4040 	"rss",
4041 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4042 	"rss_huge",
4043 #endif
4044 	"shmem",
4045 	"mapped_file",
4046 	"dirty",
4047 	"writeback",
4048 	"workingset_refault_anon",
4049 	"workingset_refault_file",
4050 	"swap",
4051 };
4052 
4053 /* Universal VM events cgroup1 shows, original sort order */
4054 static const unsigned int memcg1_events[] = {
4055 	PGPGIN,
4056 	PGPGOUT,
4057 	PGFAULT,
4058 	PGMAJFAULT,
4059 };
4060 
memcg_stat_show(struct seq_file * m,void * v)4061 static int memcg_stat_show(struct seq_file *m, void *v)
4062 {
4063 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4064 	unsigned long memory, memsw;
4065 	struct mem_cgroup *mi;
4066 	unsigned int i;
4067 
4068 	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
4069 
4070 	mem_cgroup_flush_stats();
4071 
4072 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4073 		unsigned long nr;
4074 
4075 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
4076 			continue;
4077 		nr = memcg_page_state_local(memcg, memcg1_stats[i]);
4078 		seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
4079 			   nr * memcg_page_state_unit(memcg1_stats[i]));
4080 	}
4081 
4082 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4083 		seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
4084 			   memcg_events_local(memcg, memcg1_events[i]));
4085 
4086 	for (i = 0; i < NR_LRU_LISTS; i++)
4087 		seq_printf(m, "%s %lu\n", lru_list_name(i),
4088 			   memcg_page_state_local(memcg, NR_LRU_BASE + i) *
4089 			   PAGE_SIZE);
4090 
4091 	/* Hierarchical information */
4092 	memory = memsw = PAGE_COUNTER_MAX;
4093 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
4094 		memory = min(memory, READ_ONCE(mi->memory.max));
4095 		memsw = min(memsw, READ_ONCE(mi->memsw.max));
4096 	}
4097 	seq_printf(m, "hierarchical_memory_limit %llu\n",
4098 		   (u64)memory * PAGE_SIZE);
4099 	if (do_memsw_account())
4100 		seq_printf(m, "hierarchical_memsw_limit %llu\n",
4101 			   (u64)memsw * PAGE_SIZE);
4102 
4103 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4104 		unsigned long nr;
4105 
4106 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
4107 			continue;
4108 		nr = memcg_page_state(memcg, memcg1_stats[i]);
4109 		seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
4110 			   (u64)nr * memcg_page_state_unit(memcg1_stats[i]));
4111 	}
4112 
4113 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4114 		seq_printf(m, "total_%s %llu\n",
4115 			   vm_event_name(memcg1_events[i]),
4116 			   (u64)memcg_events(memcg, memcg1_events[i]));
4117 
4118 	for (i = 0; i < NR_LRU_LISTS; i++)
4119 		seq_printf(m, "total_%s %llu\n", lru_list_name(i),
4120 			   (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4121 			   PAGE_SIZE);
4122 
4123 #ifdef CONFIG_DEBUG_VM
4124 	{
4125 		pg_data_t *pgdat;
4126 		struct mem_cgroup_per_node *mz;
4127 		unsigned long anon_cost = 0;
4128 		unsigned long file_cost = 0;
4129 
4130 		for_each_online_pgdat(pgdat) {
4131 			mz = memcg->nodeinfo[pgdat->node_id];
4132 
4133 			anon_cost += mz->lruvec.anon_cost;
4134 			file_cost += mz->lruvec.file_cost;
4135 		}
4136 		seq_printf(m, "anon_cost %lu\n", anon_cost);
4137 		seq_printf(m, "file_cost %lu\n", file_cost);
4138 	}
4139 #endif
4140 
4141 	return 0;
4142 }
4143 
mem_cgroup_swappiness_read(struct cgroup_subsys_state * css,struct cftype * cft)4144 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
4145 				      struct cftype *cft)
4146 {
4147 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4148 
4149 	return mem_cgroup_swappiness(memcg);
4150 }
4151 
mem_cgroup_swappiness_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)4152 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
4153 				       struct cftype *cft, u64 val)
4154 {
4155 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4156 
4157 	if (val > 200)
4158 		return -EINVAL;
4159 
4160 	if (!mem_cgroup_is_root(memcg))
4161 		memcg->swappiness = val;
4162 	else
4163 		vm_swappiness = val;
4164 
4165 	return 0;
4166 }
4167 
__mem_cgroup_threshold(struct mem_cgroup * memcg,bool swap)4168 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4169 {
4170 	struct mem_cgroup_threshold_ary *t;
4171 	unsigned long usage;
4172 	int i;
4173 
4174 	rcu_read_lock();
4175 	if (!swap)
4176 		t = rcu_dereference(memcg->thresholds.primary);
4177 	else
4178 		t = rcu_dereference(memcg->memsw_thresholds.primary);
4179 
4180 	if (!t)
4181 		goto unlock;
4182 
4183 	usage = mem_cgroup_usage(memcg, swap);
4184 
4185 	/*
4186 	 * current_threshold points to threshold just below or equal to usage.
4187 	 * If it's not true, a threshold was crossed after last
4188 	 * call of __mem_cgroup_threshold().
4189 	 */
4190 	i = t->current_threshold;
4191 
4192 	/*
4193 	 * Iterate backward over array of thresholds starting from
4194 	 * current_threshold and check if a threshold is crossed.
4195 	 * If none of thresholds below usage is crossed, we read
4196 	 * only one element of the array here.
4197 	 */
4198 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
4199 		eventfd_signal(t->entries[i].eventfd, 1);
4200 
4201 	/* i = current_threshold + 1 */
4202 	i++;
4203 
4204 	/*
4205 	 * Iterate forward over array of thresholds starting from
4206 	 * current_threshold+1 and check if a threshold is crossed.
4207 	 * If none of thresholds above usage is crossed, we read
4208 	 * only one element of the array here.
4209 	 */
4210 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
4211 		eventfd_signal(t->entries[i].eventfd, 1);
4212 
4213 	/* Update current_threshold */
4214 	t->current_threshold = i - 1;
4215 unlock:
4216 	rcu_read_unlock();
4217 }
4218 
mem_cgroup_threshold(struct mem_cgroup * memcg)4219 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4220 {
4221 	while (memcg) {
4222 		__mem_cgroup_threshold(memcg, false);
4223 		if (do_memsw_account())
4224 			__mem_cgroup_threshold(memcg, true);
4225 
4226 		memcg = parent_mem_cgroup(memcg);
4227 	}
4228 }
4229 
compare_thresholds(const void * a,const void * b)4230 static int compare_thresholds(const void *a, const void *b)
4231 {
4232 	const struct mem_cgroup_threshold *_a = a;
4233 	const struct mem_cgroup_threshold *_b = b;
4234 
4235 	if (_a->threshold > _b->threshold)
4236 		return 1;
4237 
4238 	if (_a->threshold < _b->threshold)
4239 		return -1;
4240 
4241 	return 0;
4242 }
4243 
mem_cgroup_oom_notify_cb(struct mem_cgroup * memcg)4244 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4245 {
4246 	struct mem_cgroup_eventfd_list *ev;
4247 
4248 	spin_lock(&memcg_oom_lock);
4249 
4250 	list_for_each_entry(ev, &memcg->oom_notify, list)
4251 		eventfd_signal(ev->eventfd, 1);
4252 
4253 	spin_unlock(&memcg_oom_lock);
4254 	return 0;
4255 }
4256 
mem_cgroup_oom_notify(struct mem_cgroup * memcg)4257 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4258 {
4259 	struct mem_cgroup *iter;
4260 
4261 	for_each_mem_cgroup_tree(iter, memcg)
4262 		mem_cgroup_oom_notify_cb(iter);
4263 }
4264 
__mem_cgroup_usage_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args,enum res_type type)4265 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4266 	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
4267 {
4268 	struct mem_cgroup_thresholds *thresholds;
4269 	struct mem_cgroup_threshold_ary *new;
4270 	unsigned long threshold;
4271 	unsigned long usage;
4272 	int i, size, ret;
4273 
4274 	ret = page_counter_memparse(args, "-1", &threshold);
4275 	if (ret)
4276 		return ret;
4277 
4278 	mutex_lock(&memcg->thresholds_lock);
4279 
4280 	if (type == _MEM) {
4281 		thresholds = &memcg->thresholds;
4282 		usage = mem_cgroup_usage(memcg, false);
4283 	} else if (type == _MEMSWAP) {
4284 		thresholds = &memcg->memsw_thresholds;
4285 		usage = mem_cgroup_usage(memcg, true);
4286 	} else
4287 		BUG();
4288 
4289 	/* Check if a threshold crossed before adding a new one */
4290 	if (thresholds->primary)
4291 		__mem_cgroup_threshold(memcg, type == _MEMSWAP);
4292 
4293 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4294 
4295 	/* Allocate memory for new array of thresholds */
4296 	new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
4297 	if (!new) {
4298 		ret = -ENOMEM;
4299 		goto unlock;
4300 	}
4301 	new->size = size;
4302 
4303 	/* Copy thresholds (if any) to new array */
4304 	if (thresholds->primary)
4305 		memcpy(new->entries, thresholds->primary->entries,
4306 		       flex_array_size(new, entries, size - 1));
4307 
4308 	/* Add new threshold */
4309 	new->entries[size - 1].eventfd = eventfd;
4310 	new->entries[size - 1].threshold = threshold;
4311 
4312 	/* Sort thresholds. Registering of new threshold isn't time-critical */
4313 	sort(new->entries, size, sizeof(*new->entries),
4314 			compare_thresholds, NULL);
4315 
4316 	/* Find current threshold */
4317 	new->current_threshold = -1;
4318 	for (i = 0; i < size; i++) {
4319 		if (new->entries[i].threshold <= usage) {
4320 			/*
4321 			 * new->current_threshold will not be used until
4322 			 * rcu_assign_pointer(), so it's safe to increment
4323 			 * it here.
4324 			 */
4325 			++new->current_threshold;
4326 		} else
4327 			break;
4328 	}
4329 
4330 	/* Free old spare buffer and save old primary buffer as spare */
4331 	kfree(thresholds->spare);
4332 	thresholds->spare = thresholds->primary;
4333 
4334 	rcu_assign_pointer(thresholds->primary, new);
4335 
4336 	/* To be sure that nobody uses thresholds */
4337 	synchronize_rcu();
4338 
4339 unlock:
4340 	mutex_unlock(&memcg->thresholds_lock);
4341 
4342 	return ret;
4343 }
4344 
mem_cgroup_usage_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args)4345 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4346 	struct eventfd_ctx *eventfd, const char *args)
4347 {
4348 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4349 }
4350 
memsw_cgroup_usage_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args)4351 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4352 	struct eventfd_ctx *eventfd, const char *args)
4353 {
4354 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4355 }
4356 
__mem_cgroup_usage_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,enum res_type type)4357 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4358 	struct eventfd_ctx *eventfd, enum res_type type)
4359 {
4360 	struct mem_cgroup_thresholds *thresholds;
4361 	struct mem_cgroup_threshold_ary *new;
4362 	unsigned long usage;
4363 	int i, j, size, entries;
4364 
4365 	mutex_lock(&memcg->thresholds_lock);
4366 
4367 	if (type == _MEM) {
4368 		thresholds = &memcg->thresholds;
4369 		usage = mem_cgroup_usage(memcg, false);
4370 	} else if (type == _MEMSWAP) {
4371 		thresholds = &memcg->memsw_thresholds;
4372 		usage = mem_cgroup_usage(memcg, true);
4373 	} else
4374 		BUG();
4375 
4376 	if (!thresholds->primary)
4377 		goto unlock;
4378 
4379 	/* Check if a threshold crossed before removing */
4380 	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
4381 
4382 	/* Calculate new number of threshold */
4383 	size = entries = 0;
4384 	for (i = 0; i < thresholds->primary->size; i++) {
4385 		if (thresholds->primary->entries[i].eventfd != eventfd)
4386 			size++;
4387 		else
4388 			entries++;
4389 	}
4390 
4391 	new = thresholds->spare;
4392 
4393 	/* If no items related to eventfd have been cleared, nothing to do */
4394 	if (!entries)
4395 		goto unlock;
4396 
4397 	/* Set thresholds array to NULL if we don't have thresholds */
4398 	if (!size) {
4399 		kfree(new);
4400 		new = NULL;
4401 		goto swap_buffers;
4402 	}
4403 
4404 	new->size = size;
4405 
4406 	/* Copy thresholds and find current threshold */
4407 	new->current_threshold = -1;
4408 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4409 		if (thresholds->primary->entries[i].eventfd == eventfd)
4410 			continue;
4411 
4412 		new->entries[j] = thresholds->primary->entries[i];
4413 		if (new->entries[j].threshold <= usage) {
4414 			/*
4415 			 * new->current_threshold will not be used
4416 			 * until rcu_assign_pointer(), so it's safe to increment
4417 			 * it here.
4418 			 */
4419 			++new->current_threshold;
4420 		}
4421 		j++;
4422 	}
4423 
4424 swap_buffers:
4425 	/* Swap primary and spare array */
4426 	thresholds->spare = thresholds->primary;
4427 
4428 	rcu_assign_pointer(thresholds->primary, new);
4429 
4430 	/* To be sure that nobody uses thresholds */
4431 	synchronize_rcu();
4432 
4433 	/* If all events are unregistered, free the spare array */
4434 	if (!new) {
4435 		kfree(thresholds->spare);
4436 		thresholds->spare = NULL;
4437 	}
4438 unlock:
4439 	mutex_unlock(&memcg->thresholds_lock);
4440 }
4441 
mem_cgroup_usage_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd)4442 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4443 	struct eventfd_ctx *eventfd)
4444 {
4445 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4446 }
4447 
memsw_cgroup_usage_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd)4448 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4449 	struct eventfd_ctx *eventfd)
4450 {
4451 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4452 }
4453 
mem_cgroup_oom_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args)4454 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4455 	struct eventfd_ctx *eventfd, const char *args)
4456 {
4457 	struct mem_cgroup_eventfd_list *event;
4458 
4459 	event = kmalloc(sizeof(*event),	GFP_KERNEL);
4460 	if (!event)
4461 		return -ENOMEM;
4462 
4463 	spin_lock(&memcg_oom_lock);
4464 
4465 	event->eventfd = eventfd;
4466 	list_add(&event->list, &memcg->oom_notify);
4467 
4468 	/* already in OOM ? */
4469 	if (memcg->under_oom)
4470 		eventfd_signal(eventfd, 1);
4471 	spin_unlock(&memcg_oom_lock);
4472 
4473 	return 0;
4474 }
4475 
mem_cgroup_oom_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd)4476 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4477 	struct eventfd_ctx *eventfd)
4478 {
4479 	struct mem_cgroup_eventfd_list *ev, *tmp;
4480 
4481 	spin_lock(&memcg_oom_lock);
4482 
4483 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
4484 		if (ev->eventfd == eventfd) {
4485 			list_del(&ev->list);
4486 			kfree(ev);
4487 		}
4488 	}
4489 
4490 	spin_unlock(&memcg_oom_lock);
4491 }
4492 
mem_cgroup_oom_control_read(struct seq_file * sf,void * v)4493 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
4494 {
4495 	struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4496 
4497 	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
4498 	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4499 	seq_printf(sf, "oom_kill %lu\n",
4500 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4501 	return 0;
4502 }
4503 
mem_cgroup_oom_control_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)4504 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
4505 	struct cftype *cft, u64 val)
4506 {
4507 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4508 
4509 	/* cannot set to root cgroup and only 0 and 1 are allowed */
4510 	if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
4511 		return -EINVAL;
4512 
4513 	memcg->oom_kill_disable = val;
4514 	if (!val)
4515 		memcg_oom_recover(memcg);
4516 
4517 	return 0;
4518 }
4519 
4520 #ifdef CONFIG_CGROUP_WRITEBACK
4521 
4522 #include <trace/events/writeback.h>
4523 
memcg_wb_domain_init(struct mem_cgroup * memcg,gfp_t gfp)4524 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4525 {
4526 	return wb_domain_init(&memcg->cgwb_domain, gfp);
4527 }
4528 
memcg_wb_domain_exit(struct mem_cgroup * memcg)4529 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4530 {
4531 	wb_domain_exit(&memcg->cgwb_domain);
4532 }
4533 
memcg_wb_domain_size_changed(struct mem_cgroup * memcg)4534 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4535 {
4536 	wb_domain_size_changed(&memcg->cgwb_domain);
4537 }
4538 
mem_cgroup_wb_domain(struct bdi_writeback * wb)4539 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4540 {
4541 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4542 
4543 	if (!memcg->css.parent)
4544 		return NULL;
4545 
4546 	return &memcg->cgwb_domain;
4547 }
4548 
4549 /**
4550  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4551  * @wb: bdi_writeback in question
4552  * @pfilepages: out parameter for number of file pages
4553  * @pheadroom: out parameter for number of allocatable pages according to memcg
4554  * @pdirty: out parameter for number of dirty pages
4555  * @pwriteback: out parameter for number of pages under writeback
4556  *
4557  * Determine the numbers of file, headroom, dirty, and writeback pages in
4558  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
4559  * is a bit more involved.
4560  *
4561  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
4562  * headroom is calculated as the lowest headroom of itself and the
4563  * ancestors.  Note that this doesn't consider the actual amount of
4564  * available memory in the system.  The caller should further cap
4565  * *@pheadroom accordingly.
4566  */
mem_cgroup_wb_stats(struct bdi_writeback * wb,unsigned long * pfilepages,unsigned long * pheadroom,unsigned long * pdirty,unsigned long * pwriteback)4567 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4568 			 unsigned long *pheadroom, unsigned long *pdirty,
4569 			 unsigned long *pwriteback)
4570 {
4571 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4572 	struct mem_cgroup *parent;
4573 
4574 	mem_cgroup_flush_stats();
4575 
4576 	*pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
4577 	*pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
4578 	*pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
4579 			memcg_page_state(memcg, NR_ACTIVE_FILE);
4580 
4581 	*pheadroom = PAGE_COUNTER_MAX;
4582 	while ((parent = parent_mem_cgroup(memcg))) {
4583 		unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
4584 					    READ_ONCE(memcg->memory.high));
4585 		unsigned long used = page_counter_read(&memcg->memory);
4586 
4587 		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
4588 		memcg = parent;
4589 	}
4590 }
4591 
4592 /*
4593  * Foreign dirty flushing
4594  *
4595  * There's an inherent mismatch between memcg and writeback.  The former
4596  * tracks ownership per-page while the latter per-inode.  This was a
4597  * deliberate design decision because honoring per-page ownership in the
4598  * writeback path is complicated, may lead to higher CPU and IO overheads
4599  * and deemed unnecessary given that write-sharing an inode across
4600  * different cgroups isn't a common use-case.
4601  *
4602  * Combined with inode majority-writer ownership switching, this works well
4603  * enough in most cases but there are some pathological cases.  For
4604  * example, let's say there are two cgroups A and B which keep writing to
4605  * different but confined parts of the same inode.  B owns the inode and
4606  * A's memory is limited far below B's.  A's dirty ratio can rise enough to
4607  * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
4608  * triggering background writeback.  A will be slowed down without a way to
4609  * make writeback of the dirty pages happen.
4610  *
4611  * Conditions like the above can lead to a cgroup getting repeatedly and
4612  * severely throttled after making some progress after each
4613  * dirty_expire_interval while the underlying IO device is almost
4614  * completely idle.
4615  *
4616  * Solving this problem completely requires matching the ownership tracking
4617  * granularities between memcg and writeback in either direction.  However,
4618  * the more egregious behaviors can be avoided by simply remembering the
4619  * most recent foreign dirtying events and initiating remote flushes on
4620  * them when local writeback isn't enough to keep the memory clean enough.
4621  *
4622  * The following two functions implement such mechanism.  When a foreign
4623  * page - a page whose memcg and writeback ownerships don't match - is
4624  * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
4625  * bdi_writeback on the page owning memcg.  When balance_dirty_pages()
4626  * decides that the memcg needs to sleep due to high dirty ratio, it calls
4627  * mem_cgroup_flush_foreign() which queues writeback on the recorded
4628  * foreign bdi_writebacks which haven't expired.  Both the numbers of
4629  * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
4630  * limited to MEMCG_CGWB_FRN_CNT.
4631  *
4632  * The mechanism only remembers IDs and doesn't hold any object references.
4633  * As being wrong occasionally doesn't matter, updates and accesses to the
4634  * records are lockless and racy.
4635  */
mem_cgroup_track_foreign_dirty_slowpath(struct folio * folio,struct bdi_writeback * wb)4636 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
4637 					     struct bdi_writeback *wb)
4638 {
4639 	struct mem_cgroup *memcg = folio_memcg(folio);
4640 	struct memcg_cgwb_frn *frn;
4641 	u64 now = get_jiffies_64();
4642 	u64 oldest_at = now;
4643 	int oldest = -1;
4644 	int i;
4645 
4646 	trace_track_foreign_dirty(folio, wb);
4647 
4648 	/*
4649 	 * Pick the slot to use.  If there is already a slot for @wb, keep
4650 	 * using it.  If not replace the oldest one which isn't being
4651 	 * written out.
4652 	 */
4653 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4654 		frn = &memcg->cgwb_frn[i];
4655 		if (frn->bdi_id == wb->bdi->id &&
4656 		    frn->memcg_id == wb->memcg_css->id)
4657 			break;
4658 		if (time_before64(frn->at, oldest_at) &&
4659 		    atomic_read(&frn->done.cnt) == 1) {
4660 			oldest = i;
4661 			oldest_at = frn->at;
4662 		}
4663 	}
4664 
4665 	if (i < MEMCG_CGWB_FRN_CNT) {
4666 		/*
4667 		 * Re-using an existing one.  Update timestamp lazily to
4668 		 * avoid making the cacheline hot.  We want them to be
4669 		 * reasonably up-to-date and significantly shorter than
4670 		 * dirty_expire_interval as that's what expires the record.
4671 		 * Use the shorter of 1s and dirty_expire_interval / 8.
4672 		 */
4673 		unsigned long update_intv =
4674 			min_t(unsigned long, HZ,
4675 			      msecs_to_jiffies(dirty_expire_interval * 10) / 8);
4676 
4677 		if (time_before64(frn->at, now - update_intv))
4678 			frn->at = now;
4679 	} else if (oldest >= 0) {
4680 		/* replace the oldest free one */
4681 		frn = &memcg->cgwb_frn[oldest];
4682 		frn->bdi_id = wb->bdi->id;
4683 		frn->memcg_id = wb->memcg_css->id;
4684 		frn->at = now;
4685 	}
4686 }
4687 
4688 /* issue foreign writeback flushes for recorded foreign dirtying events */
mem_cgroup_flush_foreign(struct bdi_writeback * wb)4689 void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
4690 {
4691 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4692 	unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
4693 	u64 now = jiffies_64;
4694 	int i;
4695 
4696 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4697 		struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4698 
4699 		/*
4700 		 * If the record is older than dirty_expire_interval,
4701 		 * writeback on it has already started.  No need to kick it
4702 		 * off again.  Also, don't start a new one if there's
4703 		 * already one in flight.
4704 		 */
4705 		if (time_after64(frn->at, now - intv) &&
4706 		    atomic_read(&frn->done.cnt) == 1) {
4707 			frn->at = 0;
4708 			trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
4709 			cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id,
4710 					       WB_REASON_FOREIGN_FLUSH,
4711 					       &frn->done);
4712 		}
4713 	}
4714 }
4715 
4716 #else	/* CONFIG_CGROUP_WRITEBACK */
4717 
memcg_wb_domain_init(struct mem_cgroup * memcg,gfp_t gfp)4718 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4719 {
4720 	return 0;
4721 }
4722 
memcg_wb_domain_exit(struct mem_cgroup * memcg)4723 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4724 {
4725 }
4726 
memcg_wb_domain_size_changed(struct mem_cgroup * memcg)4727 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4728 {
4729 }
4730 
4731 #endif	/* CONFIG_CGROUP_WRITEBACK */
4732 
4733 /*
4734  * DO NOT USE IN NEW FILES.
4735  *
4736  * "cgroup.event_control" implementation.
4737  *
4738  * This is way over-engineered.  It tries to support fully configurable
4739  * events for each user.  Such level of flexibility is completely
4740  * unnecessary especially in the light of the planned unified hierarchy.
4741  *
4742  * Please deprecate this and replace with something simpler if at all
4743  * possible.
4744  */
4745 
4746 /*
4747  * Unregister event and free resources.
4748  *
4749  * Gets called from workqueue.
4750  */
memcg_event_remove(struct work_struct * work)4751 static void memcg_event_remove(struct work_struct *work)
4752 {
4753 	struct mem_cgroup_event *event =
4754 		container_of(work, struct mem_cgroup_event, remove);
4755 	struct mem_cgroup *memcg = event->memcg;
4756 
4757 	remove_wait_queue(event->wqh, &event->wait);
4758 
4759 	event->unregister_event(memcg, event->eventfd);
4760 
4761 	/* Notify userspace the event is going away. */
4762 	eventfd_signal(event->eventfd, 1);
4763 
4764 	eventfd_ctx_put(event->eventfd);
4765 	kfree(event);
4766 	css_put(&memcg->css);
4767 }
4768 
4769 /*
4770  * Gets called on EPOLLHUP on eventfd when user closes it.
4771  *
4772  * Called with wqh->lock held and interrupts disabled.
4773  */
memcg_event_wake(wait_queue_entry_t * wait,unsigned mode,int sync,void * key)4774 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
4775 			    int sync, void *key)
4776 {
4777 	struct mem_cgroup_event *event =
4778 		container_of(wait, struct mem_cgroup_event, wait);
4779 	struct mem_cgroup *memcg = event->memcg;
4780 	__poll_t flags = key_to_poll(key);
4781 
4782 	if (flags & EPOLLHUP) {
4783 		/*
4784 		 * If the event has been detached at cgroup removal, we
4785 		 * can simply return knowing the other side will cleanup
4786 		 * for us.
4787 		 *
4788 		 * We can't race against event freeing since the other
4789 		 * side will require wqh->lock via remove_wait_queue(),
4790 		 * which we hold.
4791 		 */
4792 		spin_lock(&memcg->event_list_lock);
4793 		if (!list_empty(&event->list)) {
4794 			list_del_init(&event->list);
4795 			/*
4796 			 * We are in atomic context, but cgroup_event_remove()
4797 			 * may sleep, so we have to call it in workqueue.
4798 			 */
4799 			schedule_work(&event->remove);
4800 		}
4801 		spin_unlock(&memcg->event_list_lock);
4802 	}
4803 
4804 	return 0;
4805 }
4806 
memcg_event_ptable_queue_proc(struct file * file,wait_queue_head_t * wqh,poll_table * pt)4807 static void memcg_event_ptable_queue_proc(struct file *file,
4808 		wait_queue_head_t *wqh, poll_table *pt)
4809 {
4810 	struct mem_cgroup_event *event =
4811 		container_of(pt, struct mem_cgroup_event, pt);
4812 
4813 	event->wqh = wqh;
4814 	add_wait_queue(wqh, &event->wait);
4815 }
4816 
4817 /*
4818  * DO NOT USE IN NEW FILES.
4819  *
4820  * Parse input and register new cgroup event handler.
4821  *
4822  * Input must be in format '<event_fd> <control_fd> <args>'.
4823  * Interpretation of args is defined by control file implementation.
4824  */
memcg_write_event_control(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)4825 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4826 					 char *buf, size_t nbytes, loff_t off)
4827 {
4828 	struct cgroup_subsys_state *css = of_css(of);
4829 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4830 	struct mem_cgroup_event *event;
4831 	struct cgroup_subsys_state *cfile_css;
4832 	unsigned int efd, cfd;
4833 	struct fd efile;
4834 	struct fd cfile;
4835 	struct dentry *cdentry;
4836 	const char *name;
4837 	char *endp;
4838 	int ret;
4839 
4840 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
4841 		return -EOPNOTSUPP;
4842 
4843 	buf = strstrip(buf);
4844 
4845 	efd = simple_strtoul(buf, &endp, 10);
4846 	if (*endp != ' ')
4847 		return -EINVAL;
4848 	buf = endp + 1;
4849 
4850 	cfd = simple_strtoul(buf, &endp, 10);
4851 	if ((*endp != ' ') && (*endp != '\0'))
4852 		return -EINVAL;
4853 	buf = endp + 1;
4854 
4855 	event = kzalloc(sizeof(*event), GFP_KERNEL);
4856 	if (!event)
4857 		return -ENOMEM;
4858 
4859 	event->memcg = memcg;
4860 	INIT_LIST_HEAD(&event->list);
4861 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4862 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4863 	INIT_WORK(&event->remove, memcg_event_remove);
4864 
4865 	efile = fdget(efd);
4866 	if (!efile.file) {
4867 		ret = -EBADF;
4868 		goto out_kfree;
4869 	}
4870 
4871 	event->eventfd = eventfd_ctx_fileget(efile.file);
4872 	if (IS_ERR(event->eventfd)) {
4873 		ret = PTR_ERR(event->eventfd);
4874 		goto out_put_efile;
4875 	}
4876 
4877 	cfile = fdget(cfd);
4878 	if (!cfile.file) {
4879 		ret = -EBADF;
4880 		goto out_put_eventfd;
4881 	}
4882 
4883 	/* the process need read permission on control file */
4884 	/* AV: shouldn't we check that it's been opened for read instead? */
4885 	ret = file_permission(cfile.file, MAY_READ);
4886 	if (ret < 0)
4887 		goto out_put_cfile;
4888 
4889 	/*
4890 	 * The control file must be a regular cgroup1 file. As a regular cgroup
4891 	 * file can't be renamed, it's safe to access its name afterwards.
4892 	 */
4893 	cdentry = cfile.file->f_path.dentry;
4894 	if (cdentry->d_sb->s_type != &cgroup_fs_type || !d_is_reg(cdentry)) {
4895 		ret = -EINVAL;
4896 		goto out_put_cfile;
4897 	}
4898 
4899 	/*
4900 	 * Determine the event callbacks and set them in @event.  This used
4901 	 * to be done via struct cftype but cgroup core no longer knows
4902 	 * about these events.  The following is crude but the whole thing
4903 	 * is for compatibility anyway.
4904 	 *
4905 	 * DO NOT ADD NEW FILES.
4906 	 */
4907 	name = cdentry->d_name.name;
4908 
4909 	if (!strcmp(name, "memory.usage_in_bytes")) {
4910 		event->register_event = mem_cgroup_usage_register_event;
4911 		event->unregister_event = mem_cgroup_usage_unregister_event;
4912 	} else if (!strcmp(name, "memory.oom_control")) {
4913 		event->register_event = mem_cgroup_oom_register_event;
4914 		event->unregister_event = mem_cgroup_oom_unregister_event;
4915 	} else if (!strcmp(name, "memory.pressure_level")) {
4916 		event->register_event = vmpressure_register_event;
4917 		event->unregister_event = vmpressure_unregister_event;
4918 	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4919 		event->register_event = memsw_cgroup_usage_register_event;
4920 		event->unregister_event = memsw_cgroup_usage_unregister_event;
4921 	} else {
4922 		ret = -EINVAL;
4923 		goto out_put_cfile;
4924 	}
4925 
4926 	/*
4927 	 * Verify @cfile should belong to @css.  Also, remaining events are
4928 	 * automatically removed on cgroup destruction but the removal is
4929 	 * asynchronous, so take an extra ref on @css.
4930 	 */
4931 	cfile_css = css_tryget_online_from_dir(cdentry->d_parent,
4932 					       &memory_cgrp_subsys);
4933 	ret = -EINVAL;
4934 	if (IS_ERR(cfile_css))
4935 		goto out_put_cfile;
4936 	if (cfile_css != css) {
4937 		css_put(cfile_css);
4938 		goto out_put_cfile;
4939 	}
4940 
4941 	ret = event->register_event(memcg, event->eventfd, buf);
4942 	if (ret)
4943 		goto out_put_css;
4944 
4945 	vfs_poll(efile.file, &event->pt);
4946 
4947 	spin_lock_irq(&memcg->event_list_lock);
4948 	list_add(&event->list, &memcg->event_list);
4949 	spin_unlock_irq(&memcg->event_list_lock);
4950 
4951 	fdput(cfile);
4952 	fdput(efile);
4953 
4954 	return nbytes;
4955 
4956 out_put_css:
4957 	css_put(css);
4958 out_put_cfile:
4959 	fdput(cfile);
4960 out_put_eventfd:
4961 	eventfd_ctx_put(event->eventfd);
4962 out_put_efile:
4963 	fdput(efile);
4964 out_kfree:
4965 	kfree(event);
4966 
4967 	return ret;
4968 }
4969 
4970 #if defined(CONFIG_MEMCG_KMEM) && (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
mem_cgroup_slab_show(struct seq_file * m,void * p)4971 static int mem_cgroup_slab_show(struct seq_file *m, void *p)
4972 {
4973 	/*
4974 	 * Deprecated.
4975 	 * Please, take a look at tools/cgroup/memcg_slabinfo.py .
4976 	 */
4977 	return 0;
4978 }
4979 #endif
4980 
4981 static struct cftype mem_cgroup_legacy_files[] = {
4982 	{
4983 		.name = "usage_in_bytes",
4984 		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4985 		.read_u64 = mem_cgroup_read_u64,
4986 	},
4987 	{
4988 		.name = "max_usage_in_bytes",
4989 		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4990 		.write = mem_cgroup_reset,
4991 		.read_u64 = mem_cgroup_read_u64,
4992 	},
4993 	{
4994 		.name = "limit_in_bytes",
4995 		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4996 		.write = mem_cgroup_write,
4997 		.read_u64 = mem_cgroup_read_u64,
4998 	},
4999 	{
5000 		.name = "soft_limit_in_bytes",
5001 		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
5002 		.write = mem_cgroup_write,
5003 		.read_u64 = mem_cgroup_read_u64,
5004 	},
5005 	{
5006 		.name = "failcnt",
5007 		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
5008 		.write = mem_cgroup_reset,
5009 		.read_u64 = mem_cgroup_read_u64,
5010 	},
5011 	{
5012 		.name = "stat",
5013 		.seq_show = memcg_stat_show,
5014 	},
5015 	{
5016 		.name = "force_empty",
5017 		.write = mem_cgroup_force_empty_write,
5018 	},
5019 	{
5020 		.name = "use_hierarchy",
5021 		.write_u64 = mem_cgroup_hierarchy_write,
5022 		.read_u64 = mem_cgroup_hierarchy_read,
5023 	},
5024 	{
5025 		.name = "cgroup.event_control",		/* XXX: for compat */
5026 		.write = memcg_write_event_control,
5027 		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
5028 	},
5029 	{
5030 		.name = "swappiness",
5031 		.read_u64 = mem_cgroup_swappiness_read,
5032 		.write_u64 = mem_cgroup_swappiness_write,
5033 	},
5034 	{
5035 		.name = "move_charge_at_immigrate",
5036 		.read_u64 = mem_cgroup_move_charge_read,
5037 		.write_u64 = mem_cgroup_move_charge_write,
5038 	},
5039 	{
5040 		.name = "oom_control",
5041 		.seq_show = mem_cgroup_oom_control_read,
5042 		.write_u64 = mem_cgroup_oom_control_write,
5043 	},
5044 	{
5045 		.name = "pressure_level",
5046 	},
5047 #ifdef CONFIG_NUMA
5048 	{
5049 		.name = "numa_stat",
5050 		.seq_show = memcg_numa_stat_show,
5051 	},
5052 #endif
5053 	{
5054 		.name = "kmem.limit_in_bytes",
5055 		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
5056 		.write = mem_cgroup_write,
5057 		.read_u64 = mem_cgroup_read_u64,
5058 	},
5059 	{
5060 		.name = "kmem.usage_in_bytes",
5061 		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
5062 		.read_u64 = mem_cgroup_read_u64,
5063 	},
5064 	{
5065 		.name = "kmem.failcnt",
5066 		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
5067 		.write = mem_cgroup_reset,
5068 		.read_u64 = mem_cgroup_read_u64,
5069 	},
5070 	{
5071 		.name = "kmem.max_usage_in_bytes",
5072 		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
5073 		.write = mem_cgroup_reset,
5074 		.read_u64 = mem_cgroup_read_u64,
5075 	},
5076 #if defined(CONFIG_MEMCG_KMEM) && \
5077 	(defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
5078 	{
5079 		.name = "kmem.slabinfo",
5080 		.seq_show = mem_cgroup_slab_show,
5081 	},
5082 #endif
5083 	{
5084 		.name = "kmem.tcp.limit_in_bytes",
5085 		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
5086 		.write = mem_cgroup_write,
5087 		.read_u64 = mem_cgroup_read_u64,
5088 	},
5089 	{
5090 		.name = "kmem.tcp.usage_in_bytes",
5091 		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
5092 		.read_u64 = mem_cgroup_read_u64,
5093 	},
5094 	{
5095 		.name = "kmem.tcp.failcnt",
5096 		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
5097 		.write = mem_cgroup_reset,
5098 		.read_u64 = mem_cgroup_read_u64,
5099 	},
5100 	{
5101 		.name = "kmem.tcp.max_usage_in_bytes",
5102 		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
5103 		.write = mem_cgroup_reset,
5104 		.read_u64 = mem_cgroup_read_u64,
5105 	},
5106 	{ },	/* terminate */
5107 };
5108 
5109 /*
5110  * Private memory cgroup IDR
5111  *
5112  * Swap-out records and page cache shadow entries need to store memcg
5113  * references in constrained space, so we maintain an ID space that is
5114  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
5115  * memory-controlled cgroups to 64k.
5116  *
5117  * However, there usually are many references to the offline CSS after
5118  * the cgroup has been destroyed, such as page cache or reclaimable
5119  * slab objects, that don't need to hang on to the ID. We want to keep
5120  * those dead CSS from occupying IDs, or we might quickly exhaust the
5121  * relatively small ID space and prevent the creation of new cgroups
5122  * even when there are much fewer than 64k cgroups - possibly none.
5123  *
5124  * Maintain a private 16-bit ID space for memcg, and allow the ID to
5125  * be freed and recycled when it's no longer needed, which is usually
5126  * when the CSS is offlined.
5127  *
5128  * The only exception to that are records of swapped out tmpfs/shmem
5129  * pages that need to be attributed to live ancestors on swapin. But
5130  * those references are manageable from userspace.
5131  */
5132 
5133 static DEFINE_IDR(mem_cgroup_idr);
5134 
mem_cgroup_id_remove(struct mem_cgroup * memcg)5135 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5136 {
5137 	if (memcg->id.id > 0) {
5138 		idr_remove(&mem_cgroup_idr, memcg->id.id);
5139 		memcg->id.id = 0;
5140 	}
5141 }
5142 
mem_cgroup_id_get_many(struct mem_cgroup * memcg,unsigned int n)5143 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5144 						  unsigned int n)
5145 {
5146 	refcount_add(n, &memcg->id.ref);
5147 }
5148 
mem_cgroup_id_put_many(struct mem_cgroup * memcg,unsigned int n)5149 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5150 {
5151 	if (refcount_sub_and_test(n, &memcg->id.ref)) {
5152 		mem_cgroup_id_remove(memcg);
5153 
5154 		/* Memcg ID pins CSS */
5155 		css_put(&memcg->css);
5156 	}
5157 }
5158 
mem_cgroup_id_put(struct mem_cgroup * memcg)5159 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5160 {
5161 	mem_cgroup_id_put_many(memcg, 1);
5162 }
5163 
5164 /**
5165  * mem_cgroup_from_id - look up a memcg from a memcg id
5166  * @id: the memcg id to look up
5167  *
5168  * Caller must hold rcu_read_lock().
5169  */
mem_cgroup_from_id(unsigned short id)5170 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
5171 {
5172 	WARN_ON_ONCE(!rcu_read_lock_held());
5173 	return idr_find(&mem_cgroup_idr, id);
5174 }
5175 
5176 #ifdef CONFIG_SHRINKER_DEBUG
mem_cgroup_get_from_ino(unsigned long ino)5177 struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino)
5178 {
5179 	struct cgroup *cgrp;
5180 	struct cgroup_subsys_state *css;
5181 	struct mem_cgroup *memcg;
5182 
5183 	cgrp = cgroup_get_from_id(ino);
5184 	if (IS_ERR(cgrp))
5185 		return ERR_CAST(cgrp);
5186 
5187 	css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys);
5188 	if (css)
5189 		memcg = container_of(css, struct mem_cgroup, css);
5190 	else
5191 		memcg = ERR_PTR(-ENOENT);
5192 
5193 	cgroup_put(cgrp);
5194 
5195 	return memcg;
5196 }
5197 #endif
5198 
alloc_mem_cgroup_per_node_info(struct mem_cgroup * memcg,int node)5199 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5200 {
5201 	struct mem_cgroup_per_node *pn;
5202 
5203 	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, node);
5204 	if (!pn)
5205 		return 1;
5206 
5207 	pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu,
5208 						   GFP_KERNEL_ACCOUNT);
5209 	if (!pn->lruvec_stats_percpu) {
5210 		kfree(pn);
5211 		return 1;
5212 	}
5213 
5214 	lruvec_init(&pn->lruvec);
5215 	pn->memcg = memcg;
5216 
5217 	memcg->nodeinfo[node] = pn;
5218 	return 0;
5219 }
5220 
free_mem_cgroup_per_node_info(struct mem_cgroup * memcg,int node)5221 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5222 {
5223 	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5224 
5225 	if (!pn)
5226 		return;
5227 
5228 	free_percpu(pn->lruvec_stats_percpu);
5229 	kfree(pn);
5230 }
5231 
__mem_cgroup_free(struct mem_cgroup * memcg)5232 static void __mem_cgroup_free(struct mem_cgroup *memcg)
5233 {
5234 	int node;
5235 
5236 	for_each_node(node)
5237 		free_mem_cgroup_per_node_info(memcg, node);
5238 	kfree(memcg->vmstats);
5239 	free_percpu(memcg->vmstats_percpu);
5240 	kfree(memcg);
5241 }
5242 
mem_cgroup_free(struct mem_cgroup * memcg)5243 static void mem_cgroup_free(struct mem_cgroup *memcg)
5244 {
5245 	lru_gen_exit_memcg(memcg);
5246 	memcg_wb_domain_exit(memcg);
5247 	__mem_cgroup_free(memcg);
5248 }
5249 
mem_cgroup_alloc(void)5250 static struct mem_cgroup *mem_cgroup_alloc(void)
5251 {
5252 	struct mem_cgroup *memcg;
5253 	int node;
5254 	int __maybe_unused i;
5255 	long error = -ENOMEM;
5256 
5257 	memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL);
5258 	if (!memcg)
5259 		return ERR_PTR(error);
5260 
5261 	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5262 				 1, MEM_CGROUP_ID_MAX + 1, GFP_KERNEL);
5263 	if (memcg->id.id < 0) {
5264 		error = memcg->id.id;
5265 		goto fail;
5266 	}
5267 
5268 	memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL);
5269 	if (!memcg->vmstats)
5270 		goto fail;
5271 
5272 	memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5273 						 GFP_KERNEL_ACCOUNT);
5274 	if (!memcg->vmstats_percpu)
5275 		goto fail;
5276 
5277 	for_each_node(node)
5278 		if (alloc_mem_cgroup_per_node_info(memcg, node))
5279 			goto fail;
5280 
5281 	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5282 		goto fail;
5283 
5284 	INIT_WORK(&memcg->high_work, high_work_func);
5285 	INIT_LIST_HEAD(&memcg->oom_notify);
5286 	mutex_init(&memcg->thresholds_lock);
5287 	spin_lock_init(&memcg->move_lock);
5288 	vmpressure_init(&memcg->vmpressure);
5289 	INIT_LIST_HEAD(&memcg->event_list);
5290 	spin_lock_init(&memcg->event_list_lock);
5291 	memcg->socket_pressure = jiffies;
5292 #ifdef CONFIG_MEMCG_KMEM
5293 	memcg->kmemcg_id = -1;
5294 	INIT_LIST_HEAD(&memcg->objcg_list);
5295 #endif
5296 #ifdef CONFIG_CGROUP_WRITEBACK
5297 	INIT_LIST_HEAD(&memcg->cgwb_list);
5298 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5299 		memcg->cgwb_frn[i].done =
5300 			__WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
5301 #endif
5302 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5303 	spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5304 	INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5305 	memcg->deferred_split_queue.split_queue_len = 0;
5306 #endif
5307 	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5308 	lru_gen_init_memcg(memcg);
5309 	return memcg;
5310 fail:
5311 	mem_cgroup_id_remove(memcg);
5312 	__mem_cgroup_free(memcg);
5313 	return ERR_PTR(error);
5314 }
5315 
5316 static struct cgroup_subsys_state * __ref
mem_cgroup_css_alloc(struct cgroup_subsys_state * parent_css)5317 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
5318 {
5319 	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
5320 	struct mem_cgroup *memcg, *old_memcg;
5321 
5322 	old_memcg = set_active_memcg(parent);
5323 	memcg = mem_cgroup_alloc();
5324 	set_active_memcg(old_memcg);
5325 	if (IS_ERR(memcg))
5326 		return ERR_CAST(memcg);
5327 
5328 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5329 	memcg->soft_limit = PAGE_COUNTER_MAX;
5330 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
5331 	memcg->zswap_max = PAGE_COUNTER_MAX;
5332 #endif
5333 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5334 	if (parent) {
5335 		memcg->swappiness = mem_cgroup_swappiness(parent);
5336 		memcg->oom_kill_disable = parent->oom_kill_disable;
5337 
5338 		page_counter_init(&memcg->memory, &parent->memory);
5339 		page_counter_init(&memcg->swap, &parent->swap);
5340 		page_counter_init(&memcg->kmem, &parent->kmem);
5341 		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5342 	} else {
5343 		init_memcg_events();
5344 		page_counter_init(&memcg->memory, NULL);
5345 		page_counter_init(&memcg->swap, NULL);
5346 		page_counter_init(&memcg->kmem, NULL);
5347 		page_counter_init(&memcg->tcpmem, NULL);
5348 
5349 		root_mem_cgroup = memcg;
5350 		return &memcg->css;
5351 	}
5352 
5353 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5354 		static_branch_inc(&memcg_sockets_enabled_key);
5355 
5356 	return &memcg->css;
5357 }
5358 
mem_cgroup_css_online(struct cgroup_subsys_state * css)5359 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
5360 {
5361 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5362 
5363 	if (memcg_online_kmem(memcg))
5364 		goto remove_id;
5365 
5366 	/*
5367 	 * A memcg must be visible for expand_shrinker_info()
5368 	 * by the time the maps are allocated. So, we allocate maps
5369 	 * here, when for_each_mem_cgroup() can't skip it.
5370 	 */
5371 	if (alloc_shrinker_info(memcg))
5372 		goto offline_kmem;
5373 
5374 	/* Online state pins memcg ID, memcg ID pins CSS */
5375 	refcount_set(&memcg->id.ref, 1);
5376 	css_get(css);
5377 
5378 	if (unlikely(mem_cgroup_is_root(memcg)))
5379 		queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
5380 				   2UL*HZ);
5381 	return 0;
5382 offline_kmem:
5383 	memcg_offline_kmem(memcg);
5384 remove_id:
5385 	mem_cgroup_id_remove(memcg);
5386 	return -ENOMEM;
5387 }
5388 
mem_cgroup_css_offline(struct cgroup_subsys_state * css)5389 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
5390 {
5391 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5392 	struct mem_cgroup_event *event, *tmp;
5393 
5394 	/*
5395 	 * Unregister events and notify userspace.
5396 	 * Notify userspace about cgroup removing only after rmdir of cgroup
5397 	 * directory to avoid race between userspace and kernelspace.
5398 	 */
5399 	spin_lock_irq(&memcg->event_list_lock);
5400 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5401 		list_del_init(&event->list);
5402 		schedule_work(&event->remove);
5403 	}
5404 	spin_unlock_irq(&memcg->event_list_lock);
5405 
5406 	page_counter_set_min(&memcg->memory, 0);
5407 	page_counter_set_low(&memcg->memory, 0);
5408 
5409 	memcg_offline_kmem(memcg);
5410 	reparent_shrinker_deferred(memcg);
5411 	wb_memcg_offline(memcg);
5412 
5413 	drain_all_stock(memcg);
5414 
5415 	mem_cgroup_id_put(memcg);
5416 }
5417 
mem_cgroup_css_released(struct cgroup_subsys_state * css)5418 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
5419 {
5420 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5421 
5422 	invalidate_reclaim_iterators(memcg);
5423 }
5424 
mem_cgroup_css_free(struct cgroup_subsys_state * css)5425 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
5426 {
5427 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5428 	int __maybe_unused i;
5429 
5430 #ifdef CONFIG_CGROUP_WRITEBACK
5431 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5432 		wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5433 #endif
5434 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5435 		static_branch_dec(&memcg_sockets_enabled_key);
5436 
5437 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5438 		static_branch_dec(&memcg_sockets_enabled_key);
5439 
5440 	vmpressure_cleanup(&memcg->vmpressure);
5441 	cancel_work_sync(&memcg->high_work);
5442 	mem_cgroup_remove_from_trees(memcg);
5443 	free_shrinker_info(memcg);
5444 	mem_cgroup_free(memcg);
5445 }
5446 
5447 /**
5448  * mem_cgroup_css_reset - reset the states of a mem_cgroup
5449  * @css: the target css
5450  *
5451  * Reset the states of the mem_cgroup associated with @css.  This is
5452  * invoked when the userland requests disabling on the default hierarchy
5453  * but the memcg is pinned through dependency.  The memcg should stop
5454  * applying policies and should revert to the vanilla state as it may be
5455  * made visible again.
5456  *
5457  * The current implementation only resets the essential configurations.
5458  * This needs to be expanded to cover all the visible parts.
5459  */
mem_cgroup_css_reset(struct cgroup_subsys_state * css)5460 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
5461 {
5462 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5463 
5464 	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5465 	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5466 	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5467 	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5468 	page_counter_set_min(&memcg->memory, 0);
5469 	page_counter_set_low(&memcg->memory, 0);
5470 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5471 	memcg->soft_limit = PAGE_COUNTER_MAX;
5472 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5473 	memcg_wb_domain_size_changed(memcg);
5474 }
5475 
mem_cgroup_css_rstat_flush(struct cgroup_subsys_state * css,int cpu)5476 static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
5477 {
5478 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5479 	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5480 	struct memcg_vmstats_percpu *statc;
5481 	long delta, v;
5482 	int i, nid;
5483 
5484 	statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
5485 
5486 	for (i = 0; i < MEMCG_NR_STAT; i++) {
5487 		/*
5488 		 * Collect the aggregated propagation counts of groups
5489 		 * below us. We're in a per-cpu loop here and this is
5490 		 * a global counter, so the first cycle will get them.
5491 		 */
5492 		delta = memcg->vmstats->state_pending[i];
5493 		if (delta)
5494 			memcg->vmstats->state_pending[i] = 0;
5495 
5496 		/* Add CPU changes on this level since the last flush */
5497 		v = READ_ONCE(statc->state[i]);
5498 		if (v != statc->state_prev[i]) {
5499 			delta += v - statc->state_prev[i];
5500 			statc->state_prev[i] = v;
5501 		}
5502 
5503 		if (!delta)
5504 			continue;
5505 
5506 		/* Aggregate counts on this level and propagate upwards */
5507 		memcg->vmstats->state[i] += delta;
5508 		if (parent)
5509 			parent->vmstats->state_pending[i] += delta;
5510 	}
5511 
5512 	for (i = 0; i < NR_MEMCG_EVENTS; i++) {
5513 		delta = memcg->vmstats->events_pending[i];
5514 		if (delta)
5515 			memcg->vmstats->events_pending[i] = 0;
5516 
5517 		v = READ_ONCE(statc->events[i]);
5518 		if (v != statc->events_prev[i]) {
5519 			delta += v - statc->events_prev[i];
5520 			statc->events_prev[i] = v;
5521 		}
5522 
5523 		if (!delta)
5524 			continue;
5525 
5526 		memcg->vmstats->events[i] += delta;
5527 		if (parent)
5528 			parent->vmstats->events_pending[i] += delta;
5529 	}
5530 
5531 	for_each_node_state(nid, N_MEMORY) {
5532 		struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
5533 		struct mem_cgroup_per_node *ppn = NULL;
5534 		struct lruvec_stats_percpu *lstatc;
5535 
5536 		if (parent)
5537 			ppn = parent->nodeinfo[nid];
5538 
5539 		lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu);
5540 
5541 		for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
5542 			delta = pn->lruvec_stats.state_pending[i];
5543 			if (delta)
5544 				pn->lruvec_stats.state_pending[i] = 0;
5545 
5546 			v = READ_ONCE(lstatc->state[i]);
5547 			if (v != lstatc->state_prev[i]) {
5548 				delta += v - lstatc->state_prev[i];
5549 				lstatc->state_prev[i] = v;
5550 			}
5551 
5552 			if (!delta)
5553 				continue;
5554 
5555 			pn->lruvec_stats.state[i] += delta;
5556 			if (ppn)
5557 				ppn->lruvec_stats.state_pending[i] += delta;
5558 		}
5559 	}
5560 }
5561 
5562 #ifdef CONFIG_MMU
5563 /* Handlers for move charge at task migration. */
mem_cgroup_do_precharge(unsigned long count)5564 static int mem_cgroup_do_precharge(unsigned long count)
5565 {
5566 	int ret;
5567 
5568 	/* Try a single bulk charge without reclaim first, kswapd may wake */
5569 	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
5570 	if (!ret) {
5571 		mc.precharge += count;
5572 		return ret;
5573 	}
5574 
5575 	/* Try charges one by one with reclaim, but do not retry */
5576 	while (count--) {
5577 		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
5578 		if (ret)
5579 			return ret;
5580 		mc.precharge++;
5581 		cond_resched();
5582 	}
5583 	return 0;
5584 }
5585 
5586 union mc_target {
5587 	struct page	*page;
5588 	swp_entry_t	ent;
5589 };
5590 
5591 enum mc_target_type {
5592 	MC_TARGET_NONE = 0,
5593 	MC_TARGET_PAGE,
5594 	MC_TARGET_SWAP,
5595 	MC_TARGET_DEVICE,
5596 };
5597 
mc_handle_present_pte(struct vm_area_struct * vma,unsigned long addr,pte_t ptent)5598 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5599 						unsigned long addr, pte_t ptent)
5600 {
5601 	struct page *page = vm_normal_page(vma, addr, ptent);
5602 
5603 	if (!page || !page_mapped(page))
5604 		return NULL;
5605 	if (PageAnon(page)) {
5606 		if (!(mc.flags & MOVE_ANON))
5607 			return NULL;
5608 	} else {
5609 		if (!(mc.flags & MOVE_FILE))
5610 			return NULL;
5611 	}
5612 	if (!get_page_unless_zero(page))
5613 		return NULL;
5614 
5615 	return page;
5616 }
5617 
5618 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
mc_handle_swap_pte(struct vm_area_struct * vma,pte_t ptent,swp_entry_t * entry)5619 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5620 			pte_t ptent, swp_entry_t *entry)
5621 {
5622 	struct page *page = NULL;
5623 	swp_entry_t ent = pte_to_swp_entry(ptent);
5624 
5625 	if (!(mc.flags & MOVE_ANON))
5626 		return NULL;
5627 
5628 	/*
5629 	 * Handle device private pages that are not accessible by the CPU, but
5630 	 * stored as special swap entries in the page table.
5631 	 */
5632 	if (is_device_private_entry(ent)) {
5633 		page = pfn_swap_entry_to_page(ent);
5634 		if (!get_page_unless_zero(page))
5635 			return NULL;
5636 		return page;
5637 	}
5638 
5639 	if (non_swap_entry(ent))
5640 		return NULL;
5641 
5642 	/*
5643 	 * Because swap_cache_get_folio() updates some statistics counter,
5644 	 * we call find_get_page() with swapper_space directly.
5645 	 */
5646 	page = find_get_page(swap_address_space(ent), swp_offset(ent));
5647 	entry->val = ent.val;
5648 
5649 	return page;
5650 }
5651 #else
mc_handle_swap_pte(struct vm_area_struct * vma,pte_t ptent,swp_entry_t * entry)5652 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5653 			pte_t ptent, swp_entry_t *entry)
5654 {
5655 	return NULL;
5656 }
5657 #endif
5658 
mc_handle_file_pte(struct vm_area_struct * vma,unsigned long addr,pte_t ptent)5659 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5660 			unsigned long addr, pte_t ptent)
5661 {
5662 	if (!vma->vm_file) /* anonymous vma */
5663 		return NULL;
5664 	if (!(mc.flags & MOVE_FILE))
5665 		return NULL;
5666 
5667 	/* page is moved even if it's not RSS of this task(page-faulted). */
5668 	/* shmem/tmpfs may report page out on swap: account for that too. */
5669 	return find_get_incore_page(vma->vm_file->f_mapping,
5670 			linear_page_index(vma, addr));
5671 }
5672 
5673 /**
5674  * mem_cgroup_move_account - move account of the page
5675  * @page: the page
5676  * @compound: charge the page as compound or small page
5677  * @from: mem_cgroup which the page is moved from.
5678  * @to:	mem_cgroup which the page is moved to. @from != @to.
5679  *
5680  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
5681  *
5682  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5683  * from old cgroup.
5684  */
mem_cgroup_move_account(struct page * page,bool compound,struct mem_cgroup * from,struct mem_cgroup * to)5685 static int mem_cgroup_move_account(struct page *page,
5686 				   bool compound,
5687 				   struct mem_cgroup *from,
5688 				   struct mem_cgroup *to)
5689 {
5690 	struct folio *folio = page_folio(page);
5691 	struct lruvec *from_vec, *to_vec;
5692 	struct pglist_data *pgdat;
5693 	unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1;
5694 	int nid, ret;
5695 
5696 	VM_BUG_ON(from == to);
5697 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
5698 	VM_BUG_ON(compound && !folio_test_large(folio));
5699 
5700 	/*
5701 	 * Prevent mem_cgroup_migrate() from looking at
5702 	 * page's memory cgroup of its source page while we change it.
5703 	 */
5704 	ret = -EBUSY;
5705 	if (!folio_trylock(folio))
5706 		goto out;
5707 
5708 	ret = -EINVAL;
5709 	if (folio_memcg(folio) != from)
5710 		goto out_unlock;
5711 
5712 	pgdat = folio_pgdat(folio);
5713 	from_vec = mem_cgroup_lruvec(from, pgdat);
5714 	to_vec = mem_cgroup_lruvec(to, pgdat);
5715 
5716 	folio_memcg_lock(folio);
5717 
5718 	if (folio_test_anon(folio)) {
5719 		if (folio_mapped(folio)) {
5720 			__mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5721 			__mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
5722 			if (folio_test_transhuge(folio)) {
5723 				__mod_lruvec_state(from_vec, NR_ANON_THPS,
5724 						   -nr_pages);
5725 				__mod_lruvec_state(to_vec, NR_ANON_THPS,
5726 						   nr_pages);
5727 			}
5728 		}
5729 	} else {
5730 		__mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
5731 		__mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
5732 
5733 		if (folio_test_swapbacked(folio)) {
5734 			__mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
5735 			__mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
5736 		}
5737 
5738 		if (folio_mapped(folio)) {
5739 			__mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5740 			__mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5741 		}
5742 
5743 		if (folio_test_dirty(folio)) {
5744 			struct address_space *mapping = folio_mapping(folio);
5745 
5746 			if (mapping_can_writeback(mapping)) {
5747 				__mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5748 						   -nr_pages);
5749 				__mod_lruvec_state(to_vec, NR_FILE_DIRTY,
5750 						   nr_pages);
5751 			}
5752 		}
5753 	}
5754 
5755 	if (folio_test_writeback(folio)) {
5756 		__mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5757 		__mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
5758 	}
5759 
5760 	/*
5761 	 * All state has been migrated, let's switch to the new memcg.
5762 	 *
5763 	 * It is safe to change page's memcg here because the page
5764 	 * is referenced, charged, isolated, and locked: we can't race
5765 	 * with (un)charging, migration, LRU putback, or anything else
5766 	 * that would rely on a stable page's memory cgroup.
5767 	 *
5768 	 * Note that lock_page_memcg is a memcg lock, not a page lock,
5769 	 * to save space. As soon as we switch page's memory cgroup to a
5770 	 * new memcg that isn't locked, the above state can change
5771 	 * concurrently again. Make sure we're truly done with it.
5772 	 */
5773 	smp_mb();
5774 
5775 	css_get(&to->css);
5776 	css_put(&from->css);
5777 
5778 	folio->memcg_data = (unsigned long)to;
5779 
5780 	__folio_memcg_unlock(from);
5781 
5782 	ret = 0;
5783 	nid = folio_nid(folio);
5784 
5785 	local_irq_disable();
5786 	mem_cgroup_charge_statistics(to, nr_pages);
5787 	memcg_check_events(to, nid);
5788 	mem_cgroup_charge_statistics(from, -nr_pages);
5789 	memcg_check_events(from, nid);
5790 	local_irq_enable();
5791 out_unlock:
5792 	folio_unlock(folio);
5793 out:
5794 	return ret;
5795 }
5796 
5797 /**
5798  * get_mctgt_type - get target type of moving charge
5799  * @vma: the vma the pte to be checked belongs
5800  * @addr: the address corresponding to the pte to be checked
5801  * @ptent: the pte to be checked
5802  * @target: the pointer the target page or swap ent will be stored(can be NULL)
5803  *
5804  * Returns
5805  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
5806  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
5807  *     move charge. if @target is not NULL, the page is stored in target->page
5808  *     with extra refcnt got(Callers should handle it).
5809  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
5810  *     target for charge migration. if @target is not NULL, the entry is stored
5811  *     in target->ent.
5812  *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is device memory and
5813  *   thus not on the lru.
5814  *     For now we such page is charge like a regular page would be as for all
5815  *     intent and purposes it is just special memory taking the place of a
5816  *     regular page.
5817  *
5818  *     See Documentations/vm/hmm.txt and include/linux/hmm.h
5819  *
5820  * Called with pte lock held.
5821  */
5822 
get_mctgt_type(struct vm_area_struct * vma,unsigned long addr,pte_t ptent,union mc_target * target)5823 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
5824 		unsigned long addr, pte_t ptent, union mc_target *target)
5825 {
5826 	struct page *page = NULL;
5827 	enum mc_target_type ret = MC_TARGET_NONE;
5828 	swp_entry_t ent = { .val = 0 };
5829 
5830 	if (pte_present(ptent))
5831 		page = mc_handle_present_pte(vma, addr, ptent);
5832 	else if (pte_none_mostly(ptent))
5833 		/*
5834 		 * PTE markers should be treated as a none pte here, separated
5835 		 * from other swap handling below.
5836 		 */
5837 		page = mc_handle_file_pte(vma, addr, ptent);
5838 	else if (is_swap_pte(ptent))
5839 		page = mc_handle_swap_pte(vma, ptent, &ent);
5840 
5841 	if (!page && !ent.val)
5842 		return ret;
5843 	if (page) {
5844 		/*
5845 		 * Do only loose check w/o serialization.
5846 		 * mem_cgroup_move_account() checks the page is valid or
5847 		 * not under LRU exclusion.
5848 		 */
5849 		if (page_memcg(page) == mc.from) {
5850 			ret = MC_TARGET_PAGE;
5851 			if (is_device_private_page(page) ||
5852 			    is_device_coherent_page(page))
5853 				ret = MC_TARGET_DEVICE;
5854 			if (target)
5855 				target->page = page;
5856 		}
5857 		if (!ret || !target)
5858 			put_page(page);
5859 	}
5860 	/*
5861 	 * There is a swap entry and a page doesn't exist or isn't charged.
5862 	 * But we cannot move a tail-page in a THP.
5863 	 */
5864 	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
5865 	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
5866 		ret = MC_TARGET_SWAP;
5867 		if (target)
5868 			target->ent = ent;
5869 	}
5870 	return ret;
5871 }
5872 
5873 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5874 /*
5875  * We don't consider PMD mapped swapping or file mapped pages because THP does
5876  * not support them for now.
5877  * Caller should make sure that pmd_trans_huge(pmd) is true.
5878  */
get_mctgt_type_thp(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd,union mc_target * target)5879 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5880 		unsigned long addr, pmd_t pmd, union mc_target *target)
5881 {
5882 	struct page *page = NULL;
5883 	enum mc_target_type ret = MC_TARGET_NONE;
5884 
5885 	if (unlikely(is_swap_pmd(pmd))) {
5886 		VM_BUG_ON(thp_migration_supported() &&
5887 				  !is_pmd_migration_entry(pmd));
5888 		return ret;
5889 	}
5890 	page = pmd_page(pmd);
5891 	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
5892 	if (!(mc.flags & MOVE_ANON))
5893 		return ret;
5894 	if (page_memcg(page) == mc.from) {
5895 		ret = MC_TARGET_PAGE;
5896 		if (target) {
5897 			get_page(page);
5898 			target->page = page;
5899 		}
5900 	}
5901 	return ret;
5902 }
5903 #else
get_mctgt_type_thp(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd,union mc_target * target)5904 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5905 		unsigned long addr, pmd_t pmd, union mc_target *target)
5906 {
5907 	return MC_TARGET_NONE;
5908 }
5909 #endif
5910 
mem_cgroup_count_precharge_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)5911 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5912 					unsigned long addr, unsigned long end,
5913 					struct mm_walk *walk)
5914 {
5915 	struct vm_area_struct *vma = walk->vma;
5916 	pte_t *pte;
5917 	spinlock_t *ptl;
5918 
5919 	ptl = pmd_trans_huge_lock(pmd, vma);
5920 	if (ptl) {
5921 		/*
5922 		 * Note their can not be MC_TARGET_DEVICE for now as we do not
5923 		 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
5924 		 * this might change.
5925 		 */
5926 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5927 			mc.precharge += HPAGE_PMD_NR;
5928 		spin_unlock(ptl);
5929 		return 0;
5930 	}
5931 
5932 	if (pmd_trans_unstable(pmd))
5933 		return 0;
5934 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5935 	for (; addr != end; pte++, addr += PAGE_SIZE)
5936 		if (get_mctgt_type(vma, addr, *pte, NULL))
5937 			mc.precharge++;	/* increment precharge temporarily */
5938 	pte_unmap_unlock(pte - 1, ptl);
5939 	cond_resched();
5940 
5941 	return 0;
5942 }
5943 
5944 static const struct mm_walk_ops precharge_walk_ops = {
5945 	.pmd_entry	= mem_cgroup_count_precharge_pte_range,
5946 };
5947 
mem_cgroup_count_precharge(struct mm_struct * mm)5948 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5949 {
5950 	unsigned long precharge;
5951 
5952 	mmap_read_lock(mm);
5953 	walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL);
5954 	mmap_read_unlock(mm);
5955 
5956 	precharge = mc.precharge;
5957 	mc.precharge = 0;
5958 
5959 	return precharge;
5960 }
5961 
mem_cgroup_precharge_mc(struct mm_struct * mm)5962 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5963 {
5964 	unsigned long precharge = mem_cgroup_count_precharge(mm);
5965 
5966 	VM_BUG_ON(mc.moving_task);
5967 	mc.moving_task = current;
5968 	return mem_cgroup_do_precharge(precharge);
5969 }
5970 
5971 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
__mem_cgroup_clear_mc(void)5972 static void __mem_cgroup_clear_mc(void)
5973 {
5974 	struct mem_cgroup *from = mc.from;
5975 	struct mem_cgroup *to = mc.to;
5976 
5977 	/* we must uncharge all the leftover precharges from mc.to */
5978 	if (mc.precharge) {
5979 		cancel_charge(mc.to, mc.precharge);
5980 		mc.precharge = 0;
5981 	}
5982 	/*
5983 	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5984 	 * we must uncharge here.
5985 	 */
5986 	if (mc.moved_charge) {
5987 		cancel_charge(mc.from, mc.moved_charge);
5988 		mc.moved_charge = 0;
5989 	}
5990 	/* we must fixup refcnts and charges */
5991 	if (mc.moved_swap) {
5992 		/* uncharge swap account from the old cgroup */
5993 		if (!mem_cgroup_is_root(mc.from))
5994 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5995 
5996 		mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5997 
5998 		/*
5999 		 * we charged both to->memory and to->memsw, so we
6000 		 * should uncharge to->memory.
6001 		 */
6002 		if (!mem_cgroup_is_root(mc.to))
6003 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);
6004 
6005 		mc.moved_swap = 0;
6006 	}
6007 	memcg_oom_recover(from);
6008 	memcg_oom_recover(to);
6009 	wake_up_all(&mc.waitq);
6010 }
6011 
mem_cgroup_clear_mc(void)6012 static void mem_cgroup_clear_mc(void)
6013 {
6014 	struct mm_struct *mm = mc.mm;
6015 
6016 	/*
6017 	 * we must clear moving_task before waking up waiters at the end of
6018 	 * task migration.
6019 	 */
6020 	mc.moving_task = NULL;
6021 	__mem_cgroup_clear_mc();
6022 	spin_lock(&mc.lock);
6023 	mc.from = NULL;
6024 	mc.to = NULL;
6025 	mc.mm = NULL;
6026 	spin_unlock(&mc.lock);
6027 
6028 	mmput(mm);
6029 }
6030 
mem_cgroup_can_attach(struct cgroup_taskset * tset)6031 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
6032 {
6033 	struct cgroup_subsys_state *css;
6034 	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
6035 	struct mem_cgroup *from;
6036 	struct task_struct *leader, *p;
6037 	struct mm_struct *mm;
6038 	unsigned long move_flags;
6039 	int ret = 0;
6040 
6041 	/* charge immigration isn't supported on the default hierarchy */
6042 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6043 		return 0;
6044 
6045 	/*
6046 	 * Multi-process migrations only happen on the default hierarchy
6047 	 * where charge immigration is not used.  Perform charge
6048 	 * immigration if @tset contains a leader and whine if there are
6049 	 * multiple.
6050 	 */
6051 	p = NULL;
6052 	cgroup_taskset_for_each_leader(leader, css, tset) {
6053 		WARN_ON_ONCE(p);
6054 		p = leader;
6055 		memcg = mem_cgroup_from_css(css);
6056 	}
6057 	if (!p)
6058 		return 0;
6059 
6060 	/*
6061 	 * We are now committed to this value whatever it is. Changes in this
6062 	 * tunable will only affect upcoming migrations, not the current one.
6063 	 * So we need to save it, and keep it going.
6064 	 */
6065 	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
6066 	if (!move_flags)
6067 		return 0;
6068 
6069 	from = mem_cgroup_from_task(p);
6070 
6071 	VM_BUG_ON(from == memcg);
6072 
6073 	mm = get_task_mm(p);
6074 	if (!mm)
6075 		return 0;
6076 	/* We move charges only when we move a owner of the mm */
6077 	if (mm->owner == p) {
6078 		VM_BUG_ON(mc.from);
6079 		VM_BUG_ON(mc.to);
6080 		VM_BUG_ON(mc.precharge);
6081 		VM_BUG_ON(mc.moved_charge);
6082 		VM_BUG_ON(mc.moved_swap);
6083 
6084 		spin_lock(&mc.lock);
6085 		mc.mm = mm;
6086 		mc.from = from;
6087 		mc.to = memcg;
6088 		mc.flags = move_flags;
6089 		spin_unlock(&mc.lock);
6090 		/* We set mc.moving_task later */
6091 
6092 		ret = mem_cgroup_precharge_mc(mm);
6093 		if (ret)
6094 			mem_cgroup_clear_mc();
6095 	} else {
6096 		mmput(mm);
6097 	}
6098 	return ret;
6099 }
6100 
mem_cgroup_cancel_attach(struct cgroup_taskset * tset)6101 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
6102 {
6103 	if (mc.to)
6104 		mem_cgroup_clear_mc();
6105 }
6106 
mem_cgroup_move_charge_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)6107 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
6108 				unsigned long addr, unsigned long end,
6109 				struct mm_walk *walk)
6110 {
6111 	int ret = 0;
6112 	struct vm_area_struct *vma = walk->vma;
6113 	pte_t *pte;
6114 	spinlock_t *ptl;
6115 	enum mc_target_type target_type;
6116 	union mc_target target;
6117 	struct page *page;
6118 
6119 	ptl = pmd_trans_huge_lock(pmd, vma);
6120 	if (ptl) {
6121 		if (mc.precharge < HPAGE_PMD_NR) {
6122 			spin_unlock(ptl);
6123 			return 0;
6124 		}
6125 		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
6126 		if (target_type == MC_TARGET_PAGE) {
6127 			page = target.page;
6128 			if (!isolate_lru_page(page)) {
6129 				if (!mem_cgroup_move_account(page, true,
6130 							     mc.from, mc.to)) {
6131 					mc.precharge -= HPAGE_PMD_NR;
6132 					mc.moved_charge += HPAGE_PMD_NR;
6133 				}
6134 				putback_lru_page(page);
6135 			}
6136 			put_page(page);
6137 		} else if (target_type == MC_TARGET_DEVICE) {
6138 			page = target.page;
6139 			if (!mem_cgroup_move_account(page, true,
6140 						     mc.from, mc.to)) {
6141 				mc.precharge -= HPAGE_PMD_NR;
6142 				mc.moved_charge += HPAGE_PMD_NR;
6143 			}
6144 			put_page(page);
6145 		}
6146 		spin_unlock(ptl);
6147 		return 0;
6148 	}
6149 
6150 	if (pmd_trans_unstable(pmd))
6151 		return 0;
6152 retry:
6153 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6154 	for (; addr != end; addr += PAGE_SIZE) {
6155 		pte_t ptent = *(pte++);
6156 		bool device = false;
6157 		swp_entry_t ent;
6158 
6159 		if (!mc.precharge)
6160 			break;
6161 
6162 		switch (get_mctgt_type(vma, addr, ptent, &target)) {
6163 		case MC_TARGET_DEVICE:
6164 			device = true;
6165 			fallthrough;
6166 		case MC_TARGET_PAGE:
6167 			page = target.page;
6168 			/*
6169 			 * We can have a part of the split pmd here. Moving it
6170 			 * can be done but it would be too convoluted so simply
6171 			 * ignore such a partial THP and keep it in original
6172 			 * memcg. There should be somebody mapping the head.
6173 			 */
6174 			if (PageTransCompound(page))
6175 				goto put;
6176 			if (!device && isolate_lru_page(page))
6177 				goto put;
6178 			if (!mem_cgroup_move_account(page, false,
6179 						mc.from, mc.to)) {
6180 				mc.precharge--;
6181 				/* we uncharge from mc.from later. */
6182 				mc.moved_charge++;
6183 			}
6184 			if (!device)
6185 				putback_lru_page(page);
6186 put:			/* get_mctgt_type() gets the page */
6187 			put_page(page);
6188 			break;
6189 		case MC_TARGET_SWAP:
6190 			ent = target.ent;
6191 			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
6192 				mc.precharge--;
6193 				mem_cgroup_id_get_many(mc.to, 1);
6194 				/* we fixup other refcnts and charges later. */
6195 				mc.moved_swap++;
6196 			}
6197 			break;
6198 		default:
6199 			break;
6200 		}
6201 	}
6202 	pte_unmap_unlock(pte - 1, ptl);
6203 	cond_resched();
6204 
6205 	if (addr != end) {
6206 		/*
6207 		 * We have consumed all precharges we got in can_attach().
6208 		 * We try charge one by one, but don't do any additional
6209 		 * charges to mc.to if we have failed in charge once in attach()
6210 		 * phase.
6211 		 */
6212 		ret = mem_cgroup_do_precharge(1);
6213 		if (!ret)
6214 			goto retry;
6215 	}
6216 
6217 	return ret;
6218 }
6219 
6220 static const struct mm_walk_ops charge_walk_ops = {
6221 	.pmd_entry	= mem_cgroup_move_charge_pte_range,
6222 };
6223 
mem_cgroup_move_charge(void)6224 static void mem_cgroup_move_charge(void)
6225 {
6226 	lru_add_drain_all();
6227 	/*
6228 	 * Signal lock_page_memcg() to take the memcg's move_lock
6229 	 * while we're moving its pages to another memcg. Then wait
6230 	 * for already started RCU-only updates to finish.
6231 	 */
6232 	atomic_inc(&mc.from->moving_account);
6233 	synchronize_rcu();
6234 retry:
6235 	if (unlikely(!mmap_read_trylock(mc.mm))) {
6236 		/*
6237 		 * Someone who are holding the mmap_lock might be waiting in
6238 		 * waitq. So we cancel all extra charges, wake up all waiters,
6239 		 * and retry. Because we cancel precharges, we might not be able
6240 		 * to move enough charges, but moving charge is a best-effort
6241 		 * feature anyway, so it wouldn't be a big problem.
6242 		 */
6243 		__mem_cgroup_clear_mc();
6244 		cond_resched();
6245 		goto retry;
6246 	}
6247 	/*
6248 	 * When we have consumed all precharges and failed in doing
6249 	 * additional charge, the page walk just aborts.
6250 	 */
6251 	walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL);
6252 	mmap_read_unlock(mc.mm);
6253 	atomic_dec(&mc.from->moving_account);
6254 }
6255 
mem_cgroup_move_task(void)6256 static void mem_cgroup_move_task(void)
6257 {
6258 	if (mc.to) {
6259 		mem_cgroup_move_charge();
6260 		mem_cgroup_clear_mc();
6261 	}
6262 }
6263 #else	/* !CONFIG_MMU */
mem_cgroup_can_attach(struct cgroup_taskset * tset)6264 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
6265 {
6266 	return 0;
6267 }
mem_cgroup_cancel_attach(struct cgroup_taskset * tset)6268 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
6269 {
6270 }
mem_cgroup_move_task(void)6271 static void mem_cgroup_move_task(void)
6272 {
6273 }
6274 #endif
6275 
6276 #ifdef CONFIG_LRU_GEN
mem_cgroup_attach(struct cgroup_taskset * tset)6277 static void mem_cgroup_attach(struct cgroup_taskset *tset)
6278 {
6279 	struct task_struct *task;
6280 	struct cgroup_subsys_state *css;
6281 
6282 	/* find the first leader if there is any */
6283 	cgroup_taskset_for_each_leader(task, css, tset)
6284 		break;
6285 
6286 	if (!task)
6287 		return;
6288 
6289 	task_lock(task);
6290 	if (task->mm && READ_ONCE(task->mm->owner) == task)
6291 		lru_gen_migrate_mm(task->mm);
6292 	task_unlock(task);
6293 }
6294 #else
mem_cgroup_attach(struct cgroup_taskset * tset)6295 static void mem_cgroup_attach(struct cgroup_taskset *tset)
6296 {
6297 }
6298 #endif /* CONFIG_LRU_GEN */
6299 
seq_puts_memcg_tunable(struct seq_file * m,unsigned long value)6300 static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
6301 {
6302 	if (value == PAGE_COUNTER_MAX)
6303 		seq_puts(m, "max\n");
6304 	else
6305 		seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
6306 
6307 	return 0;
6308 }
6309 
memory_current_read(struct cgroup_subsys_state * css,struct cftype * cft)6310 static u64 memory_current_read(struct cgroup_subsys_state *css,
6311 			       struct cftype *cft)
6312 {
6313 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6314 
6315 	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6316 }
6317 
memory_peak_read(struct cgroup_subsys_state * css,struct cftype * cft)6318 static u64 memory_peak_read(struct cgroup_subsys_state *css,
6319 			    struct cftype *cft)
6320 {
6321 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6322 
6323 	return (u64)memcg->memory.watermark * PAGE_SIZE;
6324 }
6325 
memory_min_show(struct seq_file * m,void * v)6326 static int memory_min_show(struct seq_file *m, void *v)
6327 {
6328 	return seq_puts_memcg_tunable(m,
6329 		READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
6330 }
6331 
memory_min_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6332 static ssize_t memory_min_write(struct kernfs_open_file *of,
6333 				char *buf, size_t nbytes, loff_t off)
6334 {
6335 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6336 	unsigned long min;
6337 	int err;
6338 
6339 	buf = strstrip(buf);
6340 	err = page_counter_memparse(buf, "max", &min);
6341 	if (err)
6342 		return err;
6343 
6344 	page_counter_set_min(&memcg->memory, min);
6345 
6346 	return nbytes;
6347 }
6348 
memory_low_show(struct seq_file * m,void * v)6349 static int memory_low_show(struct seq_file *m, void *v)
6350 {
6351 	return seq_puts_memcg_tunable(m,
6352 		READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
6353 }
6354 
memory_low_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6355 static ssize_t memory_low_write(struct kernfs_open_file *of,
6356 				char *buf, size_t nbytes, loff_t off)
6357 {
6358 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6359 	unsigned long low;
6360 	int err;
6361 
6362 	buf = strstrip(buf);
6363 	err = page_counter_memparse(buf, "max", &low);
6364 	if (err)
6365 		return err;
6366 
6367 	page_counter_set_low(&memcg->memory, low);
6368 
6369 	return nbytes;
6370 }
6371 
memory_high_show(struct seq_file * m,void * v)6372 static int memory_high_show(struct seq_file *m, void *v)
6373 {
6374 	return seq_puts_memcg_tunable(m,
6375 		READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
6376 }
6377 
memory_high_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6378 static ssize_t memory_high_write(struct kernfs_open_file *of,
6379 				 char *buf, size_t nbytes, loff_t off)
6380 {
6381 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6382 	unsigned int nr_retries = MAX_RECLAIM_RETRIES;
6383 	bool drained = false;
6384 	unsigned long high;
6385 	int err;
6386 
6387 	buf = strstrip(buf);
6388 	err = page_counter_memparse(buf, "max", &high);
6389 	if (err)
6390 		return err;
6391 
6392 	page_counter_set_high(&memcg->memory, high);
6393 
6394 	for (;;) {
6395 		unsigned long nr_pages = page_counter_read(&memcg->memory);
6396 		unsigned long reclaimed;
6397 
6398 		if (nr_pages <= high)
6399 			break;
6400 
6401 		if (signal_pending(current))
6402 			break;
6403 
6404 		if (!drained) {
6405 			drain_all_stock(memcg);
6406 			drained = true;
6407 			continue;
6408 		}
6409 
6410 		reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6411 					GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP);
6412 
6413 		if (!reclaimed && !nr_retries--)
6414 			break;
6415 	}
6416 
6417 	memcg_wb_domain_size_changed(memcg);
6418 	return nbytes;
6419 }
6420 
memory_max_show(struct seq_file * m,void * v)6421 static int memory_max_show(struct seq_file *m, void *v)
6422 {
6423 	return seq_puts_memcg_tunable(m,
6424 		READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
6425 }
6426 
memory_max_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6427 static ssize_t memory_max_write(struct kernfs_open_file *of,
6428 				char *buf, size_t nbytes, loff_t off)
6429 {
6430 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6431 	unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
6432 	bool drained = false;
6433 	unsigned long max;
6434 	int err;
6435 
6436 	buf = strstrip(buf);
6437 	err = page_counter_memparse(buf, "max", &max);
6438 	if (err)
6439 		return err;
6440 
6441 	xchg(&memcg->memory.max, max);
6442 
6443 	for (;;) {
6444 		unsigned long nr_pages = page_counter_read(&memcg->memory);
6445 
6446 		if (nr_pages <= max)
6447 			break;
6448 
6449 		if (signal_pending(current))
6450 			break;
6451 
6452 		if (!drained) {
6453 			drain_all_stock(memcg);
6454 			drained = true;
6455 			continue;
6456 		}
6457 
6458 		if (nr_reclaims) {
6459 			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6460 					GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP))
6461 				nr_reclaims--;
6462 			continue;
6463 		}
6464 
6465 		memcg_memory_event(memcg, MEMCG_OOM);
6466 		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6467 			break;
6468 	}
6469 
6470 	memcg_wb_domain_size_changed(memcg);
6471 	return nbytes;
6472 }
6473 
__memory_events_show(struct seq_file * m,atomic_long_t * events)6474 static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
6475 {
6476 	seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
6477 	seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
6478 	seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
6479 	seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
6480 	seq_printf(m, "oom_kill %lu\n",
6481 		   atomic_long_read(&events[MEMCG_OOM_KILL]));
6482 	seq_printf(m, "oom_group_kill %lu\n",
6483 		   atomic_long_read(&events[MEMCG_OOM_GROUP_KILL]));
6484 }
6485 
memory_events_show(struct seq_file * m,void * v)6486 static int memory_events_show(struct seq_file *m, void *v)
6487 {
6488 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6489 
6490 	__memory_events_show(m, memcg->memory_events);
6491 	return 0;
6492 }
6493 
memory_events_local_show(struct seq_file * m,void * v)6494 static int memory_events_local_show(struct seq_file *m, void *v)
6495 {
6496 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6497 
6498 	__memory_events_show(m, memcg->memory_events_local);
6499 	return 0;
6500 }
6501 
memory_stat_show(struct seq_file * m,void * v)6502 static int memory_stat_show(struct seq_file *m, void *v)
6503 {
6504 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6505 	char *buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
6506 
6507 	if (!buf)
6508 		return -ENOMEM;
6509 	memory_stat_format(memcg, buf, PAGE_SIZE);
6510 	seq_puts(m, buf);
6511 	kfree(buf);
6512 	return 0;
6513 }
6514 
6515 #ifdef CONFIG_NUMA
lruvec_page_state_output(struct lruvec * lruvec,int item)6516 static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
6517 						     int item)
6518 {
6519 	return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item);
6520 }
6521 
memory_numa_stat_show(struct seq_file * m,void * v)6522 static int memory_numa_stat_show(struct seq_file *m, void *v)
6523 {
6524 	int i;
6525 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6526 
6527 	mem_cgroup_flush_stats();
6528 
6529 	for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
6530 		int nid;
6531 
6532 		if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
6533 			continue;
6534 
6535 		seq_printf(m, "%s", memory_stats[i].name);
6536 		for_each_node_state(nid, N_MEMORY) {
6537 			u64 size;
6538 			struct lruvec *lruvec;
6539 
6540 			lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
6541 			size = lruvec_page_state_output(lruvec,
6542 							memory_stats[i].idx);
6543 			seq_printf(m, " N%d=%llu", nid, size);
6544 		}
6545 		seq_putc(m, '\n');
6546 	}
6547 
6548 	return 0;
6549 }
6550 #endif
6551 
memory_oom_group_show(struct seq_file * m,void * v)6552 static int memory_oom_group_show(struct seq_file *m, void *v)
6553 {
6554 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6555 
6556 	seq_printf(m, "%d\n", memcg->oom_group);
6557 
6558 	return 0;
6559 }
6560 
memory_oom_group_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6561 static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
6562 				      char *buf, size_t nbytes, loff_t off)
6563 {
6564 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6565 	int ret, oom_group;
6566 
6567 	buf = strstrip(buf);
6568 	if (!buf)
6569 		return -EINVAL;
6570 
6571 	ret = kstrtoint(buf, 0, &oom_group);
6572 	if (ret)
6573 		return ret;
6574 
6575 	if (oom_group != 0 && oom_group != 1)
6576 		return -EINVAL;
6577 
6578 	memcg->oom_group = oom_group;
6579 
6580 	return nbytes;
6581 }
6582 
memory_reclaim(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6583 static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf,
6584 			      size_t nbytes, loff_t off)
6585 {
6586 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6587 	unsigned int nr_retries = MAX_RECLAIM_RETRIES;
6588 	unsigned long nr_to_reclaim, nr_reclaimed = 0;
6589 	unsigned int reclaim_options;
6590 	int err;
6591 
6592 	buf = strstrip(buf);
6593 	err = page_counter_memparse(buf, "", &nr_to_reclaim);
6594 	if (err)
6595 		return err;
6596 
6597 	reclaim_options	= MEMCG_RECLAIM_MAY_SWAP | MEMCG_RECLAIM_PROACTIVE;
6598 	while (nr_reclaimed < nr_to_reclaim) {
6599 		unsigned long reclaimed;
6600 
6601 		if (signal_pending(current))
6602 			return -EINTR;
6603 
6604 		/*
6605 		 * This is the final attempt, drain percpu lru caches in the
6606 		 * hope of introducing more evictable pages for
6607 		 * try_to_free_mem_cgroup_pages().
6608 		 */
6609 		if (!nr_retries)
6610 			lru_add_drain_all();
6611 
6612 		reclaimed = try_to_free_mem_cgroup_pages(memcg,
6613 						nr_to_reclaim - nr_reclaimed,
6614 						GFP_KERNEL, reclaim_options);
6615 
6616 		if (!reclaimed && !nr_retries--)
6617 			return -EAGAIN;
6618 
6619 		nr_reclaimed += reclaimed;
6620 	}
6621 
6622 	return nbytes;
6623 }
6624 
6625 static struct cftype memory_files[] = {
6626 	{
6627 		.name = "current",
6628 		.flags = CFTYPE_NOT_ON_ROOT,
6629 		.read_u64 = memory_current_read,
6630 	},
6631 	{
6632 		.name = "peak",
6633 		.flags = CFTYPE_NOT_ON_ROOT,
6634 		.read_u64 = memory_peak_read,
6635 	},
6636 	{
6637 		.name = "min",
6638 		.flags = CFTYPE_NOT_ON_ROOT,
6639 		.seq_show = memory_min_show,
6640 		.write = memory_min_write,
6641 	},
6642 	{
6643 		.name = "low",
6644 		.flags = CFTYPE_NOT_ON_ROOT,
6645 		.seq_show = memory_low_show,
6646 		.write = memory_low_write,
6647 	},
6648 	{
6649 		.name = "high",
6650 		.flags = CFTYPE_NOT_ON_ROOT,
6651 		.seq_show = memory_high_show,
6652 		.write = memory_high_write,
6653 	},
6654 	{
6655 		.name = "max",
6656 		.flags = CFTYPE_NOT_ON_ROOT,
6657 		.seq_show = memory_max_show,
6658 		.write = memory_max_write,
6659 	},
6660 	{
6661 		.name = "events",
6662 		.flags = CFTYPE_NOT_ON_ROOT,
6663 		.file_offset = offsetof(struct mem_cgroup, events_file),
6664 		.seq_show = memory_events_show,
6665 	},
6666 	{
6667 		.name = "events.local",
6668 		.flags = CFTYPE_NOT_ON_ROOT,
6669 		.file_offset = offsetof(struct mem_cgroup, events_local_file),
6670 		.seq_show = memory_events_local_show,
6671 	},
6672 	{
6673 		.name = "stat",
6674 		.seq_show = memory_stat_show,
6675 	},
6676 #ifdef CONFIG_NUMA
6677 	{
6678 		.name = "numa_stat",
6679 		.seq_show = memory_numa_stat_show,
6680 	},
6681 #endif
6682 	{
6683 		.name = "oom.group",
6684 		.flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
6685 		.seq_show = memory_oom_group_show,
6686 		.write = memory_oom_group_write,
6687 	},
6688 	{
6689 		.name = "reclaim",
6690 		.flags = CFTYPE_NS_DELEGATABLE,
6691 		.write = memory_reclaim,
6692 	},
6693 	{ }	/* terminate */
6694 };
6695 
6696 struct cgroup_subsys memory_cgrp_subsys = {
6697 	.css_alloc = mem_cgroup_css_alloc,
6698 	.css_online = mem_cgroup_css_online,
6699 	.css_offline = mem_cgroup_css_offline,
6700 	.css_released = mem_cgroup_css_released,
6701 	.css_free = mem_cgroup_css_free,
6702 	.css_reset = mem_cgroup_css_reset,
6703 	.css_rstat_flush = mem_cgroup_css_rstat_flush,
6704 	.can_attach = mem_cgroup_can_attach,
6705 	.attach = mem_cgroup_attach,
6706 	.cancel_attach = mem_cgroup_cancel_attach,
6707 	.post_attach = mem_cgroup_move_task,
6708 	.dfl_cftypes = memory_files,
6709 	.legacy_cftypes = mem_cgroup_legacy_files,
6710 	.early_init = 0,
6711 };
6712 
6713 /*
6714  * This function calculates an individual cgroup's effective
6715  * protection which is derived from its own memory.min/low, its
6716  * parent's and siblings' settings, as well as the actual memory
6717  * distribution in the tree.
6718  *
6719  * The following rules apply to the effective protection values:
6720  *
6721  * 1. At the first level of reclaim, effective protection is equal to
6722  *    the declared protection in memory.min and memory.low.
6723  *
6724  * 2. To enable safe delegation of the protection configuration, at
6725  *    subsequent levels the effective protection is capped to the
6726  *    parent's effective protection.
6727  *
6728  * 3. To make complex and dynamic subtrees easier to configure, the
6729  *    user is allowed to overcommit the declared protection at a given
6730  *    level. If that is the case, the parent's effective protection is
6731  *    distributed to the children in proportion to how much protection
6732  *    they have declared and how much of it they are utilizing.
6733  *
6734  *    This makes distribution proportional, but also work-conserving:
6735  *    if one cgroup claims much more protection than it uses memory,
6736  *    the unused remainder is available to its siblings.
6737  *
6738  * 4. Conversely, when the declared protection is undercommitted at a
6739  *    given level, the distribution of the larger parental protection
6740  *    budget is NOT proportional. A cgroup's protection from a sibling
6741  *    is capped to its own memory.min/low setting.
6742  *
6743  * 5. However, to allow protecting recursive subtrees from each other
6744  *    without having to declare each individual cgroup's fixed share
6745  *    of the ancestor's claim to protection, any unutilized -
6746  *    "floating" - protection from up the tree is distributed in
6747  *    proportion to each cgroup's *usage*. This makes the protection
6748  *    neutral wrt sibling cgroups and lets them compete freely over
6749  *    the shared parental protection budget, but it protects the
6750  *    subtree as a whole from neighboring subtrees.
6751  *
6752  * Note that 4. and 5. are not in conflict: 4. is about protecting
6753  * against immediate siblings whereas 5. is about protecting against
6754  * neighboring subtrees.
6755  */
effective_protection(unsigned long usage,unsigned long parent_usage,unsigned long setting,unsigned long parent_effective,unsigned long siblings_protected)6756 static unsigned long effective_protection(unsigned long usage,
6757 					  unsigned long parent_usage,
6758 					  unsigned long setting,
6759 					  unsigned long parent_effective,
6760 					  unsigned long siblings_protected)
6761 {
6762 	unsigned long protected;
6763 	unsigned long ep;
6764 
6765 	protected = min(usage, setting);
6766 	/*
6767 	 * If all cgroups at this level combined claim and use more
6768 	 * protection then what the parent affords them, distribute
6769 	 * shares in proportion to utilization.
6770 	 *
6771 	 * We are using actual utilization rather than the statically
6772 	 * claimed protection in order to be work-conserving: claimed
6773 	 * but unused protection is available to siblings that would
6774 	 * otherwise get a smaller chunk than what they claimed.
6775 	 */
6776 	if (siblings_protected > parent_effective)
6777 		return protected * parent_effective / siblings_protected;
6778 
6779 	/*
6780 	 * Ok, utilized protection of all children is within what the
6781 	 * parent affords them, so we know whatever this child claims
6782 	 * and utilizes is effectively protected.
6783 	 *
6784 	 * If there is unprotected usage beyond this value, reclaim
6785 	 * will apply pressure in proportion to that amount.
6786 	 *
6787 	 * If there is unutilized protection, the cgroup will be fully
6788 	 * shielded from reclaim, but we do return a smaller value for
6789 	 * protection than what the group could enjoy in theory. This
6790 	 * is okay. With the overcommit distribution above, effective
6791 	 * protection is always dependent on how memory is actually
6792 	 * consumed among the siblings anyway.
6793 	 */
6794 	ep = protected;
6795 
6796 	/*
6797 	 * If the children aren't claiming (all of) the protection
6798 	 * afforded to them by the parent, distribute the remainder in
6799 	 * proportion to the (unprotected) memory of each cgroup. That
6800 	 * way, cgroups that aren't explicitly prioritized wrt each
6801 	 * other compete freely over the allowance, but they are
6802 	 * collectively protected from neighboring trees.
6803 	 *
6804 	 * We're using unprotected memory for the weight so that if
6805 	 * some cgroups DO claim explicit protection, we don't protect
6806 	 * the same bytes twice.
6807 	 *
6808 	 * Check both usage and parent_usage against the respective
6809 	 * protected values. One should imply the other, but they
6810 	 * aren't read atomically - make sure the division is sane.
6811 	 */
6812 	if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
6813 		return ep;
6814 	if (parent_effective > siblings_protected &&
6815 	    parent_usage > siblings_protected &&
6816 	    usage > protected) {
6817 		unsigned long unclaimed;
6818 
6819 		unclaimed = parent_effective - siblings_protected;
6820 		unclaimed *= usage - protected;
6821 		unclaimed /= parent_usage - siblings_protected;
6822 
6823 		ep += unclaimed;
6824 	}
6825 
6826 	return ep;
6827 }
6828 
6829 /**
6830  * mem_cgroup_calculate_protection - check if memory consumption is in the normal range
6831  * @root: the top ancestor of the sub-tree being checked
6832  * @memcg: the memory cgroup to check
6833  *
6834  * WARNING: This function is not stateless! It can only be used as part
6835  *          of a top-down tree iteration, not for isolated queries.
6836  */
mem_cgroup_calculate_protection(struct mem_cgroup * root,struct mem_cgroup * memcg)6837 void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6838 				     struct mem_cgroup *memcg)
6839 {
6840 	unsigned long usage, parent_usage;
6841 	struct mem_cgroup *parent;
6842 
6843 	if (mem_cgroup_disabled())
6844 		return;
6845 
6846 	if (!root)
6847 		root = root_mem_cgroup;
6848 
6849 	/*
6850 	 * Effective values of the reclaim targets are ignored so they
6851 	 * can be stale. Have a look at mem_cgroup_protection for more
6852 	 * details.
6853 	 * TODO: calculation should be more robust so that we do not need
6854 	 * that special casing.
6855 	 */
6856 	if (memcg == root)
6857 		return;
6858 
6859 	usage = page_counter_read(&memcg->memory);
6860 	if (!usage)
6861 		return;
6862 
6863 	parent = parent_mem_cgroup(memcg);
6864 
6865 	if (parent == root) {
6866 		memcg->memory.emin = READ_ONCE(memcg->memory.min);
6867 		memcg->memory.elow = READ_ONCE(memcg->memory.low);
6868 		return;
6869 	}
6870 
6871 	parent_usage = page_counter_read(&parent->memory);
6872 
6873 	WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
6874 			READ_ONCE(memcg->memory.min),
6875 			READ_ONCE(parent->memory.emin),
6876 			atomic_long_read(&parent->memory.children_min_usage)));
6877 
6878 	WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
6879 			READ_ONCE(memcg->memory.low),
6880 			READ_ONCE(parent->memory.elow),
6881 			atomic_long_read(&parent->memory.children_low_usage)));
6882 }
6883 
charge_memcg(struct folio * folio,struct mem_cgroup * memcg,gfp_t gfp)6884 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
6885 			gfp_t gfp)
6886 {
6887 	long nr_pages = folio_nr_pages(folio);
6888 	int ret;
6889 
6890 	ret = try_charge(memcg, gfp, nr_pages);
6891 	if (ret)
6892 		goto out;
6893 
6894 	css_get(&memcg->css);
6895 	commit_charge(folio, memcg);
6896 
6897 	local_irq_disable();
6898 	mem_cgroup_charge_statistics(memcg, nr_pages);
6899 	memcg_check_events(memcg, folio_nid(folio));
6900 	local_irq_enable();
6901 out:
6902 	return ret;
6903 }
6904 
__mem_cgroup_charge(struct folio * folio,struct mm_struct * mm,gfp_t gfp)6905 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp)
6906 {
6907 	struct mem_cgroup *memcg;
6908 	int ret;
6909 
6910 	memcg = get_mem_cgroup_from_mm(mm);
6911 	ret = charge_memcg(folio, memcg, gfp);
6912 	css_put(&memcg->css);
6913 
6914 	return ret;
6915 }
6916 
6917 /**
6918  * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin.
6919  * @folio: folio to charge.
6920  * @mm: mm context of the victim
6921  * @gfp: reclaim mode
6922  * @entry: swap entry for which the folio is allocated
6923  *
6924  * This function charges a folio allocated for swapin. Please call this before
6925  * adding the folio to the swapcache.
6926  *
6927  * Returns 0 on success. Otherwise, an error code is returned.
6928  */
mem_cgroup_swapin_charge_folio(struct folio * folio,struct mm_struct * mm,gfp_t gfp,swp_entry_t entry)6929 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
6930 				  gfp_t gfp, swp_entry_t entry)
6931 {
6932 	struct mem_cgroup *memcg;
6933 	unsigned short id;
6934 	int ret;
6935 
6936 	if (mem_cgroup_disabled())
6937 		return 0;
6938 
6939 	id = lookup_swap_cgroup_id(entry);
6940 	rcu_read_lock();
6941 	memcg = mem_cgroup_from_id(id);
6942 	if (!memcg || !css_tryget_online(&memcg->css))
6943 		memcg = get_mem_cgroup_from_mm(mm);
6944 	rcu_read_unlock();
6945 
6946 	ret = charge_memcg(folio, memcg, gfp);
6947 
6948 	css_put(&memcg->css);
6949 	return ret;
6950 }
6951 
6952 /*
6953  * mem_cgroup_swapin_uncharge_swap - uncharge swap slot
6954  * @entry: swap entry for which the page is charged
6955  *
6956  * Call this function after successfully adding the charged page to swapcache.
6957  *
6958  * Note: This function assumes the page for which swap slot is being uncharged
6959  * is order 0 page.
6960  */
mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)6961 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
6962 {
6963 	/*
6964 	 * Cgroup1's unified memory+swap counter has been charged with the
6965 	 * new swapcache page, finish the transfer by uncharging the swap
6966 	 * slot. The swap slot would also get uncharged when it dies, but
6967 	 * it can stick around indefinitely and we'd count the page twice
6968 	 * the entire time.
6969 	 *
6970 	 * Cgroup2 has separate resource counters for memory and swap,
6971 	 * so this is a non-issue here. Memory and swap charge lifetimes
6972 	 * correspond 1:1 to page and swap slot lifetimes: we charge the
6973 	 * page to memory here, and uncharge swap when the slot is freed.
6974 	 */
6975 	if (!mem_cgroup_disabled() && do_memsw_account()) {
6976 		/*
6977 		 * The swap entry might not get freed for a long time,
6978 		 * let's not wait for it.  The page already received a
6979 		 * memory+swap charge, drop the swap entry duplicate.
6980 		 */
6981 		mem_cgroup_uncharge_swap(entry, 1);
6982 	}
6983 }
6984 
6985 struct uncharge_gather {
6986 	struct mem_cgroup *memcg;
6987 	unsigned long nr_memory;
6988 	unsigned long pgpgout;
6989 	unsigned long nr_kmem;
6990 	int nid;
6991 };
6992 
uncharge_gather_clear(struct uncharge_gather * ug)6993 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6994 {
6995 	memset(ug, 0, sizeof(*ug));
6996 }
6997 
uncharge_batch(const struct uncharge_gather * ug)6998 static void uncharge_batch(const struct uncharge_gather *ug)
6999 {
7000 	unsigned long flags;
7001 
7002 	if (ug->nr_memory) {
7003 		page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
7004 		if (do_memsw_account())
7005 			page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory);
7006 		if (ug->nr_kmem)
7007 			memcg_account_kmem(ug->memcg, -ug->nr_kmem);
7008 		memcg_oom_recover(ug->memcg);
7009 	}
7010 
7011 	local_irq_save(flags);
7012 	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
7013 	__this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory);
7014 	memcg_check_events(ug->memcg, ug->nid);
7015 	local_irq_restore(flags);
7016 
7017 	/* drop reference from uncharge_folio */
7018 	css_put(&ug->memcg->css);
7019 }
7020 
uncharge_folio(struct folio * folio,struct uncharge_gather * ug)7021 static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
7022 {
7023 	long nr_pages;
7024 	struct mem_cgroup *memcg;
7025 	struct obj_cgroup *objcg;
7026 
7027 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
7028 
7029 	/*
7030 	 * Nobody should be changing or seriously looking at
7031 	 * folio memcg or objcg at this point, we have fully
7032 	 * exclusive access to the folio.
7033 	 */
7034 	if (folio_memcg_kmem(folio)) {
7035 		objcg = __folio_objcg(folio);
7036 		/*
7037 		 * This get matches the put at the end of the function and
7038 		 * kmem pages do not hold memcg references anymore.
7039 		 */
7040 		memcg = get_mem_cgroup_from_objcg(objcg);
7041 	} else {
7042 		memcg = __folio_memcg(folio);
7043 	}
7044 
7045 	if (!memcg)
7046 		return;
7047 
7048 	if (ug->memcg != memcg) {
7049 		if (ug->memcg) {
7050 			uncharge_batch(ug);
7051 			uncharge_gather_clear(ug);
7052 		}
7053 		ug->memcg = memcg;
7054 		ug->nid = folio_nid(folio);
7055 
7056 		/* pairs with css_put in uncharge_batch */
7057 		css_get(&memcg->css);
7058 	}
7059 
7060 	nr_pages = folio_nr_pages(folio);
7061 
7062 	if (folio_memcg_kmem(folio)) {
7063 		ug->nr_memory += nr_pages;
7064 		ug->nr_kmem += nr_pages;
7065 
7066 		folio->memcg_data = 0;
7067 		obj_cgroup_put(objcg);
7068 	} else {
7069 		/* LRU pages aren't accounted at the root level */
7070 		if (!mem_cgroup_is_root(memcg))
7071 			ug->nr_memory += nr_pages;
7072 		ug->pgpgout++;
7073 
7074 		folio->memcg_data = 0;
7075 	}
7076 
7077 	css_put(&memcg->css);
7078 }
7079 
__mem_cgroup_uncharge(struct folio * folio)7080 void __mem_cgroup_uncharge(struct folio *folio)
7081 {
7082 	struct uncharge_gather ug;
7083 
7084 	/* Don't touch folio->lru of any random page, pre-check: */
7085 	if (!folio_memcg(folio))
7086 		return;
7087 
7088 	uncharge_gather_clear(&ug);
7089 	uncharge_folio(folio, &ug);
7090 	uncharge_batch(&ug);
7091 }
7092 
7093 /**
7094  * __mem_cgroup_uncharge_list - uncharge a list of page
7095  * @page_list: list of pages to uncharge
7096  *
7097  * Uncharge a list of pages previously charged with
7098  * __mem_cgroup_charge().
7099  */
__mem_cgroup_uncharge_list(struct list_head * page_list)7100 void __mem_cgroup_uncharge_list(struct list_head *page_list)
7101 {
7102 	struct uncharge_gather ug;
7103 	struct folio *folio;
7104 
7105 	uncharge_gather_clear(&ug);
7106 	list_for_each_entry(folio, page_list, lru)
7107 		uncharge_folio(folio, &ug);
7108 	if (ug.memcg)
7109 		uncharge_batch(&ug);
7110 }
7111 
7112 /**
7113  * mem_cgroup_migrate - Charge a folio's replacement.
7114  * @old: Currently circulating folio.
7115  * @new: Replacement folio.
7116  *
7117  * Charge @new as a replacement folio for @old. @old will
7118  * be uncharged upon free.
7119  *
7120  * Both folios must be locked, @new->mapping must be set up.
7121  */
mem_cgroup_migrate(struct folio * old,struct folio * new)7122 void mem_cgroup_migrate(struct folio *old, struct folio *new)
7123 {
7124 	struct mem_cgroup *memcg;
7125 	long nr_pages = folio_nr_pages(new);
7126 	unsigned long flags;
7127 
7128 	VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
7129 	VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
7130 	VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new);
7131 	VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new);
7132 
7133 	if (mem_cgroup_disabled())
7134 		return;
7135 
7136 	/* Page cache replacement: new folio already charged? */
7137 	if (folio_memcg(new))
7138 		return;
7139 
7140 	memcg = folio_memcg(old);
7141 	VM_WARN_ON_ONCE_FOLIO(!memcg, old);
7142 	if (!memcg)
7143 		return;
7144 
7145 	/* Force-charge the new page. The old one will be freed soon */
7146 	if (!mem_cgroup_is_root(memcg)) {
7147 		page_counter_charge(&memcg->memory, nr_pages);
7148 		if (do_memsw_account())
7149 			page_counter_charge(&memcg->memsw, nr_pages);
7150 	}
7151 
7152 	css_get(&memcg->css);
7153 	commit_charge(new, memcg);
7154 
7155 	local_irq_save(flags);
7156 	mem_cgroup_charge_statistics(memcg, nr_pages);
7157 	memcg_check_events(memcg, folio_nid(new));
7158 	local_irq_restore(flags);
7159 }
7160 
7161 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
7162 EXPORT_SYMBOL(memcg_sockets_enabled_key);
7163 
mem_cgroup_sk_alloc(struct sock * sk)7164 void mem_cgroup_sk_alloc(struct sock *sk)
7165 {
7166 	struct mem_cgroup *memcg;
7167 
7168 	if (!mem_cgroup_sockets_enabled)
7169 		return;
7170 
7171 	/* Do not associate the sock with unrelated interrupted task's memcg. */
7172 	if (!in_task())
7173 		return;
7174 
7175 	rcu_read_lock();
7176 	memcg = mem_cgroup_from_task(current);
7177 	if (memcg == root_mem_cgroup)
7178 		goto out;
7179 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
7180 		goto out;
7181 	if (css_tryget(&memcg->css))
7182 		sk->sk_memcg = memcg;
7183 out:
7184 	rcu_read_unlock();
7185 }
7186 
mem_cgroup_sk_free(struct sock * sk)7187 void mem_cgroup_sk_free(struct sock *sk)
7188 {
7189 	if (sk->sk_memcg)
7190 		css_put(&sk->sk_memcg->css);
7191 }
7192 
7193 /**
7194  * mem_cgroup_charge_skmem - charge socket memory
7195  * @memcg: memcg to charge
7196  * @nr_pages: number of pages to charge
7197  * @gfp_mask: reclaim mode
7198  *
7199  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
7200  * @memcg's configured limit, %false if it doesn't.
7201  */
mem_cgroup_charge_skmem(struct mem_cgroup * memcg,unsigned int nr_pages,gfp_t gfp_mask)7202 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
7203 			     gfp_t gfp_mask)
7204 {
7205 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
7206 		struct page_counter *fail;
7207 
7208 		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
7209 			memcg->tcpmem_pressure = 0;
7210 			return true;
7211 		}
7212 		memcg->tcpmem_pressure = 1;
7213 		if (gfp_mask & __GFP_NOFAIL) {
7214 			page_counter_charge(&memcg->tcpmem, nr_pages);
7215 			return true;
7216 		}
7217 		return false;
7218 	}
7219 
7220 	if (try_charge(memcg, gfp_mask, nr_pages) == 0) {
7221 		mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
7222 		return true;
7223 	}
7224 
7225 	return false;
7226 }
7227 
7228 /**
7229  * mem_cgroup_uncharge_skmem - uncharge socket memory
7230  * @memcg: memcg to uncharge
7231  * @nr_pages: number of pages to uncharge
7232  */
mem_cgroup_uncharge_skmem(struct mem_cgroup * memcg,unsigned int nr_pages)7233 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7234 {
7235 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
7236 		page_counter_uncharge(&memcg->tcpmem, nr_pages);
7237 		return;
7238 	}
7239 
7240 	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7241 
7242 	refill_stock(memcg, nr_pages);
7243 }
7244 
cgroup_memory(char * s)7245 static int __init cgroup_memory(char *s)
7246 {
7247 	char *token;
7248 
7249 	while ((token = strsep(&s, ",")) != NULL) {
7250 		if (!*token)
7251 			continue;
7252 		if (!strcmp(token, "nosocket"))
7253 			cgroup_memory_nosocket = true;
7254 		if (!strcmp(token, "nokmem"))
7255 			cgroup_memory_nokmem = true;
7256 	}
7257 	return 1;
7258 }
7259 __setup("cgroup.memory=", cgroup_memory);
7260 
7261 /*
7262  * subsys_initcall() for memory controller.
7263  *
7264  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
7265  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
7266  * basically everything that doesn't depend on a specific mem_cgroup structure
7267  * should be initialized from here.
7268  */
mem_cgroup_init(void)7269 static int __init mem_cgroup_init(void)
7270 {
7271 	int cpu, node;
7272 
7273 	/*
7274 	 * Currently s32 type (can refer to struct batched_lruvec_stat) is
7275 	 * used for per-memcg-per-cpu caching of per-node statistics. In order
7276 	 * to work fine, we should make sure that the overfill threshold can't
7277 	 * exceed S32_MAX / PAGE_SIZE.
7278 	 */
7279 	BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
7280 
7281 	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
7282 				  memcg_hotplug_cpu_dead);
7283 
7284 	for_each_possible_cpu(cpu)
7285 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
7286 			  drain_local_stock);
7287 
7288 	for_each_node(node) {
7289 		struct mem_cgroup_tree_per_node *rtpn;
7290 
7291 		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
7292 				    node_online(node) ? node : NUMA_NO_NODE);
7293 
7294 		rtpn->rb_root = RB_ROOT;
7295 		rtpn->rb_rightmost = NULL;
7296 		spin_lock_init(&rtpn->lock);
7297 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
7298 	}
7299 
7300 	return 0;
7301 }
7302 subsys_initcall(mem_cgroup_init);
7303 
7304 #ifdef CONFIG_SWAP
mem_cgroup_id_get_online(struct mem_cgroup * memcg)7305 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7306 {
7307 	while (!refcount_inc_not_zero(&memcg->id.ref)) {
7308 		/*
7309 		 * The root cgroup cannot be destroyed, so it's refcount must
7310 		 * always be >= 1.
7311 		 */
7312 		if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
7313 			VM_BUG_ON(1);
7314 			break;
7315 		}
7316 		memcg = parent_mem_cgroup(memcg);
7317 		if (!memcg)
7318 			memcg = root_mem_cgroup;
7319 	}
7320 	return memcg;
7321 }
7322 
7323 /**
7324  * mem_cgroup_swapout - transfer a memsw charge to swap
7325  * @folio: folio whose memsw charge to transfer
7326  * @entry: swap entry to move the charge to
7327  *
7328  * Transfer the memsw charge of @folio to @entry.
7329  */
mem_cgroup_swapout(struct folio * folio,swp_entry_t entry)7330 void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
7331 {
7332 	struct mem_cgroup *memcg, *swap_memcg;
7333 	unsigned int nr_entries;
7334 	unsigned short oldid;
7335 
7336 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
7337 	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
7338 
7339 	if (mem_cgroup_disabled())
7340 		return;
7341 
7342 	if (!do_memsw_account())
7343 		return;
7344 
7345 	memcg = folio_memcg(folio);
7346 
7347 	VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
7348 	if (!memcg)
7349 		return;
7350 
7351 	/*
7352 	 * In case the memcg owning these pages has been offlined and doesn't
7353 	 * have an ID allocated to it anymore, charge the closest online
7354 	 * ancestor for the swap instead and transfer the memory+swap charge.
7355 	 */
7356 	swap_memcg = mem_cgroup_id_get_online(memcg);
7357 	nr_entries = folio_nr_pages(folio);
7358 	/* Get references for the tail pages, too */
7359 	if (nr_entries > 1)
7360 		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
7361 	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
7362 				   nr_entries);
7363 	VM_BUG_ON_FOLIO(oldid, folio);
7364 	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
7365 
7366 	folio->memcg_data = 0;
7367 
7368 	if (!mem_cgroup_is_root(memcg))
7369 		page_counter_uncharge(&memcg->memory, nr_entries);
7370 
7371 	if (memcg != swap_memcg) {
7372 		if (!mem_cgroup_is_root(swap_memcg))
7373 			page_counter_charge(&swap_memcg->memsw, nr_entries);
7374 		page_counter_uncharge(&memcg->memsw, nr_entries);
7375 	}
7376 
7377 	/*
7378 	 * Interrupts should be disabled here because the caller holds the
7379 	 * i_pages lock which is taken with interrupts-off. It is
7380 	 * important here to have the interrupts disabled because it is the
7381 	 * only synchronisation we have for updating the per-CPU variables.
7382 	 */
7383 	memcg_stats_lock();
7384 	mem_cgroup_charge_statistics(memcg, -nr_entries);
7385 	memcg_stats_unlock();
7386 	memcg_check_events(memcg, folio_nid(folio));
7387 
7388 	css_put(&memcg->css);
7389 }
7390 
7391 /**
7392  * __mem_cgroup_try_charge_swap - try charging swap space for a folio
7393  * @folio: folio being added to swap
7394  * @entry: swap entry to charge
7395  *
7396  * Try to charge @folio's memcg for the swap space at @entry.
7397  *
7398  * Returns 0 on success, -ENOMEM on failure.
7399  */
__mem_cgroup_try_charge_swap(struct folio * folio,swp_entry_t entry)7400 int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)
7401 {
7402 	unsigned int nr_pages = folio_nr_pages(folio);
7403 	struct page_counter *counter;
7404 	struct mem_cgroup *memcg;
7405 	unsigned short oldid;
7406 
7407 	if (do_memsw_account())
7408 		return 0;
7409 
7410 	memcg = folio_memcg(folio);
7411 
7412 	VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
7413 	if (!memcg)
7414 		return 0;
7415 
7416 	if (!entry.val) {
7417 		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7418 		return 0;
7419 	}
7420 
7421 	memcg = mem_cgroup_id_get_online(memcg);
7422 
7423 	if (!mem_cgroup_is_root(memcg) &&
7424 	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7425 		memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7426 		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7427 		mem_cgroup_id_put(memcg);
7428 		return -ENOMEM;
7429 	}
7430 
7431 	/* Get references for the tail pages, too */
7432 	if (nr_pages > 1)
7433 		mem_cgroup_id_get_many(memcg, nr_pages - 1);
7434 	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7435 	VM_BUG_ON_FOLIO(oldid, folio);
7436 	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
7437 
7438 	return 0;
7439 }
7440 
7441 /**
7442  * __mem_cgroup_uncharge_swap - uncharge swap space
7443  * @entry: swap entry to uncharge
7444  * @nr_pages: the amount of swap space to uncharge
7445  */
__mem_cgroup_uncharge_swap(swp_entry_t entry,unsigned int nr_pages)7446 void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
7447 {
7448 	struct mem_cgroup *memcg;
7449 	unsigned short id;
7450 
7451 	if (mem_cgroup_disabled())
7452 		return;
7453 
7454 	id = swap_cgroup_record(entry, 0, nr_pages);
7455 	rcu_read_lock();
7456 	memcg = mem_cgroup_from_id(id);
7457 	if (memcg) {
7458 		if (!mem_cgroup_is_root(memcg)) {
7459 			if (do_memsw_account())
7460 				page_counter_uncharge(&memcg->memsw, nr_pages);
7461 			else
7462 				page_counter_uncharge(&memcg->swap, nr_pages);
7463 		}
7464 		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
7465 		mem_cgroup_id_put_many(memcg, nr_pages);
7466 	}
7467 	rcu_read_unlock();
7468 }
7469 
mem_cgroup_get_nr_swap_pages(struct mem_cgroup * memcg)7470 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7471 {
7472 	long nr_swap_pages = get_nr_swap_pages();
7473 
7474 	if (mem_cgroup_disabled() || do_memsw_account())
7475 		return nr_swap_pages;
7476 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7477 		nr_swap_pages = min_t(long, nr_swap_pages,
7478 				      READ_ONCE(memcg->swap.max) -
7479 				      page_counter_read(&memcg->swap));
7480 	return nr_swap_pages;
7481 }
7482 
mem_cgroup_swap_full(struct folio * folio)7483 bool mem_cgroup_swap_full(struct folio *folio)
7484 {
7485 	struct mem_cgroup *memcg;
7486 
7487 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
7488 
7489 	if (vm_swap_full())
7490 		return true;
7491 	if (do_memsw_account())
7492 		return false;
7493 
7494 	memcg = folio_memcg(folio);
7495 	if (!memcg)
7496 		return false;
7497 
7498 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7499 		unsigned long usage = page_counter_read(&memcg->swap);
7500 
7501 		if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7502 		    usage * 2 >= READ_ONCE(memcg->swap.max))
7503 			return true;
7504 	}
7505 
7506 	return false;
7507 }
7508 
setup_swap_account(char * s)7509 static int __init setup_swap_account(char *s)
7510 {
7511 	pr_warn_once("The swapaccount= commandline option is deprecated. "
7512 		     "Please report your usecase to linux-mm@kvack.org if you "
7513 		     "depend on this functionality.\n");
7514 	return 1;
7515 }
7516 __setup("swapaccount=", setup_swap_account);
7517 
swap_current_read(struct cgroup_subsys_state * css,struct cftype * cft)7518 static u64 swap_current_read(struct cgroup_subsys_state *css,
7519 			     struct cftype *cft)
7520 {
7521 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7522 
7523 	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7524 }
7525 
swap_high_show(struct seq_file * m,void * v)7526 static int swap_high_show(struct seq_file *m, void *v)
7527 {
7528 	return seq_puts_memcg_tunable(m,
7529 		READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
7530 }
7531 
swap_high_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)7532 static ssize_t swap_high_write(struct kernfs_open_file *of,
7533 			       char *buf, size_t nbytes, loff_t off)
7534 {
7535 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7536 	unsigned long high;
7537 	int err;
7538 
7539 	buf = strstrip(buf);
7540 	err = page_counter_memparse(buf, "max", &high);
7541 	if (err)
7542 		return err;
7543 
7544 	page_counter_set_high(&memcg->swap, high);
7545 
7546 	return nbytes;
7547 }
7548 
swap_max_show(struct seq_file * m,void * v)7549 static int swap_max_show(struct seq_file *m, void *v)
7550 {
7551 	return seq_puts_memcg_tunable(m,
7552 		READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
7553 }
7554 
swap_max_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)7555 static ssize_t swap_max_write(struct kernfs_open_file *of,
7556 			      char *buf, size_t nbytes, loff_t off)
7557 {
7558 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7559 	unsigned long max;
7560 	int err;
7561 
7562 	buf = strstrip(buf);
7563 	err = page_counter_memparse(buf, "max", &max);
7564 	if (err)
7565 		return err;
7566 
7567 	xchg(&memcg->swap.max, max);
7568 
7569 	return nbytes;
7570 }
7571 
swap_events_show(struct seq_file * m,void * v)7572 static int swap_events_show(struct seq_file *m, void *v)
7573 {
7574 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7575 
7576 	seq_printf(m, "high %lu\n",
7577 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
7578 	seq_printf(m, "max %lu\n",
7579 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7580 	seq_printf(m, "fail %lu\n",
7581 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7582 
7583 	return 0;
7584 }
7585 
7586 static struct cftype swap_files[] = {
7587 	{
7588 		.name = "swap.current",
7589 		.flags = CFTYPE_NOT_ON_ROOT,
7590 		.read_u64 = swap_current_read,
7591 	},
7592 	{
7593 		.name = "swap.high",
7594 		.flags = CFTYPE_NOT_ON_ROOT,
7595 		.seq_show = swap_high_show,
7596 		.write = swap_high_write,
7597 	},
7598 	{
7599 		.name = "swap.max",
7600 		.flags = CFTYPE_NOT_ON_ROOT,
7601 		.seq_show = swap_max_show,
7602 		.write = swap_max_write,
7603 	},
7604 	{
7605 		.name = "swap.events",
7606 		.flags = CFTYPE_NOT_ON_ROOT,
7607 		.file_offset = offsetof(struct mem_cgroup, swap_events_file),
7608 		.seq_show = swap_events_show,
7609 	},
7610 	{ }	/* terminate */
7611 };
7612 
7613 static struct cftype memsw_files[] = {
7614 	{
7615 		.name = "memsw.usage_in_bytes",
7616 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
7617 		.read_u64 = mem_cgroup_read_u64,
7618 	},
7619 	{
7620 		.name = "memsw.max_usage_in_bytes",
7621 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
7622 		.write = mem_cgroup_reset,
7623 		.read_u64 = mem_cgroup_read_u64,
7624 	},
7625 	{
7626 		.name = "memsw.limit_in_bytes",
7627 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
7628 		.write = mem_cgroup_write,
7629 		.read_u64 = mem_cgroup_read_u64,
7630 	},
7631 	{
7632 		.name = "memsw.failcnt",
7633 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
7634 		.write = mem_cgroup_reset,
7635 		.read_u64 = mem_cgroup_read_u64,
7636 	},
7637 	{ },	/* terminate */
7638 };
7639 
7640 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
7641 /**
7642  * obj_cgroup_may_zswap - check if this cgroup can zswap
7643  * @objcg: the object cgroup
7644  *
7645  * Check if the hierarchical zswap limit has been reached.
7646  *
7647  * This doesn't check for specific headroom, and it is not atomic
7648  * either. But with zswap, the size of the allocation is only known
7649  * once compression has occured, and this optimistic pre-check avoids
7650  * spending cycles on compression when there is already no room left
7651  * or zswap is disabled altogether somewhere in the hierarchy.
7652  */
obj_cgroup_may_zswap(struct obj_cgroup * objcg)7653 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
7654 {
7655 	struct mem_cgroup *memcg, *original_memcg;
7656 	bool ret = true;
7657 
7658 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7659 		return true;
7660 
7661 	original_memcg = get_mem_cgroup_from_objcg(objcg);
7662 	for (memcg = original_memcg; memcg != root_mem_cgroup;
7663 	     memcg = parent_mem_cgroup(memcg)) {
7664 		unsigned long max = READ_ONCE(memcg->zswap_max);
7665 		unsigned long pages;
7666 
7667 		if (max == PAGE_COUNTER_MAX)
7668 			continue;
7669 		if (max == 0) {
7670 			ret = false;
7671 			break;
7672 		}
7673 
7674 		cgroup_rstat_flush(memcg->css.cgroup);
7675 		pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE;
7676 		if (pages < max)
7677 			continue;
7678 		ret = false;
7679 		break;
7680 	}
7681 	mem_cgroup_put(original_memcg);
7682 	return ret;
7683 }
7684 
7685 /**
7686  * obj_cgroup_charge_zswap - charge compression backend memory
7687  * @objcg: the object cgroup
7688  * @size: size of compressed object
7689  *
7690  * This forces the charge after obj_cgroup_may_swap() allowed
7691  * compression and storage in zwap for this cgroup to go ahead.
7692  */
obj_cgroup_charge_zswap(struct obj_cgroup * objcg,size_t size)7693 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
7694 {
7695 	struct mem_cgroup *memcg;
7696 
7697 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7698 		return;
7699 
7700 	VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC));
7701 
7702 	/* PF_MEMALLOC context, charging must succeed */
7703 	if (obj_cgroup_charge(objcg, GFP_KERNEL, size))
7704 		VM_WARN_ON_ONCE(1);
7705 
7706 	rcu_read_lock();
7707 	memcg = obj_cgroup_memcg(objcg);
7708 	mod_memcg_state(memcg, MEMCG_ZSWAP_B, size);
7709 	mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1);
7710 	rcu_read_unlock();
7711 }
7712 
7713 /**
7714  * obj_cgroup_uncharge_zswap - uncharge compression backend memory
7715  * @objcg: the object cgroup
7716  * @size: size of compressed object
7717  *
7718  * Uncharges zswap memory on page in.
7719  */
obj_cgroup_uncharge_zswap(struct obj_cgroup * objcg,size_t size)7720 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)
7721 {
7722 	struct mem_cgroup *memcg;
7723 
7724 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7725 		return;
7726 
7727 	obj_cgroup_uncharge(objcg, size);
7728 
7729 	rcu_read_lock();
7730 	memcg = obj_cgroup_memcg(objcg);
7731 	mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size);
7732 	mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1);
7733 	rcu_read_unlock();
7734 }
7735 
zswap_current_read(struct cgroup_subsys_state * css,struct cftype * cft)7736 static u64 zswap_current_read(struct cgroup_subsys_state *css,
7737 			      struct cftype *cft)
7738 {
7739 	cgroup_rstat_flush(css->cgroup);
7740 	return memcg_page_state(mem_cgroup_from_css(css), MEMCG_ZSWAP_B);
7741 }
7742 
zswap_max_show(struct seq_file * m,void * v)7743 static int zswap_max_show(struct seq_file *m, void *v)
7744 {
7745 	return seq_puts_memcg_tunable(m,
7746 		READ_ONCE(mem_cgroup_from_seq(m)->zswap_max));
7747 }
7748 
zswap_max_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)7749 static ssize_t zswap_max_write(struct kernfs_open_file *of,
7750 			       char *buf, size_t nbytes, loff_t off)
7751 {
7752 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7753 	unsigned long max;
7754 	int err;
7755 
7756 	buf = strstrip(buf);
7757 	err = page_counter_memparse(buf, "max", &max);
7758 	if (err)
7759 		return err;
7760 
7761 	xchg(&memcg->zswap_max, max);
7762 
7763 	return nbytes;
7764 }
7765 
7766 static struct cftype zswap_files[] = {
7767 	{
7768 		.name = "zswap.current",
7769 		.flags = CFTYPE_NOT_ON_ROOT,
7770 		.read_u64 = zswap_current_read,
7771 	},
7772 	{
7773 		.name = "zswap.max",
7774 		.flags = CFTYPE_NOT_ON_ROOT,
7775 		.seq_show = zswap_max_show,
7776 		.write = zswap_max_write,
7777 	},
7778 	{ }	/* terminate */
7779 };
7780 #endif /* CONFIG_MEMCG_KMEM && CONFIG_ZSWAP */
7781 
mem_cgroup_swap_init(void)7782 static int __init mem_cgroup_swap_init(void)
7783 {
7784 	if (mem_cgroup_disabled())
7785 		return 0;
7786 
7787 	WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7788 	WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7789 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
7790 	WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, zswap_files));
7791 #endif
7792 	return 0;
7793 }
7794 subsys_initcall(mem_cgroup_swap_init);
7795 
7796 #endif /* CONFIG_SWAP */
7797