1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* memcontrol.h - Memory Controller
3  *
4  * Copyright IBM Corporation, 2007
5  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6  *
7  * Copyright 2007 OpenVZ SWsoft Inc
8  * Author: Pavel Emelianov <xemul@openvz.org>
9  */
10 
11 #ifndef _LINUX_MEMCONTROL_H
12 #define _LINUX_MEMCONTROL_H
13 #include <linux/cgroup.h>
14 #include <linux/vm_event_item.h>
15 #include <linux/hardirq.h>
16 #include <linux/jump_label.h>
17 #include <linux/page_counter.h>
18 #include <linux/vmpressure.h>
19 #include <linux/eventfd.h>
20 #include <linux/mm.h>
21 #include <linux/vmstat.h>
22 #include <linux/writeback.h>
23 #include <linux/page-flags.h>
24 
25 struct mem_cgroup;
26 struct obj_cgroup;
27 struct page;
28 struct mm_struct;
29 struct kmem_cache;
30 
31 /* Cgroup-specific page state, on top of universal node page state */
32 enum memcg_stat_item {
33 	MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
34 	MEMCG_SOCK,
35 	MEMCG_PERCPU_B,
36 	MEMCG_VMALLOC,
37 	MEMCG_KMEM,
38 	MEMCG_ZSWAP_B,
39 	MEMCG_ZSWAPPED,
40 	MEMCG_NR_STAT,
41 };
42 
43 enum memcg_memory_event {
44 	MEMCG_LOW,
45 	MEMCG_HIGH,
46 	MEMCG_MAX,
47 	MEMCG_OOM,
48 	MEMCG_OOM_KILL,
49 	MEMCG_OOM_GROUP_KILL,
50 	MEMCG_SWAP_HIGH,
51 	MEMCG_SWAP_MAX,
52 	MEMCG_SWAP_FAIL,
53 	MEMCG_NR_MEMORY_EVENTS,
54 };
55 
56 struct mem_cgroup_reclaim_cookie {
57 	pg_data_t *pgdat;
58 	unsigned int generation;
59 };
60 
61 #ifdef CONFIG_MEMCG
62 
63 #define MEM_CGROUP_ID_SHIFT	16
64 #define MEM_CGROUP_ID_MAX	USHRT_MAX
65 
66 struct mem_cgroup_id {
67 	int id;
68 	refcount_t ref;
69 };
70 
71 /*
72  * Per memcg event counter is incremented at every pagein/pageout. With THP,
73  * it will be incremented by the number of pages. This counter is used
74  * to trigger some periodic events. This is straightforward and better
75  * than using jiffies etc. to handle periodic memcg event.
76  */
77 enum mem_cgroup_events_target {
78 	MEM_CGROUP_TARGET_THRESH,
79 	MEM_CGROUP_TARGET_SOFTLIMIT,
80 	MEM_CGROUP_NTARGETS,
81 };
82 
83 struct memcg_vmstats_percpu;
84 struct memcg_vmstats;
85 
86 struct mem_cgroup_reclaim_iter {
87 	struct mem_cgroup *position;
88 	/* scan generation, increased every round-trip */
89 	unsigned int generation;
90 };
91 
92 /*
93  * Bitmap and deferred work of shrinker::id corresponding to memcg-aware
94  * shrinkers, which have elements charged to this memcg.
95  */
96 struct shrinker_info {
97 	struct rcu_head rcu;
98 	atomic_long_t *nr_deferred;
99 	unsigned long *map;
100 };
101 
102 struct lruvec_stats_percpu {
103 	/* Local (CPU and cgroup) state */
104 	long state[NR_VM_NODE_STAT_ITEMS];
105 
106 	/* Delta calculation for lockless upward propagation */
107 	long state_prev[NR_VM_NODE_STAT_ITEMS];
108 };
109 
110 struct lruvec_stats {
111 	/* Aggregated (CPU and subtree) state */
112 	long state[NR_VM_NODE_STAT_ITEMS];
113 
114 	/* Pending child counts during tree propagation */
115 	long state_pending[NR_VM_NODE_STAT_ITEMS];
116 };
117 
118 /*
119  * per-node information in memory controller.
120  */
121 struct mem_cgroup_per_node {
122 	struct lruvec		lruvec;
123 
124 	struct lruvec_stats_percpu __percpu	*lruvec_stats_percpu;
125 	struct lruvec_stats			lruvec_stats;
126 
127 	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
128 
129 	struct mem_cgroup_reclaim_iter	iter;
130 
131 	struct shrinker_info __rcu	*shrinker_info;
132 
133 	struct rb_node		tree_node;	/* RB tree node */
134 	unsigned long		usage_in_excess;/* Set to the value by which */
135 						/* the soft limit is exceeded*/
136 	bool			on_tree;
137 	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
138 						/* use container_of	   */
139 };
140 
141 struct mem_cgroup_threshold {
142 	struct eventfd_ctx *eventfd;
143 	unsigned long threshold;
144 };
145 
146 /* For threshold */
147 struct mem_cgroup_threshold_ary {
148 	/* An array index points to threshold just below or equal to usage. */
149 	int current_threshold;
150 	/* Size of entries[] */
151 	unsigned int size;
152 	/* Array of thresholds */
153 	struct mem_cgroup_threshold entries[];
154 };
155 
156 struct mem_cgroup_thresholds {
157 	/* Primary thresholds array */
158 	struct mem_cgroup_threshold_ary *primary;
159 	/*
160 	 * Spare threshold array.
161 	 * This is needed to make mem_cgroup_unregister_event() "never fail".
162 	 * It must be able to store at least primary->size - 1 entries.
163 	 */
164 	struct mem_cgroup_threshold_ary *spare;
165 };
166 
167 /*
168  * Remember four most recent foreign writebacks with dirty pages in this
169  * cgroup.  Inode sharing is expected to be uncommon and, even if we miss
170  * one in a given round, we're likely to catch it later if it keeps
171  * foreign-dirtying, so a fairly low count should be enough.
172  *
173  * See mem_cgroup_track_foreign_dirty_slowpath() for details.
174  */
175 #define MEMCG_CGWB_FRN_CNT	4
176 
177 struct memcg_cgwb_frn {
178 	u64 bdi_id;			/* bdi->id of the foreign inode */
179 	int memcg_id;			/* memcg->css.id of foreign inode */
180 	u64 at;				/* jiffies_64 at the time of dirtying */
181 	struct wb_completion done;	/* tracks in-flight foreign writebacks */
182 };
183 
184 /*
185  * Bucket for arbitrarily byte-sized objects charged to a memory
186  * cgroup. The bucket can be reparented in one piece when the cgroup
187  * is destroyed, without having to round up the individual references
188  * of all live memory objects in the wild.
189  */
190 struct obj_cgroup {
191 	struct percpu_ref refcnt;
192 	struct mem_cgroup *memcg;
193 	atomic_t nr_charged_bytes;
194 	union {
195 		struct list_head list; /* protected by objcg_lock */
196 		struct rcu_head rcu;
197 	};
198 };
199 
200 /*
201  * The memory controller data structure. The memory controller controls both
202  * page cache and RSS per cgroup. We would eventually like to provide
203  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
204  * to help the administrator determine what knobs to tune.
205  */
206 struct mem_cgroup {
207 	struct cgroup_subsys_state css;
208 
209 	/* Private memcg ID. Used to ID objects that outlive the cgroup */
210 	struct mem_cgroup_id id;
211 
212 	/* Accounted resources */
213 	struct page_counter memory;		/* Both v1 & v2 */
214 
215 	union {
216 		struct page_counter swap;	/* v2 only */
217 		struct page_counter memsw;	/* v1 only */
218 	};
219 
220 	/* Legacy consumer-oriented counters */
221 	struct page_counter kmem;		/* v1 only */
222 	struct page_counter tcpmem;		/* v1 only */
223 
224 	/* Range enforcement for interrupt charges */
225 	struct work_struct high_work;
226 
227 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
228 	unsigned long zswap_max;
229 #endif
230 
231 	unsigned long soft_limit;
232 
233 	/* vmpressure notifications */
234 	struct vmpressure vmpressure;
235 
236 	/*
237 	 * Should the OOM killer kill all belonging tasks, had it kill one?
238 	 */
239 	bool oom_group;
240 
241 	/* protected by memcg_oom_lock */
242 	bool		oom_lock;
243 	int		under_oom;
244 
245 	int	swappiness;
246 	/* OOM-Killer disable */
247 	int		oom_kill_disable;
248 
249 	/* memory.events and memory.events.local */
250 	struct cgroup_file events_file;
251 	struct cgroup_file events_local_file;
252 
253 	/* handle for "memory.swap.events" */
254 	struct cgroup_file swap_events_file;
255 
256 	/* protect arrays of thresholds */
257 	struct mutex thresholds_lock;
258 
259 	/* thresholds for memory usage. RCU-protected */
260 	struct mem_cgroup_thresholds thresholds;
261 
262 	/* thresholds for mem+swap usage. RCU-protected */
263 	struct mem_cgroup_thresholds memsw_thresholds;
264 
265 	/* For oom notifier event fd */
266 	struct list_head oom_notify;
267 
268 	/*
269 	 * Should we move charges of a task when a task is moved into this
270 	 * mem_cgroup ? And what type of charges should we move ?
271 	 */
272 	unsigned long move_charge_at_immigrate;
273 	/* taken only while moving_account > 0 */
274 	spinlock_t		move_lock;
275 	unsigned long		move_lock_flags;
276 
277 	CACHELINE_PADDING(_pad1_);
278 
279 	/* memory.stat */
280 	struct memcg_vmstats	*vmstats;
281 
282 	/* memory.events */
283 	atomic_long_t		memory_events[MEMCG_NR_MEMORY_EVENTS];
284 	atomic_long_t		memory_events_local[MEMCG_NR_MEMORY_EVENTS];
285 
286 	unsigned long		socket_pressure;
287 
288 	/* Legacy tcp memory accounting */
289 	bool			tcpmem_active;
290 	int			tcpmem_pressure;
291 
292 #ifdef CONFIG_MEMCG_KMEM
293 	int kmemcg_id;
294 	struct obj_cgroup __rcu *objcg;
295 	/* list of inherited objcgs, protected by objcg_lock */
296 	struct list_head objcg_list;
297 #endif
298 
299 	CACHELINE_PADDING(_pad2_);
300 
301 	/*
302 	 * set > 0 if pages under this cgroup are moving to other cgroup.
303 	 */
304 	atomic_t		moving_account;
305 	struct task_struct	*move_lock_task;
306 
307 	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
308 
309 #ifdef CONFIG_CGROUP_WRITEBACK
310 	struct list_head cgwb_list;
311 	struct wb_domain cgwb_domain;
312 	struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
313 #endif
314 
315 	/* List of events which userspace want to receive */
316 	struct list_head event_list;
317 	spinlock_t event_list_lock;
318 
319 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
320 	struct deferred_split deferred_split_queue;
321 #endif
322 
323 #ifdef CONFIG_LRU_GEN
324 	/* per-memcg mm_struct list */
325 	struct lru_gen_mm_list mm_list;
326 #endif
327 
328 	struct mem_cgroup_per_node *nodeinfo[];
329 };
330 
331 /*
332  * size of first charge trial.
333  * TODO: maybe necessary to use big numbers in big irons or dynamic based of the
334  * workload.
335  */
336 #define MEMCG_CHARGE_BATCH 64U
337 
338 extern struct mem_cgroup *root_mem_cgroup;
339 
340 enum page_memcg_data_flags {
341 	/* page->memcg_data is a pointer to an objcgs vector */
342 	MEMCG_DATA_OBJCGS = (1UL << 0),
343 	/* page has been accounted as a non-slab kernel page */
344 	MEMCG_DATA_KMEM = (1UL << 1),
345 	/* the next bit after the last actual flag */
346 	__NR_MEMCG_DATA_FLAGS  = (1UL << 2),
347 };
348 
349 #define MEMCG_DATA_FLAGS_MASK (__NR_MEMCG_DATA_FLAGS - 1)
350 
351 static inline bool folio_memcg_kmem(struct folio *folio);
352 
353 /*
354  * After the initialization objcg->memcg is always pointing at
355  * a valid memcg, but can be atomically swapped to the parent memcg.
356  *
357  * The caller must ensure that the returned memcg won't be released:
358  * e.g. acquire the rcu_read_lock or css_set_lock.
359  */
obj_cgroup_memcg(struct obj_cgroup * objcg)360 static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
361 {
362 	return READ_ONCE(objcg->memcg);
363 }
364 
365 /*
366  * __folio_memcg - Get the memory cgroup associated with a non-kmem folio
367  * @folio: Pointer to the folio.
368  *
369  * Returns a pointer to the memory cgroup associated with the folio,
370  * or NULL. This function assumes that the folio is known to have a
371  * proper memory cgroup pointer. It's not safe to call this function
372  * against some type of folios, e.g. slab folios or ex-slab folios or
373  * kmem folios.
374  */
__folio_memcg(struct folio * folio)375 static inline struct mem_cgroup *__folio_memcg(struct folio *folio)
376 {
377 	unsigned long memcg_data = folio->memcg_data;
378 
379 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
380 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);
381 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);
382 
383 	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
384 }
385 
386 /*
387  * __folio_objcg - get the object cgroup associated with a kmem folio.
388  * @folio: Pointer to the folio.
389  *
390  * Returns a pointer to the object cgroup associated with the folio,
391  * or NULL. This function assumes that the folio is known to have a
392  * proper object cgroup pointer. It's not safe to call this function
393  * against some type of folios, e.g. slab folios or ex-slab folios or
394  * LRU folios.
395  */
__folio_objcg(struct folio * folio)396 static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
397 {
398 	unsigned long memcg_data = folio->memcg_data;
399 
400 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
401 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);
402 	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);
403 
404 	return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
405 }
406 
407 /*
408  * folio_memcg - Get the memory cgroup associated with a folio.
409  * @folio: Pointer to the folio.
410  *
411  * Returns a pointer to the memory cgroup associated with the folio,
412  * or NULL. This function assumes that the folio is known to have a
413  * proper memory cgroup pointer. It's not safe to call this function
414  * against some type of folios, e.g. slab folios or ex-slab folios.
415  *
416  * For a non-kmem folio any of the following ensures folio and memcg binding
417  * stability:
418  *
419  * - the folio lock
420  * - LRU isolation
421  * - lock_page_memcg()
422  * - exclusive reference
423  * - mem_cgroup_trylock_pages()
424  *
425  * For a kmem folio a caller should hold an rcu read lock to protect memcg
426  * associated with a kmem folio from being released.
427  */
folio_memcg(struct folio * folio)428 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
429 {
430 	if (folio_memcg_kmem(folio))
431 		return obj_cgroup_memcg(__folio_objcg(folio));
432 	return __folio_memcg(folio);
433 }
434 
page_memcg(struct page * page)435 static inline struct mem_cgroup *page_memcg(struct page *page)
436 {
437 	return folio_memcg(page_folio(page));
438 }
439 
440 /**
441  * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio.
442  * @folio: Pointer to the folio.
443  *
444  * This function assumes that the folio is known to have a
445  * proper memory cgroup pointer. It's not safe to call this function
446  * against some type of folios, e.g. slab folios or ex-slab folios.
447  *
448  * Return: A pointer to the memory cgroup associated with the folio,
449  * or NULL.
450  */
folio_memcg_rcu(struct folio * folio)451 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
452 {
453 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);
454 
455 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
456 	WARN_ON_ONCE(!rcu_read_lock_held());
457 
458 	if (memcg_data & MEMCG_DATA_KMEM) {
459 		struct obj_cgroup *objcg;
460 
461 		objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
462 		return obj_cgroup_memcg(objcg);
463 	}
464 
465 	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
466 }
467 
468 /*
469  * page_memcg_check - get the memory cgroup associated with a page
470  * @page: a pointer to the page struct
471  *
472  * Returns a pointer to the memory cgroup associated with the page,
473  * or NULL. This function unlike page_memcg() can take any page
474  * as an argument. It has to be used in cases when it's not known if a page
475  * has an associated memory cgroup pointer or an object cgroups vector or
476  * an object cgroup.
477  *
478  * For a non-kmem page any of the following ensures page and memcg binding
479  * stability:
480  *
481  * - the page lock
482  * - LRU isolation
483  * - lock_page_memcg()
484  * - exclusive reference
485  * - mem_cgroup_trylock_pages()
486  *
487  * For a kmem page a caller should hold an rcu read lock to protect memcg
488  * associated with a kmem page from being released.
489  */
page_memcg_check(struct page * page)490 static inline struct mem_cgroup *page_memcg_check(struct page *page)
491 {
492 	/*
493 	 * Because page->memcg_data might be changed asynchronously
494 	 * for slab pages, READ_ONCE() should be used here.
495 	 */
496 	unsigned long memcg_data = READ_ONCE(page->memcg_data);
497 
498 	if (memcg_data & MEMCG_DATA_OBJCGS)
499 		return NULL;
500 
501 	if (memcg_data & MEMCG_DATA_KMEM) {
502 		struct obj_cgroup *objcg;
503 
504 		objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
505 		return obj_cgroup_memcg(objcg);
506 	}
507 
508 	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
509 }
510 
get_mem_cgroup_from_objcg(struct obj_cgroup * objcg)511 static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
512 {
513 	struct mem_cgroup *memcg;
514 
515 	rcu_read_lock();
516 retry:
517 	memcg = obj_cgroup_memcg(objcg);
518 	if (unlikely(!css_tryget(&memcg->css)))
519 		goto retry;
520 	rcu_read_unlock();
521 
522 	return memcg;
523 }
524 
525 #ifdef CONFIG_MEMCG_KMEM
526 /*
527  * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set.
528  * @folio: Pointer to the folio.
529  *
530  * Checks if the folio has MemcgKmem flag set. The caller must ensure
531  * that the folio has an associated memory cgroup. It's not safe to call
532  * this function against some types of folios, e.g. slab folios.
533  */
folio_memcg_kmem(struct folio * folio)534 static inline bool folio_memcg_kmem(struct folio *folio)
535 {
536 	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);
537 	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio);
538 	return folio->memcg_data & MEMCG_DATA_KMEM;
539 }
540 
541 
542 #else
folio_memcg_kmem(struct folio * folio)543 static inline bool folio_memcg_kmem(struct folio *folio)
544 {
545 	return false;
546 }
547 
548 #endif
549 
PageMemcgKmem(struct page * page)550 static inline bool PageMemcgKmem(struct page *page)
551 {
552 	return folio_memcg_kmem(page_folio(page));
553 }
554 
mem_cgroup_is_root(struct mem_cgroup * memcg)555 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
556 {
557 	return (memcg == root_mem_cgroup);
558 }
559 
mem_cgroup_disabled(void)560 static inline bool mem_cgroup_disabled(void)
561 {
562 	return !cgroup_subsys_enabled(memory_cgrp_subsys);
563 }
564 
mem_cgroup_protection(struct mem_cgroup * root,struct mem_cgroup * memcg,unsigned long * min,unsigned long * low)565 static inline void mem_cgroup_protection(struct mem_cgroup *root,
566 					 struct mem_cgroup *memcg,
567 					 unsigned long *min,
568 					 unsigned long *low)
569 {
570 	*min = *low = 0;
571 
572 	if (mem_cgroup_disabled())
573 		return;
574 
575 	/*
576 	 * There is no reclaim protection applied to a targeted reclaim.
577 	 * We are special casing this specific case here because
578 	 * mem_cgroup_protected calculation is not robust enough to keep
579 	 * the protection invariant for calculated effective values for
580 	 * parallel reclaimers with different reclaim target. This is
581 	 * especially a problem for tail memcgs (as they have pages on LRU)
582 	 * which would want to have effective values 0 for targeted reclaim
583 	 * but a different value for external reclaim.
584 	 *
585 	 * Example
586 	 * Let's have global and A's reclaim in parallel:
587 	 *  |
588 	 *  A (low=2G, usage = 3G, max = 3G, children_low_usage = 1.5G)
589 	 *  |\
590 	 *  | C (low = 1G, usage = 2.5G)
591 	 *  B (low = 1G, usage = 0.5G)
592 	 *
593 	 * For the global reclaim
594 	 * A.elow = A.low
595 	 * B.elow = min(B.usage, B.low) because children_low_usage <= A.elow
596 	 * C.elow = min(C.usage, C.low)
597 	 *
598 	 * With the effective values resetting we have A reclaim
599 	 * A.elow = 0
600 	 * B.elow = B.low
601 	 * C.elow = C.low
602 	 *
603 	 * If the global reclaim races with A's reclaim then
604 	 * B.elow = C.elow = 0 because children_low_usage > A.elow)
605 	 * is possible and reclaiming B would be violating the protection.
606 	 *
607 	 */
608 	if (root == memcg)
609 		return;
610 
611 	*min = READ_ONCE(memcg->memory.emin);
612 	*low = READ_ONCE(memcg->memory.elow);
613 }
614 
615 void mem_cgroup_calculate_protection(struct mem_cgroup *root,
616 				     struct mem_cgroup *memcg);
617 
mem_cgroup_supports_protection(struct mem_cgroup * memcg)618 static inline bool mem_cgroup_supports_protection(struct mem_cgroup *memcg)
619 {
620 	/*
621 	 * The root memcg doesn't account charges, and doesn't support
622 	 * protection.
623 	 */
624 	return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg);
625 
626 }
627 
mem_cgroup_below_low(struct mem_cgroup * memcg)628 static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg)
629 {
630 	if (!mem_cgroup_supports_protection(memcg))
631 		return false;
632 
633 	return READ_ONCE(memcg->memory.elow) >=
634 		page_counter_read(&memcg->memory);
635 }
636 
mem_cgroup_below_min(struct mem_cgroup * memcg)637 static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
638 {
639 	if (!mem_cgroup_supports_protection(memcg))
640 		return false;
641 
642 	return READ_ONCE(memcg->memory.emin) >=
643 		page_counter_read(&memcg->memory);
644 }
645 
646 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
647 
648 /**
649  * mem_cgroup_charge - Charge a newly allocated folio to a cgroup.
650  * @folio: Folio to charge.
651  * @mm: mm context of the allocating task.
652  * @gfp: Reclaim mode.
653  *
654  * Try to charge @folio to the memcg that @mm belongs to, reclaiming
655  * pages according to @gfp if necessary.  If @mm is NULL, try to
656  * charge to the active memcg.
657  *
658  * Do not use this for folios allocated for swapin.
659  *
660  * Return: 0 on success. Otherwise, an error code is returned.
661  */
mem_cgroup_charge(struct folio * folio,struct mm_struct * mm,gfp_t gfp)662 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,
663 				    gfp_t gfp)
664 {
665 	if (mem_cgroup_disabled())
666 		return 0;
667 	return __mem_cgroup_charge(folio, mm, gfp);
668 }
669 
670 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
671 				  gfp_t gfp, swp_entry_t entry);
672 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
673 
674 void __mem_cgroup_uncharge(struct folio *folio);
675 
676 /**
677  * mem_cgroup_uncharge - Uncharge a folio.
678  * @folio: Folio to uncharge.
679  *
680  * Uncharge a folio previously charged with mem_cgroup_charge().
681  */
mem_cgroup_uncharge(struct folio * folio)682 static inline void mem_cgroup_uncharge(struct folio *folio)
683 {
684 	if (mem_cgroup_disabled())
685 		return;
686 	__mem_cgroup_uncharge(folio);
687 }
688 
689 void __mem_cgroup_uncharge_list(struct list_head *page_list);
mem_cgroup_uncharge_list(struct list_head * page_list)690 static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
691 {
692 	if (mem_cgroup_disabled())
693 		return;
694 	__mem_cgroup_uncharge_list(page_list);
695 }
696 
697 void mem_cgroup_migrate(struct folio *old, struct folio *new);
698 
699 /**
700  * mem_cgroup_lruvec - get the lru list vector for a memcg & node
701  * @memcg: memcg of the wanted lruvec
702  * @pgdat: pglist_data
703  *
704  * Returns the lru list vector holding pages for a given @memcg &
705  * @pgdat combination. This can be the node lruvec, if the memory
706  * controller is disabled.
707  */
mem_cgroup_lruvec(struct mem_cgroup * memcg,struct pglist_data * pgdat)708 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
709 					       struct pglist_data *pgdat)
710 {
711 	struct mem_cgroup_per_node *mz;
712 	struct lruvec *lruvec;
713 
714 	if (mem_cgroup_disabled()) {
715 		lruvec = &pgdat->__lruvec;
716 		goto out;
717 	}
718 
719 	if (!memcg)
720 		memcg = root_mem_cgroup;
721 
722 	mz = memcg->nodeinfo[pgdat->node_id];
723 	lruvec = &mz->lruvec;
724 out:
725 	/*
726 	 * Since a node can be onlined after the mem_cgroup was created,
727 	 * we have to be prepared to initialize lruvec->pgdat here;
728 	 * and if offlined then reonlined, we need to reinitialize it.
729 	 */
730 	if (unlikely(lruvec->pgdat != pgdat))
731 		lruvec->pgdat = pgdat;
732 	return lruvec;
733 }
734 
735 /**
736  * folio_lruvec - return lruvec for isolating/putting an LRU folio
737  * @folio: Pointer to the folio.
738  *
739  * This function relies on folio->mem_cgroup being stable.
740  */
folio_lruvec(struct folio * folio)741 static inline struct lruvec *folio_lruvec(struct folio *folio)
742 {
743 	struct mem_cgroup *memcg = folio_memcg(folio);
744 
745 	VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);
746 	return mem_cgroup_lruvec(memcg, folio_pgdat(folio));
747 }
748 
749 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
750 
751 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
752 
753 struct lruvec *folio_lruvec_lock(struct folio *folio);
754 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
755 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
756 						unsigned long *flags);
757 
758 #ifdef CONFIG_DEBUG_VM
759 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
760 #else
761 static inline
lruvec_memcg_debug(struct lruvec * lruvec,struct folio * folio)762 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
763 {
764 }
765 #endif
766 
767 static inline
mem_cgroup_from_css(struct cgroup_subsys_state * css)768 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
769 	return css ? container_of(css, struct mem_cgroup, css) : NULL;
770 }
771 
obj_cgroup_tryget(struct obj_cgroup * objcg)772 static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg)
773 {
774 	return percpu_ref_tryget(&objcg->refcnt);
775 }
776 
obj_cgroup_get(struct obj_cgroup * objcg)777 static inline void obj_cgroup_get(struct obj_cgroup *objcg)
778 {
779 	percpu_ref_get(&objcg->refcnt);
780 }
781 
obj_cgroup_get_many(struct obj_cgroup * objcg,unsigned long nr)782 static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
783 				       unsigned long nr)
784 {
785 	percpu_ref_get_many(&objcg->refcnt, nr);
786 }
787 
obj_cgroup_put(struct obj_cgroup * objcg)788 static inline void obj_cgroup_put(struct obj_cgroup *objcg)
789 {
790 	percpu_ref_put(&objcg->refcnt);
791 }
792 
mem_cgroup_put(struct mem_cgroup * memcg)793 static inline void mem_cgroup_put(struct mem_cgroup *memcg)
794 {
795 	if (memcg)
796 		css_put(&memcg->css);
797 }
798 
799 #define mem_cgroup_from_counter(counter, member)	\
800 	container_of(counter, struct mem_cgroup, member)
801 
802 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
803 				   struct mem_cgroup *,
804 				   struct mem_cgroup_reclaim_cookie *);
805 void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
806 int mem_cgroup_scan_tasks(struct mem_cgroup *,
807 			  int (*)(struct task_struct *, void *), void *);
808 
mem_cgroup_id(struct mem_cgroup * memcg)809 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
810 {
811 	if (mem_cgroup_disabled())
812 		return 0;
813 
814 	return memcg->id.id;
815 }
816 struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
817 
818 #ifdef CONFIG_SHRINKER_DEBUG
mem_cgroup_ino(struct mem_cgroup * memcg)819 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)
820 {
821 	return memcg ? cgroup_ino(memcg->css.cgroup) : 0;
822 }
823 
824 struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino);
825 #endif
826 
mem_cgroup_from_seq(struct seq_file * m)827 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
828 {
829 	return mem_cgroup_from_css(seq_css(m));
830 }
831 
lruvec_memcg(struct lruvec * lruvec)832 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
833 {
834 	struct mem_cgroup_per_node *mz;
835 
836 	if (mem_cgroup_disabled())
837 		return NULL;
838 
839 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
840 	return mz->memcg;
841 }
842 
843 /**
844  * parent_mem_cgroup - find the accounting parent of a memcg
845  * @memcg: memcg whose parent to find
846  *
847  * Returns the parent memcg, or NULL if this is the root or the memory
848  * controller is in legacy no-hierarchy mode.
849  */
parent_mem_cgroup(struct mem_cgroup * memcg)850 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
851 {
852 	return mem_cgroup_from_css(memcg->css.parent);
853 }
854 
mem_cgroup_is_descendant(struct mem_cgroup * memcg,struct mem_cgroup * root)855 static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
856 			      struct mem_cgroup *root)
857 {
858 	if (root == memcg)
859 		return true;
860 	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
861 }
862 
mm_match_cgroup(struct mm_struct * mm,struct mem_cgroup * memcg)863 static inline bool mm_match_cgroup(struct mm_struct *mm,
864 				   struct mem_cgroup *memcg)
865 {
866 	struct mem_cgroup *task_memcg;
867 	bool match = false;
868 
869 	rcu_read_lock();
870 	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
871 	if (task_memcg)
872 		match = mem_cgroup_is_descendant(task_memcg, memcg);
873 	rcu_read_unlock();
874 	return match;
875 }
876 
877 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
878 ino_t page_cgroup_ino(struct page *page);
879 
mem_cgroup_online(struct mem_cgroup * memcg)880 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
881 {
882 	if (mem_cgroup_disabled())
883 		return true;
884 	return !!(memcg->css.flags & CSS_ONLINE);
885 }
886 
887 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
888 		int zid, int nr_pages);
889 
890 static inline
mem_cgroup_get_zone_lru_size(struct lruvec * lruvec,enum lru_list lru,int zone_idx)891 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
892 		enum lru_list lru, int zone_idx)
893 {
894 	struct mem_cgroup_per_node *mz;
895 
896 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
897 	return READ_ONCE(mz->lru_zone_size[zone_idx][lru]);
898 }
899 
900 void mem_cgroup_handle_over_high(void);
901 
902 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
903 
904 unsigned long mem_cgroup_size(struct mem_cgroup *memcg);
905 
906 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
907 				struct task_struct *p);
908 
909 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
910 
mem_cgroup_enter_user_fault(void)911 static inline void mem_cgroup_enter_user_fault(void)
912 {
913 	WARN_ON(current->in_user_fault);
914 	current->in_user_fault = 1;
915 }
916 
mem_cgroup_exit_user_fault(void)917 static inline void mem_cgroup_exit_user_fault(void)
918 {
919 	WARN_ON(!current->in_user_fault);
920 	current->in_user_fault = 0;
921 }
922 
task_in_memcg_oom(struct task_struct * p)923 static inline bool task_in_memcg_oom(struct task_struct *p)
924 {
925 	return p->memcg_in_oom;
926 }
927 
928 bool mem_cgroup_oom_synchronize(bool wait);
929 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
930 					    struct mem_cgroup *oom_domain);
931 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
932 
933 void folio_memcg_lock(struct folio *folio);
934 void folio_memcg_unlock(struct folio *folio);
935 void lock_page_memcg(struct page *page);
936 void unlock_page_memcg(struct page *page);
937 
938 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
939 
940 /* try to stablize folio_memcg() for all the pages in a memcg */
mem_cgroup_trylock_pages(struct mem_cgroup * memcg)941 static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
942 {
943 	rcu_read_lock();
944 
945 	if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account))
946 		return true;
947 
948 	rcu_read_unlock();
949 	return false;
950 }
951 
mem_cgroup_unlock_pages(void)952 static inline void mem_cgroup_unlock_pages(void)
953 {
954 	rcu_read_unlock();
955 }
956 
957 /* idx can be of type enum memcg_stat_item or node_stat_item */
mod_memcg_state(struct mem_cgroup * memcg,int idx,int val)958 static inline void mod_memcg_state(struct mem_cgroup *memcg,
959 				   int idx, int val)
960 {
961 	unsigned long flags;
962 
963 	local_irq_save(flags);
964 	__mod_memcg_state(memcg, idx, val);
965 	local_irq_restore(flags);
966 }
967 
mod_memcg_page_state(struct page * page,int idx,int val)968 static inline void mod_memcg_page_state(struct page *page,
969 					int idx, int val)
970 {
971 	struct mem_cgroup *memcg;
972 
973 	if (mem_cgroup_disabled())
974 		return;
975 
976 	rcu_read_lock();
977 	memcg = page_memcg(page);
978 	if (memcg)
979 		mod_memcg_state(memcg, idx, val);
980 	rcu_read_unlock();
981 }
982 
983 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx);
984 
lruvec_page_state(struct lruvec * lruvec,enum node_stat_item idx)985 static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
986 					      enum node_stat_item idx)
987 {
988 	struct mem_cgroup_per_node *pn;
989 	long x;
990 
991 	if (mem_cgroup_disabled())
992 		return node_page_state(lruvec_pgdat(lruvec), idx);
993 
994 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
995 	x = READ_ONCE(pn->lruvec_stats.state[idx]);
996 #ifdef CONFIG_SMP
997 	if (x < 0)
998 		x = 0;
999 #endif
1000 	return x;
1001 }
1002 
lruvec_page_state_local(struct lruvec * lruvec,enum node_stat_item idx)1003 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
1004 						    enum node_stat_item idx)
1005 {
1006 	struct mem_cgroup_per_node *pn;
1007 	long x = 0;
1008 	int cpu;
1009 
1010 	if (mem_cgroup_disabled())
1011 		return node_page_state(lruvec_pgdat(lruvec), idx);
1012 
1013 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1014 	for_each_possible_cpu(cpu)
1015 		x += per_cpu(pn->lruvec_stats_percpu->state[idx], cpu);
1016 #ifdef CONFIG_SMP
1017 	if (x < 0)
1018 		x = 0;
1019 #endif
1020 	return x;
1021 }
1022 
1023 void mem_cgroup_flush_stats(void);
1024 void mem_cgroup_flush_stats_delayed(void);
1025 
1026 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
1027 			      int val);
1028 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val);
1029 
mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)1030 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1031 					 int val)
1032 {
1033 	unsigned long flags;
1034 
1035 	local_irq_save(flags);
1036 	__mod_lruvec_kmem_state(p, idx, val);
1037 	local_irq_restore(flags);
1038 }
1039 
mod_memcg_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)1040 static inline void mod_memcg_lruvec_state(struct lruvec *lruvec,
1041 					  enum node_stat_item idx, int val)
1042 {
1043 	unsigned long flags;
1044 
1045 	local_irq_save(flags);
1046 	__mod_memcg_lruvec_state(lruvec, idx, val);
1047 	local_irq_restore(flags);
1048 }
1049 
1050 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
1051 			  unsigned long count);
1052 
count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)1053 static inline void count_memcg_events(struct mem_cgroup *memcg,
1054 				      enum vm_event_item idx,
1055 				      unsigned long count)
1056 {
1057 	unsigned long flags;
1058 
1059 	local_irq_save(flags);
1060 	__count_memcg_events(memcg, idx, count);
1061 	local_irq_restore(flags);
1062 }
1063 
count_memcg_page_event(struct page * page,enum vm_event_item idx)1064 static inline void count_memcg_page_event(struct page *page,
1065 					  enum vm_event_item idx)
1066 {
1067 	struct mem_cgroup *memcg = page_memcg(page);
1068 
1069 	if (memcg)
1070 		count_memcg_events(memcg, idx, 1);
1071 }
1072 
count_memcg_folio_events(struct folio * folio,enum vm_event_item idx,unsigned long nr)1073 static inline void count_memcg_folio_events(struct folio *folio,
1074 		enum vm_event_item idx, unsigned long nr)
1075 {
1076 	struct mem_cgroup *memcg = folio_memcg(folio);
1077 
1078 	if (memcg)
1079 		count_memcg_events(memcg, idx, nr);
1080 }
1081 
count_memcg_event_mm(struct mm_struct * mm,enum vm_event_item idx)1082 static inline void count_memcg_event_mm(struct mm_struct *mm,
1083 					enum vm_event_item idx)
1084 {
1085 	struct mem_cgroup *memcg;
1086 
1087 	if (mem_cgroup_disabled())
1088 		return;
1089 
1090 	rcu_read_lock();
1091 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1092 	if (likely(memcg))
1093 		count_memcg_events(memcg, idx, 1);
1094 	rcu_read_unlock();
1095 }
1096 
memcg_memory_event(struct mem_cgroup * memcg,enum memcg_memory_event event)1097 static inline void memcg_memory_event(struct mem_cgroup *memcg,
1098 				      enum memcg_memory_event event)
1099 {
1100 	bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX ||
1101 			  event == MEMCG_SWAP_FAIL;
1102 
1103 	atomic_long_inc(&memcg->memory_events_local[event]);
1104 	if (!swap_event)
1105 		cgroup_file_notify(&memcg->events_local_file);
1106 
1107 	do {
1108 		atomic_long_inc(&memcg->memory_events[event]);
1109 		if (swap_event)
1110 			cgroup_file_notify(&memcg->swap_events_file);
1111 		else
1112 			cgroup_file_notify(&memcg->events_file);
1113 
1114 		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1115 			break;
1116 		if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS)
1117 			break;
1118 	} while ((memcg = parent_mem_cgroup(memcg)) &&
1119 		 !mem_cgroup_is_root(memcg));
1120 }
1121 
memcg_memory_event_mm(struct mm_struct * mm,enum memcg_memory_event event)1122 static inline void memcg_memory_event_mm(struct mm_struct *mm,
1123 					 enum memcg_memory_event event)
1124 {
1125 	struct mem_cgroup *memcg;
1126 
1127 	if (mem_cgroup_disabled())
1128 		return;
1129 
1130 	rcu_read_lock();
1131 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1132 	if (likely(memcg))
1133 		memcg_memory_event(memcg, event);
1134 	rcu_read_unlock();
1135 }
1136 
1137 void split_page_memcg(struct page *head, unsigned int nr);
1138 
1139 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1140 						gfp_t gfp_mask,
1141 						unsigned long *total_scanned);
1142 
1143 #else /* CONFIG_MEMCG */
1144 
1145 #define MEM_CGROUP_ID_SHIFT	0
1146 #define MEM_CGROUP_ID_MAX	0
1147 
folio_memcg(struct folio * folio)1148 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
1149 {
1150 	return NULL;
1151 }
1152 
page_memcg(struct page * page)1153 static inline struct mem_cgroup *page_memcg(struct page *page)
1154 {
1155 	return NULL;
1156 }
1157 
folio_memcg_rcu(struct folio * folio)1158 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
1159 {
1160 	WARN_ON_ONCE(!rcu_read_lock_held());
1161 	return NULL;
1162 }
1163 
page_memcg_check(struct page * page)1164 static inline struct mem_cgroup *page_memcg_check(struct page *page)
1165 {
1166 	return NULL;
1167 }
1168 
folio_memcg_kmem(struct folio * folio)1169 static inline bool folio_memcg_kmem(struct folio *folio)
1170 {
1171 	return false;
1172 }
1173 
PageMemcgKmem(struct page * page)1174 static inline bool PageMemcgKmem(struct page *page)
1175 {
1176 	return false;
1177 }
1178 
mem_cgroup_is_root(struct mem_cgroup * memcg)1179 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
1180 {
1181 	return true;
1182 }
1183 
mem_cgroup_disabled(void)1184 static inline bool mem_cgroup_disabled(void)
1185 {
1186 	return true;
1187 }
1188 
memcg_memory_event(struct mem_cgroup * memcg,enum memcg_memory_event event)1189 static inline void memcg_memory_event(struct mem_cgroup *memcg,
1190 				      enum memcg_memory_event event)
1191 {
1192 }
1193 
memcg_memory_event_mm(struct mm_struct * mm,enum memcg_memory_event event)1194 static inline void memcg_memory_event_mm(struct mm_struct *mm,
1195 					 enum memcg_memory_event event)
1196 {
1197 }
1198 
mem_cgroup_protection(struct mem_cgroup * root,struct mem_cgroup * memcg,unsigned long * min,unsigned long * low)1199 static inline void mem_cgroup_protection(struct mem_cgroup *root,
1200 					 struct mem_cgroup *memcg,
1201 					 unsigned long *min,
1202 					 unsigned long *low)
1203 {
1204 	*min = *low = 0;
1205 }
1206 
mem_cgroup_calculate_protection(struct mem_cgroup * root,struct mem_cgroup * memcg)1207 static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root,
1208 						   struct mem_cgroup *memcg)
1209 {
1210 }
1211 
mem_cgroup_below_low(struct mem_cgroup * memcg)1212 static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg)
1213 {
1214 	return false;
1215 }
1216 
mem_cgroup_below_min(struct mem_cgroup * memcg)1217 static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
1218 {
1219 	return false;
1220 }
1221 
mem_cgroup_charge(struct folio * folio,struct mm_struct * mm,gfp_t gfp)1222 static inline int mem_cgroup_charge(struct folio *folio,
1223 		struct mm_struct *mm, gfp_t gfp)
1224 {
1225 	return 0;
1226 }
1227 
mem_cgroup_swapin_charge_folio(struct folio * folio,struct mm_struct * mm,gfp_t gfp,swp_entry_t entry)1228 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,
1229 			struct mm_struct *mm, gfp_t gfp, swp_entry_t entry)
1230 {
1231 	return 0;
1232 }
1233 
mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)1234 static inline void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
1235 {
1236 }
1237 
mem_cgroup_uncharge(struct folio * folio)1238 static inline void mem_cgroup_uncharge(struct folio *folio)
1239 {
1240 }
1241 
mem_cgroup_uncharge_list(struct list_head * page_list)1242 static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
1243 {
1244 }
1245 
mem_cgroup_migrate(struct folio * old,struct folio * new)1246 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)
1247 {
1248 }
1249 
mem_cgroup_lruvec(struct mem_cgroup * memcg,struct pglist_data * pgdat)1250 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
1251 					       struct pglist_data *pgdat)
1252 {
1253 	return &pgdat->__lruvec;
1254 }
1255 
folio_lruvec(struct folio * folio)1256 static inline struct lruvec *folio_lruvec(struct folio *folio)
1257 {
1258 	struct pglist_data *pgdat = folio_pgdat(folio);
1259 	return &pgdat->__lruvec;
1260 }
1261 
1262 static inline
lruvec_memcg_debug(struct lruvec * lruvec,struct folio * folio)1263 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
1264 {
1265 }
1266 
parent_mem_cgroup(struct mem_cgroup * memcg)1267 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
1268 {
1269 	return NULL;
1270 }
1271 
mm_match_cgroup(struct mm_struct * mm,struct mem_cgroup * memcg)1272 static inline bool mm_match_cgroup(struct mm_struct *mm,
1273 		struct mem_cgroup *memcg)
1274 {
1275 	return true;
1276 }
1277 
get_mem_cgroup_from_mm(struct mm_struct * mm)1278 static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1279 {
1280 	return NULL;
1281 }
1282 
1283 static inline
mem_cgroup_from_css(struct cgroup_subsys_state * css)1284 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
1285 {
1286 	return NULL;
1287 }
1288 
obj_cgroup_put(struct obj_cgroup * objcg)1289 static inline void obj_cgroup_put(struct obj_cgroup *objcg)
1290 {
1291 }
1292 
mem_cgroup_put(struct mem_cgroup * memcg)1293 static inline void mem_cgroup_put(struct mem_cgroup *memcg)
1294 {
1295 }
1296 
folio_lruvec_lock(struct folio * folio)1297 static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
1298 {
1299 	struct pglist_data *pgdat = folio_pgdat(folio);
1300 
1301 	spin_lock(&pgdat->__lruvec.lru_lock);
1302 	return &pgdat->__lruvec;
1303 }
1304 
folio_lruvec_lock_irq(struct folio * folio)1305 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
1306 {
1307 	struct pglist_data *pgdat = folio_pgdat(folio);
1308 
1309 	spin_lock_irq(&pgdat->__lruvec.lru_lock);
1310 	return &pgdat->__lruvec;
1311 }
1312 
folio_lruvec_lock_irqsave(struct folio * folio,unsigned long * flagsp)1313 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
1314 		unsigned long *flagsp)
1315 {
1316 	struct pglist_data *pgdat = folio_pgdat(folio);
1317 
1318 	spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
1319 	return &pgdat->__lruvec;
1320 }
1321 
1322 static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup * root,struct mem_cgroup * prev,struct mem_cgroup_reclaim_cookie * reclaim)1323 mem_cgroup_iter(struct mem_cgroup *root,
1324 		struct mem_cgroup *prev,
1325 		struct mem_cgroup_reclaim_cookie *reclaim)
1326 {
1327 	return NULL;
1328 }
1329 
mem_cgroup_iter_break(struct mem_cgroup * root,struct mem_cgroup * prev)1330 static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
1331 					 struct mem_cgroup *prev)
1332 {
1333 }
1334 
mem_cgroup_scan_tasks(struct mem_cgroup * memcg,int (* fn)(struct task_struct *,void *),void * arg)1335 static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1336 		int (*fn)(struct task_struct *, void *), void *arg)
1337 {
1338 	return 0;
1339 }
1340 
mem_cgroup_id(struct mem_cgroup * memcg)1341 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
1342 {
1343 	return 0;
1344 }
1345 
mem_cgroup_from_id(unsigned short id)1346 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
1347 {
1348 	WARN_ON_ONCE(id);
1349 	/* XXX: This should always return root_mem_cgroup */
1350 	return NULL;
1351 }
1352 
1353 #ifdef CONFIG_SHRINKER_DEBUG
mem_cgroup_ino(struct mem_cgroup * memcg)1354 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)
1355 {
1356 	return 0;
1357 }
1358 
mem_cgroup_get_from_ino(unsigned long ino)1359 static inline struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino)
1360 {
1361 	return NULL;
1362 }
1363 #endif
1364 
mem_cgroup_from_seq(struct seq_file * m)1365 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
1366 {
1367 	return NULL;
1368 }
1369 
lruvec_memcg(struct lruvec * lruvec)1370 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
1371 {
1372 	return NULL;
1373 }
1374 
mem_cgroup_online(struct mem_cgroup * memcg)1375 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
1376 {
1377 	return true;
1378 }
1379 
1380 static inline
mem_cgroup_get_zone_lru_size(struct lruvec * lruvec,enum lru_list lru,int zone_idx)1381 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
1382 		enum lru_list lru, int zone_idx)
1383 {
1384 	return 0;
1385 }
1386 
mem_cgroup_get_max(struct mem_cgroup * memcg)1387 static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1388 {
1389 	return 0;
1390 }
1391 
mem_cgroup_size(struct mem_cgroup * memcg)1392 static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1393 {
1394 	return 0;
1395 }
1396 
1397 static inline void
mem_cgroup_print_oom_context(struct mem_cgroup * memcg,struct task_struct * p)1398 mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1399 {
1400 }
1401 
1402 static inline void
mem_cgroup_print_oom_meminfo(struct mem_cgroup * memcg)1403 mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1404 {
1405 }
1406 
lock_page_memcg(struct page * page)1407 static inline void lock_page_memcg(struct page *page)
1408 {
1409 }
1410 
unlock_page_memcg(struct page * page)1411 static inline void unlock_page_memcg(struct page *page)
1412 {
1413 }
1414 
folio_memcg_lock(struct folio * folio)1415 static inline void folio_memcg_lock(struct folio *folio)
1416 {
1417 }
1418 
folio_memcg_unlock(struct folio * folio)1419 static inline void folio_memcg_unlock(struct folio *folio)
1420 {
1421 }
1422 
mem_cgroup_trylock_pages(struct mem_cgroup * memcg)1423 static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
1424 {
1425 	/* to match folio_memcg_rcu() */
1426 	rcu_read_lock();
1427 	return true;
1428 }
1429 
mem_cgroup_unlock_pages(void)1430 static inline void mem_cgroup_unlock_pages(void)
1431 {
1432 	rcu_read_unlock();
1433 }
1434 
mem_cgroup_handle_over_high(void)1435 static inline void mem_cgroup_handle_over_high(void)
1436 {
1437 }
1438 
mem_cgroup_enter_user_fault(void)1439 static inline void mem_cgroup_enter_user_fault(void)
1440 {
1441 }
1442 
mem_cgroup_exit_user_fault(void)1443 static inline void mem_cgroup_exit_user_fault(void)
1444 {
1445 }
1446 
task_in_memcg_oom(struct task_struct * p)1447 static inline bool task_in_memcg_oom(struct task_struct *p)
1448 {
1449 	return false;
1450 }
1451 
mem_cgroup_oom_synchronize(bool wait)1452 static inline bool mem_cgroup_oom_synchronize(bool wait)
1453 {
1454 	return false;
1455 }
1456 
mem_cgroup_get_oom_group(struct task_struct * victim,struct mem_cgroup * oom_domain)1457 static inline struct mem_cgroup *mem_cgroup_get_oom_group(
1458 	struct task_struct *victim, struct mem_cgroup *oom_domain)
1459 {
1460 	return NULL;
1461 }
1462 
mem_cgroup_print_oom_group(struct mem_cgroup * memcg)1463 static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1464 {
1465 }
1466 
__mod_memcg_state(struct mem_cgroup * memcg,int idx,int nr)1467 static inline void __mod_memcg_state(struct mem_cgroup *memcg,
1468 				     int idx,
1469 				     int nr)
1470 {
1471 }
1472 
mod_memcg_state(struct mem_cgroup * memcg,int idx,int nr)1473 static inline void mod_memcg_state(struct mem_cgroup *memcg,
1474 				   int idx,
1475 				   int nr)
1476 {
1477 }
1478 
mod_memcg_page_state(struct page * page,int idx,int val)1479 static inline void mod_memcg_page_state(struct page *page,
1480 					int idx, int val)
1481 {
1482 }
1483 
memcg_page_state(struct mem_cgroup * memcg,int idx)1484 static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
1485 {
1486 	return 0;
1487 }
1488 
lruvec_page_state(struct lruvec * lruvec,enum node_stat_item idx)1489 static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
1490 					      enum node_stat_item idx)
1491 {
1492 	return node_page_state(lruvec_pgdat(lruvec), idx);
1493 }
1494 
lruvec_page_state_local(struct lruvec * lruvec,enum node_stat_item idx)1495 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
1496 						    enum node_stat_item idx)
1497 {
1498 	return node_page_state(lruvec_pgdat(lruvec), idx);
1499 }
1500 
mem_cgroup_flush_stats(void)1501 static inline void mem_cgroup_flush_stats(void)
1502 {
1503 }
1504 
mem_cgroup_flush_stats_delayed(void)1505 static inline void mem_cgroup_flush_stats_delayed(void)
1506 {
1507 }
1508 
__mod_memcg_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)1509 static inline void __mod_memcg_lruvec_state(struct lruvec *lruvec,
1510 					    enum node_stat_item idx, int val)
1511 {
1512 }
1513 
__mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)1514 static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1515 					   int val)
1516 {
1517 	struct page *page = virt_to_head_page(p);
1518 
1519 	__mod_node_page_state(page_pgdat(page), idx, val);
1520 }
1521 
mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)1522 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1523 					 int val)
1524 {
1525 	struct page *page = virt_to_head_page(p);
1526 
1527 	mod_node_page_state(page_pgdat(page), idx, val);
1528 }
1529 
count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)1530 static inline void count_memcg_events(struct mem_cgroup *memcg,
1531 				      enum vm_event_item idx,
1532 				      unsigned long count)
1533 {
1534 }
1535 
__count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)1536 static inline void __count_memcg_events(struct mem_cgroup *memcg,
1537 					enum vm_event_item idx,
1538 					unsigned long count)
1539 {
1540 }
1541 
count_memcg_page_event(struct page * page,int idx)1542 static inline void count_memcg_page_event(struct page *page,
1543 					  int idx)
1544 {
1545 }
1546 
count_memcg_folio_events(struct folio * folio,enum vm_event_item idx,unsigned long nr)1547 static inline void count_memcg_folio_events(struct folio *folio,
1548 		enum vm_event_item idx, unsigned long nr)
1549 {
1550 }
1551 
1552 static inline
count_memcg_event_mm(struct mm_struct * mm,enum vm_event_item idx)1553 void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
1554 {
1555 }
1556 
split_page_memcg(struct page * head,unsigned int nr)1557 static inline void split_page_memcg(struct page *head, unsigned int nr)
1558 {
1559 }
1560 
1561 static inline
mem_cgroup_soft_limit_reclaim(pg_data_t * pgdat,int order,gfp_t gfp_mask,unsigned long * total_scanned)1562 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1563 					    gfp_t gfp_mask,
1564 					    unsigned long *total_scanned)
1565 {
1566 	return 0;
1567 }
1568 #endif /* CONFIG_MEMCG */
1569 
__inc_lruvec_kmem_state(void * p,enum node_stat_item idx)1570 static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx)
1571 {
1572 	__mod_lruvec_kmem_state(p, idx, 1);
1573 }
1574 
__dec_lruvec_kmem_state(void * p,enum node_stat_item idx)1575 static inline void __dec_lruvec_kmem_state(void *p, enum node_stat_item idx)
1576 {
1577 	__mod_lruvec_kmem_state(p, idx, -1);
1578 }
1579 
parent_lruvec(struct lruvec * lruvec)1580 static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
1581 {
1582 	struct mem_cgroup *memcg;
1583 
1584 	memcg = lruvec_memcg(lruvec);
1585 	if (!memcg)
1586 		return NULL;
1587 	memcg = parent_mem_cgroup(memcg);
1588 	if (!memcg)
1589 		return NULL;
1590 	return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
1591 }
1592 
unlock_page_lruvec(struct lruvec * lruvec)1593 static inline void unlock_page_lruvec(struct lruvec *lruvec)
1594 {
1595 	spin_unlock(&lruvec->lru_lock);
1596 }
1597 
unlock_page_lruvec_irq(struct lruvec * lruvec)1598 static inline void unlock_page_lruvec_irq(struct lruvec *lruvec)
1599 {
1600 	spin_unlock_irq(&lruvec->lru_lock);
1601 }
1602 
unlock_page_lruvec_irqrestore(struct lruvec * lruvec,unsigned long flags)1603 static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec,
1604 		unsigned long flags)
1605 {
1606 	spin_unlock_irqrestore(&lruvec->lru_lock, flags);
1607 }
1608 
1609 /* Test requires a stable page->memcg binding, see page_memcg() */
folio_matches_lruvec(struct folio * folio,struct lruvec * lruvec)1610 static inline bool folio_matches_lruvec(struct folio *folio,
1611 		struct lruvec *lruvec)
1612 {
1613 	return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&
1614 	       lruvec_memcg(lruvec) == folio_memcg(folio);
1615 }
1616 
1617 /* Don't lock again iff page's lruvec locked */
folio_lruvec_relock_irq(struct folio * folio,struct lruvec * locked_lruvec)1618 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
1619 		struct lruvec *locked_lruvec)
1620 {
1621 	if (locked_lruvec) {
1622 		if (folio_matches_lruvec(folio, locked_lruvec))
1623 			return locked_lruvec;
1624 
1625 		unlock_page_lruvec_irq(locked_lruvec);
1626 	}
1627 
1628 	return folio_lruvec_lock_irq(folio);
1629 }
1630 
1631 /* Don't lock again iff page's lruvec locked */
folio_lruvec_relock_irqsave(struct folio * folio,struct lruvec * locked_lruvec,unsigned long * flags)1632 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio,
1633 		struct lruvec *locked_lruvec, unsigned long *flags)
1634 {
1635 	if (locked_lruvec) {
1636 		if (folio_matches_lruvec(folio, locked_lruvec))
1637 			return locked_lruvec;
1638 
1639 		unlock_page_lruvec_irqrestore(locked_lruvec, *flags);
1640 	}
1641 
1642 	return folio_lruvec_lock_irqsave(folio, flags);
1643 }
1644 
1645 #ifdef CONFIG_CGROUP_WRITEBACK
1646 
1647 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
1648 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
1649 			 unsigned long *pheadroom, unsigned long *pdirty,
1650 			 unsigned long *pwriteback);
1651 
1652 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1653 					     struct bdi_writeback *wb);
1654 
mem_cgroup_track_foreign_dirty(struct folio * folio,struct bdi_writeback * wb)1655 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
1656 						  struct bdi_writeback *wb)
1657 {
1658 	if (mem_cgroup_disabled())
1659 		return;
1660 
1661 	if (unlikely(&folio_memcg(folio)->css != wb->memcg_css))
1662 		mem_cgroup_track_foreign_dirty_slowpath(folio, wb);
1663 }
1664 
1665 void mem_cgroup_flush_foreign(struct bdi_writeback *wb);
1666 
1667 #else	/* CONFIG_CGROUP_WRITEBACK */
1668 
mem_cgroup_wb_domain(struct bdi_writeback * wb)1669 static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
1670 {
1671 	return NULL;
1672 }
1673 
mem_cgroup_wb_stats(struct bdi_writeback * wb,unsigned long * pfilepages,unsigned long * pheadroom,unsigned long * pdirty,unsigned long * pwriteback)1674 static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
1675 				       unsigned long *pfilepages,
1676 				       unsigned long *pheadroom,
1677 				       unsigned long *pdirty,
1678 				       unsigned long *pwriteback)
1679 {
1680 }
1681 
mem_cgroup_track_foreign_dirty(struct folio * folio,struct bdi_writeback * wb)1682 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
1683 						  struct bdi_writeback *wb)
1684 {
1685 }
1686 
mem_cgroup_flush_foreign(struct bdi_writeback * wb)1687 static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
1688 {
1689 }
1690 
1691 #endif	/* CONFIG_CGROUP_WRITEBACK */
1692 
1693 struct sock;
1694 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
1695 			     gfp_t gfp_mask);
1696 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1697 #ifdef CONFIG_MEMCG
1698 extern struct static_key_false memcg_sockets_enabled_key;
1699 #define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
1700 void mem_cgroup_sk_alloc(struct sock *sk);
1701 void mem_cgroup_sk_free(struct sock *sk);
mem_cgroup_under_socket_pressure(struct mem_cgroup * memcg)1702 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1703 {
1704 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
1705 		return true;
1706 	do {
1707 		if (time_before(jiffies, READ_ONCE(memcg->socket_pressure)))
1708 			return true;
1709 	} while ((memcg = parent_mem_cgroup(memcg)));
1710 	return false;
1711 }
1712 
1713 int alloc_shrinker_info(struct mem_cgroup *memcg);
1714 void free_shrinker_info(struct mem_cgroup *memcg);
1715 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);
1716 void reparent_shrinker_deferred(struct mem_cgroup *memcg);
1717 #else
1718 #define mem_cgroup_sockets_enabled 0
mem_cgroup_sk_alloc(struct sock * sk)1719 static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
mem_cgroup_sk_free(struct sock * sk)1720 static inline void mem_cgroup_sk_free(struct sock *sk) { };
mem_cgroup_under_socket_pressure(struct mem_cgroup * memcg)1721 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1722 {
1723 	return false;
1724 }
1725 
set_shrinker_bit(struct mem_cgroup * memcg,int nid,int shrinker_id)1726 static inline void set_shrinker_bit(struct mem_cgroup *memcg,
1727 				    int nid, int shrinker_id)
1728 {
1729 }
1730 #endif
1731 
1732 #ifdef CONFIG_MEMCG_KMEM
1733 bool mem_cgroup_kmem_disabled(void);
1734 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order);
1735 void __memcg_kmem_uncharge_page(struct page *page, int order);
1736 
1737 struct obj_cgroup *get_obj_cgroup_from_current(void);
1738 struct obj_cgroup *get_obj_cgroup_from_page(struct page *page);
1739 
1740 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size);
1741 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size);
1742 
1743 extern struct static_key_false memcg_kmem_enabled_key;
1744 
memcg_kmem_enabled(void)1745 static inline bool memcg_kmem_enabled(void)
1746 {
1747 	return static_branch_likely(&memcg_kmem_enabled_key);
1748 }
1749 
memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)1750 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1751 					 int order)
1752 {
1753 	if (memcg_kmem_enabled())
1754 		return __memcg_kmem_charge_page(page, gfp, order);
1755 	return 0;
1756 }
1757 
memcg_kmem_uncharge_page(struct page * page,int order)1758 static inline void memcg_kmem_uncharge_page(struct page *page, int order)
1759 {
1760 	if (memcg_kmem_enabled())
1761 		__memcg_kmem_uncharge_page(page, order);
1762 }
1763 
1764 /*
1765  * A helper for accessing memcg's kmem_id, used for getting
1766  * corresponding LRU lists.
1767  */
memcg_kmem_id(struct mem_cgroup * memcg)1768 static inline int memcg_kmem_id(struct mem_cgroup *memcg)
1769 {
1770 	return memcg ? memcg->kmemcg_id : -1;
1771 }
1772 
1773 struct mem_cgroup *mem_cgroup_from_obj(void *p);
1774 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p);
1775 
count_objcg_event(struct obj_cgroup * objcg,enum vm_event_item idx)1776 static inline void count_objcg_event(struct obj_cgroup *objcg,
1777 				     enum vm_event_item idx)
1778 {
1779 	struct mem_cgroup *memcg;
1780 
1781 	if (!memcg_kmem_enabled())
1782 		return;
1783 
1784 	rcu_read_lock();
1785 	memcg = obj_cgroup_memcg(objcg);
1786 	count_memcg_events(memcg, idx, 1);
1787 	rcu_read_unlock();
1788 }
1789 
1790 #else
mem_cgroup_kmem_disabled(void)1791 static inline bool mem_cgroup_kmem_disabled(void)
1792 {
1793 	return true;
1794 }
1795 
memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)1796 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1797 					 int order)
1798 {
1799 	return 0;
1800 }
1801 
memcg_kmem_uncharge_page(struct page * page,int order)1802 static inline void memcg_kmem_uncharge_page(struct page *page, int order)
1803 {
1804 }
1805 
__memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)1806 static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1807 					   int order)
1808 {
1809 	return 0;
1810 }
1811 
__memcg_kmem_uncharge_page(struct page * page,int order)1812 static inline void __memcg_kmem_uncharge_page(struct page *page, int order)
1813 {
1814 }
1815 
get_obj_cgroup_from_page(struct page * page)1816 static inline struct obj_cgroup *get_obj_cgroup_from_page(struct page *page)
1817 {
1818 	return NULL;
1819 }
1820 
memcg_kmem_enabled(void)1821 static inline bool memcg_kmem_enabled(void)
1822 {
1823 	return false;
1824 }
1825 
memcg_kmem_id(struct mem_cgroup * memcg)1826 static inline int memcg_kmem_id(struct mem_cgroup *memcg)
1827 {
1828 	return -1;
1829 }
1830 
mem_cgroup_from_obj(void * p)1831 static inline struct mem_cgroup *mem_cgroup_from_obj(void *p)
1832 {
1833 	return NULL;
1834 }
1835 
mem_cgroup_from_slab_obj(void * p)1836 static inline struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)
1837 {
1838 	return NULL;
1839 }
1840 
count_objcg_event(struct obj_cgroup * objcg,enum vm_event_item idx)1841 static inline void count_objcg_event(struct obj_cgroup *objcg,
1842 				     enum vm_event_item idx)
1843 {
1844 }
1845 
1846 #endif /* CONFIG_MEMCG_KMEM */
1847 
1848 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
1849 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg);
1850 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size);
1851 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size);
1852 #else
obj_cgroup_may_zswap(struct obj_cgroup * objcg)1853 static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
1854 {
1855 	return true;
1856 }
obj_cgroup_charge_zswap(struct obj_cgroup * objcg,size_t size)1857 static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg,
1858 					   size_t size)
1859 {
1860 }
obj_cgroup_uncharge_zswap(struct obj_cgroup * objcg,size_t size)1861 static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg,
1862 					     size_t size)
1863 {
1864 }
1865 #endif
1866 
1867 #endif /* _LINUX_MEMCONTROL_H */
1868