1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* memcontrol.h - Memory Controller
3  *
4  * Copyright IBM Corporation, 2007
5  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6  *
7  * Copyright 2007 OpenVZ SWsoft Inc
8  * Author: Pavel Emelianov <xemul@openvz.org>
9  */
10 
11 #ifndef _LINUX_MEMCONTROL_H
12 #define _LINUX_MEMCONTROL_H
13 #include <linux/cgroup.h>
14 #include <linux/vm_event_item.h>
15 #include <linux/hardirq.h>
16 #include <linux/jump_label.h>
17 #include <linux/page_counter.h>
18 #include <linux/vmpressure.h>
19 #include <linux/eventfd.h>
20 #include <linux/mm.h>
21 #include <linux/vmstat.h>
22 #include <linux/writeback.h>
23 #include <linux/page-flags.h>
24 
25 struct mem_cgroup;
26 struct obj_cgroup;
27 struct page;
28 struct mm_struct;
29 struct kmem_cache;
30 
31 /* Cgroup-specific page state, on top of universal node page state */
32 enum memcg_stat_item {
33 	MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
34 	MEMCG_SOCK,
35 	MEMCG_PERCPU_B,
36 	MEMCG_VMALLOC,
37 	MEMCG_KMEM,
38 	MEMCG_ZSWAP_B,
39 	MEMCG_ZSWAPPED,
40 	MEMCG_NR_STAT,
41 };
42 
43 enum memcg_memory_event {
44 	MEMCG_LOW,
45 	MEMCG_HIGH,
46 	MEMCG_MAX,
47 	MEMCG_OOM,
48 	MEMCG_OOM_KILL,
49 	MEMCG_OOM_GROUP_KILL,
50 	MEMCG_SWAP_HIGH,
51 	MEMCG_SWAP_MAX,
52 	MEMCG_SWAP_FAIL,
53 	MEMCG_NR_MEMORY_EVENTS,
54 };
55 
56 struct mem_cgroup_reclaim_cookie {
57 	pg_data_t *pgdat;
58 	unsigned int generation;
59 };
60 
61 #ifdef CONFIG_MEMCG
62 
63 #define MEM_CGROUP_ID_SHIFT	16
64 #define MEM_CGROUP_ID_MAX	USHRT_MAX
65 
66 struct mem_cgroup_id {
67 	int id;
68 	refcount_t ref;
69 };
70 
71 /*
72  * Per memcg event counter is incremented at every pagein/pageout. With THP,
73  * it will be incremented by the number of pages. This counter is used
74  * to trigger some periodic events. This is straightforward and better
75  * than using jiffies etc. to handle periodic memcg event.
76  */
77 enum mem_cgroup_events_target {
78 	MEM_CGROUP_TARGET_THRESH,
79 	MEM_CGROUP_TARGET_SOFTLIMIT,
80 	MEM_CGROUP_NTARGETS,
81 };
82 
83 struct memcg_vmstats_percpu {
84 	/* Local (CPU and cgroup) page state & events */
85 	long			state[MEMCG_NR_STAT];
86 	unsigned long		events[NR_VM_EVENT_ITEMS];
87 
88 	/* Delta calculation for lockless upward propagation */
89 	long			state_prev[MEMCG_NR_STAT];
90 	unsigned long		events_prev[NR_VM_EVENT_ITEMS];
91 
92 	/* Cgroup1: threshold notifications & softlimit tree updates */
93 	unsigned long		nr_page_events;
94 	unsigned long		targets[MEM_CGROUP_NTARGETS];
95 };
96 
97 struct memcg_vmstats {
98 	/* Aggregated (CPU and subtree) page state & events */
99 	long			state[MEMCG_NR_STAT];
100 	unsigned long		events[NR_VM_EVENT_ITEMS];
101 
102 	/* Pending child counts during tree propagation */
103 	long			state_pending[MEMCG_NR_STAT];
104 	unsigned long		events_pending[NR_VM_EVENT_ITEMS];
105 };
106 
107 struct mem_cgroup_reclaim_iter {
108 	struct mem_cgroup *position;
109 	/* scan generation, increased every round-trip */
110 	unsigned int generation;
111 };
112 
113 /*
114  * Bitmap and deferred work of shrinker::id corresponding to memcg-aware
115  * shrinkers, which have elements charged to this memcg.
116  */
117 struct shrinker_info {
118 	struct rcu_head rcu;
119 	atomic_long_t *nr_deferred;
120 	unsigned long *map;
121 };
122 
123 struct lruvec_stats_percpu {
124 	/* Local (CPU and cgroup) state */
125 	long state[NR_VM_NODE_STAT_ITEMS];
126 
127 	/* Delta calculation for lockless upward propagation */
128 	long state_prev[NR_VM_NODE_STAT_ITEMS];
129 };
130 
131 struct lruvec_stats {
132 	/* Aggregated (CPU and subtree) state */
133 	long state[NR_VM_NODE_STAT_ITEMS];
134 
135 	/* Pending child counts during tree propagation */
136 	long state_pending[NR_VM_NODE_STAT_ITEMS];
137 };
138 
139 /*
140  * per-node information in memory controller.
141  */
142 struct mem_cgroup_per_node {
143 	struct lruvec		lruvec;
144 
145 	struct lruvec_stats_percpu __percpu	*lruvec_stats_percpu;
146 	struct lruvec_stats			lruvec_stats;
147 
148 	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
149 
150 	struct mem_cgroup_reclaim_iter	iter;
151 
152 	struct shrinker_info __rcu	*shrinker_info;
153 
154 	struct rb_node		tree_node;	/* RB tree node */
155 	unsigned long		usage_in_excess;/* Set to the value by which */
156 						/* the soft limit is exceeded*/
157 	bool			on_tree;
158 	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
159 						/* use container_of	   */
160 };
161 
162 struct mem_cgroup_threshold {
163 	struct eventfd_ctx *eventfd;
164 	unsigned long threshold;
165 };
166 
167 /* For threshold */
168 struct mem_cgroup_threshold_ary {
169 	/* An array index points to threshold just below or equal to usage. */
170 	int current_threshold;
171 	/* Size of entries[] */
172 	unsigned int size;
173 	/* Array of thresholds */
174 	struct mem_cgroup_threshold entries[];
175 };
176 
177 struct mem_cgroup_thresholds {
178 	/* Primary thresholds array */
179 	struct mem_cgroup_threshold_ary *primary;
180 	/*
181 	 * Spare threshold array.
182 	 * This is needed to make mem_cgroup_unregister_event() "never fail".
183 	 * It must be able to store at least primary->size - 1 entries.
184 	 */
185 	struct mem_cgroup_threshold_ary *spare;
186 };
187 
188 #if defined(CONFIG_SMP)
189 struct memcg_padding {
190 	char x[0];
191 } ____cacheline_internodealigned_in_smp;
192 #define MEMCG_PADDING(name)      struct memcg_padding name
193 #else
194 #define MEMCG_PADDING(name)
195 #endif
196 
197 /*
198  * Remember four most recent foreign writebacks with dirty pages in this
199  * cgroup.  Inode sharing is expected to be uncommon and, even if we miss
200  * one in a given round, we're likely to catch it later if it keeps
201  * foreign-dirtying, so a fairly low count should be enough.
202  *
203  * See mem_cgroup_track_foreign_dirty_slowpath() for details.
204  */
205 #define MEMCG_CGWB_FRN_CNT	4
206 
207 struct memcg_cgwb_frn {
208 	u64 bdi_id;			/* bdi->id of the foreign inode */
209 	int memcg_id;			/* memcg->css.id of foreign inode */
210 	u64 at;				/* jiffies_64 at the time of dirtying */
211 	struct wb_completion done;	/* tracks in-flight foreign writebacks */
212 };
213 
214 /*
215  * Bucket for arbitrarily byte-sized objects charged to a memory
216  * cgroup. The bucket can be reparented in one piece when the cgroup
217  * is destroyed, without having to round up the individual references
218  * of all live memory objects in the wild.
219  */
220 struct obj_cgroup {
221 	struct percpu_ref refcnt;
222 	struct mem_cgroup *memcg;
223 	atomic_t nr_charged_bytes;
224 	union {
225 		struct list_head list; /* protected by objcg_lock */
226 		struct rcu_head rcu;
227 	};
228 };
229 
230 /*
231  * The memory controller data structure. The memory controller controls both
232  * page cache and RSS per cgroup. We would eventually like to provide
233  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
234  * to help the administrator determine what knobs to tune.
235  */
236 struct mem_cgroup {
237 	struct cgroup_subsys_state css;
238 
239 	/* Private memcg ID. Used to ID objects that outlive the cgroup */
240 	struct mem_cgroup_id id;
241 
242 	/* Accounted resources */
243 	struct page_counter memory;		/* Both v1 & v2 */
244 
245 	union {
246 		struct page_counter swap;	/* v2 only */
247 		struct page_counter memsw;	/* v1 only */
248 	};
249 
250 	/* Legacy consumer-oriented counters */
251 	struct page_counter kmem;		/* v1 only */
252 	struct page_counter tcpmem;		/* v1 only */
253 
254 	/* Range enforcement for interrupt charges */
255 	struct work_struct high_work;
256 
257 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
258 	unsigned long zswap_max;
259 #endif
260 
261 	unsigned long soft_limit;
262 
263 	/* vmpressure notifications */
264 	struct vmpressure vmpressure;
265 
266 	/*
267 	 * Should the OOM killer kill all belonging tasks, had it kill one?
268 	 */
269 	bool oom_group;
270 
271 	/* protected by memcg_oom_lock */
272 	bool		oom_lock;
273 	int		under_oom;
274 
275 	int	swappiness;
276 	/* OOM-Killer disable */
277 	int		oom_kill_disable;
278 
279 	/* memory.events and memory.events.local */
280 	struct cgroup_file events_file;
281 	struct cgroup_file events_local_file;
282 
283 	/* handle for "memory.swap.events" */
284 	struct cgroup_file swap_events_file;
285 
286 	/* protect arrays of thresholds */
287 	struct mutex thresholds_lock;
288 
289 	/* thresholds for memory usage. RCU-protected */
290 	struct mem_cgroup_thresholds thresholds;
291 
292 	/* thresholds for mem+swap usage. RCU-protected */
293 	struct mem_cgroup_thresholds memsw_thresholds;
294 
295 	/* For oom notifier event fd */
296 	struct list_head oom_notify;
297 
298 	/*
299 	 * Should we move charges of a task when a task is moved into this
300 	 * mem_cgroup ? And what type of charges should we move ?
301 	 */
302 	unsigned long move_charge_at_immigrate;
303 	/* taken only while moving_account > 0 */
304 	spinlock_t		move_lock;
305 	unsigned long		move_lock_flags;
306 
307 	MEMCG_PADDING(_pad1_);
308 
309 	/* memory.stat */
310 	struct memcg_vmstats	vmstats;
311 
312 	/* memory.events */
313 	atomic_long_t		memory_events[MEMCG_NR_MEMORY_EVENTS];
314 	atomic_long_t		memory_events_local[MEMCG_NR_MEMORY_EVENTS];
315 
316 	unsigned long		socket_pressure;
317 
318 	/* Legacy tcp memory accounting */
319 	bool			tcpmem_active;
320 	int			tcpmem_pressure;
321 
322 #ifdef CONFIG_MEMCG_KMEM
323 	int kmemcg_id;
324 	struct obj_cgroup __rcu *objcg;
325 	/* list of inherited objcgs, protected by objcg_lock */
326 	struct list_head objcg_list;
327 #endif
328 
329 	MEMCG_PADDING(_pad2_);
330 
331 	/*
332 	 * set > 0 if pages under this cgroup are moving to other cgroup.
333 	 */
334 	atomic_t		moving_account;
335 	struct task_struct	*move_lock_task;
336 
337 	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
338 
339 #ifdef CONFIG_CGROUP_WRITEBACK
340 	struct list_head cgwb_list;
341 	struct wb_domain cgwb_domain;
342 	struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
343 #endif
344 
345 	/* List of events which userspace want to receive */
346 	struct list_head event_list;
347 	spinlock_t event_list_lock;
348 
349 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
350 	struct deferred_split deferred_split_queue;
351 #endif
352 
353 	struct mem_cgroup_per_node *nodeinfo[];
354 };
355 
356 /*
357  * size of first charge trial. "32" comes from vmscan.c's magic value.
358  * TODO: maybe necessary to use big numbers in big irons.
359  */
360 #define MEMCG_CHARGE_BATCH 32U
361 
362 extern struct mem_cgroup *root_mem_cgroup;
363 
364 enum page_memcg_data_flags {
365 	/* page->memcg_data is a pointer to an objcgs vector */
366 	MEMCG_DATA_OBJCGS = (1UL << 0),
367 	/* page has been accounted as a non-slab kernel page */
368 	MEMCG_DATA_KMEM = (1UL << 1),
369 	/* the next bit after the last actual flag */
370 	__NR_MEMCG_DATA_FLAGS  = (1UL << 2),
371 };
372 
373 #define MEMCG_DATA_FLAGS_MASK (__NR_MEMCG_DATA_FLAGS - 1)
374 
375 static inline bool folio_memcg_kmem(struct folio *folio);
376 
377 /*
378  * After the initialization objcg->memcg is always pointing at
379  * a valid memcg, but can be atomically swapped to the parent memcg.
380  *
381  * The caller must ensure that the returned memcg won't be released:
382  * e.g. acquire the rcu_read_lock or css_set_lock.
383  */
obj_cgroup_memcg(struct obj_cgroup * objcg)384 static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
385 {
386 	return READ_ONCE(objcg->memcg);
387 }
388 
389 /*
390  * __folio_memcg - Get the memory cgroup associated with a non-kmem folio
391  * @folio: Pointer to the folio.
392  *
393  * Returns a pointer to the memory cgroup associated with the folio,
394  * or NULL. This function assumes that the folio is known to have a
395  * proper memory cgroup pointer. It's not safe to call this function
396  * against some type of folios, e.g. slab folios or ex-slab folios or
397  * kmem folios.
398  */
__folio_memcg(struct folio * folio)399 static inline struct mem_cgroup *__folio_memcg(struct folio *folio)
400 {
401 	unsigned long memcg_data = folio->memcg_data;
402 
403 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
404 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);
405 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);
406 
407 	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
408 }
409 
410 /*
411  * __folio_objcg - get the object cgroup associated with a kmem folio.
412  * @folio: Pointer to the folio.
413  *
414  * Returns a pointer to the object cgroup associated with the folio,
415  * or NULL. This function assumes that the folio is known to have a
416  * proper object cgroup pointer. It's not safe to call this function
417  * against some type of folios, e.g. slab folios or ex-slab folios or
418  * LRU folios.
419  */
__folio_objcg(struct folio * folio)420 static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
421 {
422 	unsigned long memcg_data = folio->memcg_data;
423 
424 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
425 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);
426 	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);
427 
428 	return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
429 }
430 
431 /*
432  * folio_memcg - Get the memory cgroup associated with a folio.
433  * @folio: Pointer to the folio.
434  *
435  * Returns a pointer to the memory cgroup associated with the folio,
436  * or NULL. This function assumes that the folio is known to have a
437  * proper memory cgroup pointer. It's not safe to call this function
438  * against some type of folios, e.g. slab folios or ex-slab folios.
439  *
440  * For a non-kmem folio any of the following ensures folio and memcg binding
441  * stability:
442  *
443  * - the folio lock
444  * - LRU isolation
445  * - lock_page_memcg()
446  * - exclusive reference
447  *
448  * For a kmem folio a caller should hold an rcu read lock to protect memcg
449  * associated with a kmem folio from being released.
450  */
folio_memcg(struct folio * folio)451 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
452 {
453 	if (folio_memcg_kmem(folio))
454 		return obj_cgroup_memcg(__folio_objcg(folio));
455 	return __folio_memcg(folio);
456 }
457 
page_memcg(struct page * page)458 static inline struct mem_cgroup *page_memcg(struct page *page)
459 {
460 	return folio_memcg(page_folio(page));
461 }
462 
463 /**
464  * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio.
465  * @folio: Pointer to the folio.
466  *
467  * This function assumes that the folio is known to have a
468  * proper memory cgroup pointer. It's not safe to call this function
469  * against some type of folios, e.g. slab folios or ex-slab folios.
470  *
471  * Return: A pointer to the memory cgroup associated with the folio,
472  * or NULL.
473  */
folio_memcg_rcu(struct folio * folio)474 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
475 {
476 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);
477 
478 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
479 	WARN_ON_ONCE(!rcu_read_lock_held());
480 
481 	if (memcg_data & MEMCG_DATA_KMEM) {
482 		struct obj_cgroup *objcg;
483 
484 		objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
485 		return obj_cgroup_memcg(objcg);
486 	}
487 
488 	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
489 }
490 
491 /*
492  * page_memcg_check - get the memory cgroup associated with a page
493  * @page: a pointer to the page struct
494  *
495  * Returns a pointer to the memory cgroup associated with the page,
496  * or NULL. This function unlike page_memcg() can take any page
497  * as an argument. It has to be used in cases when it's not known if a page
498  * has an associated memory cgroup pointer or an object cgroups vector or
499  * an object cgroup.
500  *
501  * For a non-kmem page any of the following ensures page and memcg binding
502  * stability:
503  *
504  * - the page lock
505  * - LRU isolation
506  * - lock_page_memcg()
507  * - exclusive reference
508  *
509  * For a kmem page a caller should hold an rcu read lock to protect memcg
510  * associated with a kmem page from being released.
511  */
page_memcg_check(struct page * page)512 static inline struct mem_cgroup *page_memcg_check(struct page *page)
513 {
514 	/*
515 	 * Because page->memcg_data might be changed asynchronously
516 	 * for slab pages, READ_ONCE() should be used here.
517 	 */
518 	unsigned long memcg_data = READ_ONCE(page->memcg_data);
519 
520 	if (memcg_data & MEMCG_DATA_OBJCGS)
521 		return NULL;
522 
523 	if (memcg_data & MEMCG_DATA_KMEM) {
524 		struct obj_cgroup *objcg;
525 
526 		objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
527 		return obj_cgroup_memcg(objcg);
528 	}
529 
530 	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
531 }
532 
get_mem_cgroup_from_objcg(struct obj_cgroup * objcg)533 static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
534 {
535 	struct mem_cgroup *memcg;
536 
537 	rcu_read_lock();
538 retry:
539 	memcg = obj_cgroup_memcg(objcg);
540 	if (unlikely(!css_tryget(&memcg->css)))
541 		goto retry;
542 	rcu_read_unlock();
543 
544 	return memcg;
545 }
546 
547 #ifdef CONFIG_MEMCG_KMEM
548 /*
549  * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set.
550  * @folio: Pointer to the folio.
551  *
552  * Checks if the folio has MemcgKmem flag set. The caller must ensure
553  * that the folio has an associated memory cgroup. It's not safe to call
554  * this function against some types of folios, e.g. slab folios.
555  */
folio_memcg_kmem(struct folio * folio)556 static inline bool folio_memcg_kmem(struct folio *folio)
557 {
558 	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);
559 	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio);
560 	return folio->memcg_data & MEMCG_DATA_KMEM;
561 }
562 
563 
564 #else
folio_memcg_kmem(struct folio * folio)565 static inline bool folio_memcg_kmem(struct folio *folio)
566 {
567 	return false;
568 }
569 
570 #endif
571 
PageMemcgKmem(struct page * page)572 static inline bool PageMemcgKmem(struct page *page)
573 {
574 	return folio_memcg_kmem(page_folio(page));
575 }
576 
mem_cgroup_is_root(struct mem_cgroup * memcg)577 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
578 {
579 	return (memcg == root_mem_cgroup);
580 }
581 
mem_cgroup_disabled(void)582 static inline bool mem_cgroup_disabled(void)
583 {
584 	return !cgroup_subsys_enabled(memory_cgrp_subsys);
585 }
586 
mem_cgroup_protection(struct mem_cgroup * root,struct mem_cgroup * memcg,unsigned long * min,unsigned long * low)587 static inline void mem_cgroup_protection(struct mem_cgroup *root,
588 					 struct mem_cgroup *memcg,
589 					 unsigned long *min,
590 					 unsigned long *low)
591 {
592 	*min = *low = 0;
593 
594 	if (mem_cgroup_disabled())
595 		return;
596 
597 	/*
598 	 * There is no reclaim protection applied to a targeted reclaim.
599 	 * We are special casing this specific case here because
600 	 * mem_cgroup_protected calculation is not robust enough to keep
601 	 * the protection invariant for calculated effective values for
602 	 * parallel reclaimers with different reclaim target. This is
603 	 * especially a problem for tail memcgs (as they have pages on LRU)
604 	 * which would want to have effective values 0 for targeted reclaim
605 	 * but a different value for external reclaim.
606 	 *
607 	 * Example
608 	 * Let's have global and A's reclaim in parallel:
609 	 *  |
610 	 *  A (low=2G, usage = 3G, max = 3G, children_low_usage = 1.5G)
611 	 *  |\
612 	 *  | C (low = 1G, usage = 2.5G)
613 	 *  B (low = 1G, usage = 0.5G)
614 	 *
615 	 * For the global reclaim
616 	 * A.elow = A.low
617 	 * B.elow = min(B.usage, B.low) because children_low_usage <= A.elow
618 	 * C.elow = min(C.usage, C.low)
619 	 *
620 	 * With the effective values resetting we have A reclaim
621 	 * A.elow = 0
622 	 * B.elow = B.low
623 	 * C.elow = C.low
624 	 *
625 	 * If the global reclaim races with A's reclaim then
626 	 * B.elow = C.elow = 0 because children_low_usage > A.elow)
627 	 * is possible and reclaiming B would be violating the protection.
628 	 *
629 	 */
630 	if (root == memcg)
631 		return;
632 
633 	*min = READ_ONCE(memcg->memory.emin);
634 	*low = READ_ONCE(memcg->memory.elow);
635 }
636 
637 void mem_cgroup_calculate_protection(struct mem_cgroup *root,
638 				     struct mem_cgroup *memcg);
639 
mem_cgroup_supports_protection(struct mem_cgroup * memcg)640 static inline bool mem_cgroup_supports_protection(struct mem_cgroup *memcg)
641 {
642 	/*
643 	 * The root memcg doesn't account charges, and doesn't support
644 	 * protection.
645 	 */
646 	return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg);
647 
648 }
649 
mem_cgroup_below_low(struct mem_cgroup * memcg)650 static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg)
651 {
652 	if (!mem_cgroup_supports_protection(memcg))
653 		return false;
654 
655 	return READ_ONCE(memcg->memory.elow) >=
656 		page_counter_read(&memcg->memory);
657 }
658 
mem_cgroup_below_min(struct mem_cgroup * memcg)659 static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
660 {
661 	if (!mem_cgroup_supports_protection(memcg))
662 		return false;
663 
664 	return READ_ONCE(memcg->memory.emin) >=
665 		page_counter_read(&memcg->memory);
666 }
667 
668 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
669 
670 /**
671  * mem_cgroup_charge - Charge a newly allocated folio to a cgroup.
672  * @folio: Folio to charge.
673  * @mm: mm context of the allocating task.
674  * @gfp: Reclaim mode.
675  *
676  * Try to charge @folio to the memcg that @mm belongs to, reclaiming
677  * pages according to @gfp if necessary.  If @mm is NULL, try to
678  * charge to the active memcg.
679  *
680  * Do not use this for folios allocated for swapin.
681  *
682  * Return: 0 on success. Otherwise, an error code is returned.
683  */
mem_cgroup_charge(struct folio * folio,struct mm_struct * mm,gfp_t gfp)684 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,
685 				    gfp_t gfp)
686 {
687 	if (mem_cgroup_disabled())
688 		return 0;
689 	return __mem_cgroup_charge(folio, mm, gfp);
690 }
691 
692 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
693 				  gfp_t gfp, swp_entry_t entry);
694 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
695 
696 void __mem_cgroup_uncharge(struct folio *folio);
697 
698 /**
699  * mem_cgroup_uncharge - Uncharge a folio.
700  * @folio: Folio to uncharge.
701  *
702  * Uncharge a folio previously charged with mem_cgroup_charge().
703  */
mem_cgroup_uncharge(struct folio * folio)704 static inline void mem_cgroup_uncharge(struct folio *folio)
705 {
706 	if (mem_cgroup_disabled())
707 		return;
708 	__mem_cgroup_uncharge(folio);
709 }
710 
711 void __mem_cgroup_uncharge_list(struct list_head *page_list);
mem_cgroup_uncharge_list(struct list_head * page_list)712 static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
713 {
714 	if (mem_cgroup_disabled())
715 		return;
716 	__mem_cgroup_uncharge_list(page_list);
717 }
718 
719 void mem_cgroup_migrate(struct folio *old, struct folio *new);
720 
721 /**
722  * mem_cgroup_lruvec - get the lru list vector for a memcg & node
723  * @memcg: memcg of the wanted lruvec
724  * @pgdat: pglist_data
725  *
726  * Returns the lru list vector holding pages for a given @memcg &
727  * @pgdat combination. This can be the node lruvec, if the memory
728  * controller is disabled.
729  */
mem_cgroup_lruvec(struct mem_cgroup * memcg,struct pglist_data * pgdat)730 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
731 					       struct pglist_data *pgdat)
732 {
733 	struct mem_cgroup_per_node *mz;
734 	struct lruvec *lruvec;
735 
736 	if (mem_cgroup_disabled()) {
737 		lruvec = &pgdat->__lruvec;
738 		goto out;
739 	}
740 
741 	if (!memcg)
742 		memcg = root_mem_cgroup;
743 
744 	mz = memcg->nodeinfo[pgdat->node_id];
745 	lruvec = &mz->lruvec;
746 out:
747 	/*
748 	 * Since a node can be onlined after the mem_cgroup was created,
749 	 * we have to be prepared to initialize lruvec->pgdat here;
750 	 * and if offlined then reonlined, we need to reinitialize it.
751 	 */
752 	if (unlikely(lruvec->pgdat != pgdat))
753 		lruvec->pgdat = pgdat;
754 	return lruvec;
755 }
756 
757 /**
758  * folio_lruvec - return lruvec for isolating/putting an LRU folio
759  * @folio: Pointer to the folio.
760  *
761  * This function relies on folio->mem_cgroup being stable.
762  */
folio_lruvec(struct folio * folio)763 static inline struct lruvec *folio_lruvec(struct folio *folio)
764 {
765 	struct mem_cgroup *memcg = folio_memcg(folio);
766 
767 	VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);
768 	return mem_cgroup_lruvec(memcg, folio_pgdat(folio));
769 }
770 
771 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
772 
773 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
774 
775 struct lruvec *folio_lruvec_lock(struct folio *folio);
776 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
777 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
778 						unsigned long *flags);
779 
780 #ifdef CONFIG_DEBUG_VM
781 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
782 #else
783 static inline
lruvec_memcg_debug(struct lruvec * lruvec,struct folio * folio)784 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
785 {
786 }
787 #endif
788 
789 static inline
mem_cgroup_from_css(struct cgroup_subsys_state * css)790 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
791 	return css ? container_of(css, struct mem_cgroup, css) : NULL;
792 }
793 
obj_cgroup_tryget(struct obj_cgroup * objcg)794 static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg)
795 {
796 	return percpu_ref_tryget(&objcg->refcnt);
797 }
798 
obj_cgroup_get(struct obj_cgroup * objcg)799 static inline void obj_cgroup_get(struct obj_cgroup *objcg)
800 {
801 	percpu_ref_get(&objcg->refcnt);
802 }
803 
obj_cgroup_get_many(struct obj_cgroup * objcg,unsigned long nr)804 static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
805 				       unsigned long nr)
806 {
807 	percpu_ref_get_many(&objcg->refcnt, nr);
808 }
809 
obj_cgroup_put(struct obj_cgroup * objcg)810 static inline void obj_cgroup_put(struct obj_cgroup *objcg)
811 {
812 	percpu_ref_put(&objcg->refcnt);
813 }
814 
mem_cgroup_put(struct mem_cgroup * memcg)815 static inline void mem_cgroup_put(struct mem_cgroup *memcg)
816 {
817 	if (memcg)
818 		css_put(&memcg->css);
819 }
820 
821 #define mem_cgroup_from_counter(counter, member)	\
822 	container_of(counter, struct mem_cgroup, member)
823 
824 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
825 				   struct mem_cgroup *,
826 				   struct mem_cgroup_reclaim_cookie *);
827 void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
828 int mem_cgroup_scan_tasks(struct mem_cgroup *,
829 			  int (*)(struct task_struct *, void *), void *);
830 
mem_cgroup_id(struct mem_cgroup * memcg)831 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
832 {
833 	if (mem_cgroup_disabled())
834 		return 0;
835 
836 	return memcg->id.id;
837 }
838 struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
839 
mem_cgroup_from_seq(struct seq_file * m)840 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
841 {
842 	return mem_cgroup_from_css(seq_css(m));
843 }
844 
lruvec_memcg(struct lruvec * lruvec)845 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
846 {
847 	struct mem_cgroup_per_node *mz;
848 
849 	if (mem_cgroup_disabled())
850 		return NULL;
851 
852 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
853 	return mz->memcg;
854 }
855 
856 /**
857  * parent_mem_cgroup - find the accounting parent of a memcg
858  * @memcg: memcg whose parent to find
859  *
860  * Returns the parent memcg, or NULL if this is the root or the memory
861  * controller is in legacy no-hierarchy mode.
862  */
parent_mem_cgroup(struct mem_cgroup * memcg)863 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
864 {
865 	return mem_cgroup_from_css(memcg->css.parent);
866 }
867 
mem_cgroup_is_descendant(struct mem_cgroup * memcg,struct mem_cgroup * root)868 static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
869 			      struct mem_cgroup *root)
870 {
871 	if (root == memcg)
872 		return true;
873 	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
874 }
875 
mm_match_cgroup(struct mm_struct * mm,struct mem_cgroup * memcg)876 static inline bool mm_match_cgroup(struct mm_struct *mm,
877 				   struct mem_cgroup *memcg)
878 {
879 	struct mem_cgroup *task_memcg;
880 	bool match = false;
881 
882 	rcu_read_lock();
883 	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
884 	if (task_memcg)
885 		match = mem_cgroup_is_descendant(task_memcg, memcg);
886 	rcu_read_unlock();
887 	return match;
888 }
889 
890 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
891 ino_t page_cgroup_ino(struct page *page);
892 
mem_cgroup_online(struct mem_cgroup * memcg)893 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
894 {
895 	if (mem_cgroup_disabled())
896 		return true;
897 	return !!(memcg->css.flags & CSS_ONLINE);
898 }
899 
900 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
901 		int zid, int nr_pages);
902 
903 static inline
mem_cgroup_get_zone_lru_size(struct lruvec * lruvec,enum lru_list lru,int zone_idx)904 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
905 		enum lru_list lru, int zone_idx)
906 {
907 	struct mem_cgroup_per_node *mz;
908 
909 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
910 	return READ_ONCE(mz->lru_zone_size[zone_idx][lru]);
911 }
912 
913 void mem_cgroup_handle_over_high(void);
914 
915 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
916 
917 unsigned long mem_cgroup_size(struct mem_cgroup *memcg);
918 
919 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
920 				struct task_struct *p);
921 
922 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
923 
mem_cgroup_enter_user_fault(void)924 static inline void mem_cgroup_enter_user_fault(void)
925 {
926 	WARN_ON(current->in_user_fault);
927 	current->in_user_fault = 1;
928 }
929 
mem_cgroup_exit_user_fault(void)930 static inline void mem_cgroup_exit_user_fault(void)
931 {
932 	WARN_ON(!current->in_user_fault);
933 	current->in_user_fault = 0;
934 }
935 
task_in_memcg_oom(struct task_struct * p)936 static inline bool task_in_memcg_oom(struct task_struct *p)
937 {
938 	return p->memcg_in_oom;
939 }
940 
941 bool mem_cgroup_oom_synchronize(bool wait);
942 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
943 					    struct mem_cgroup *oom_domain);
944 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
945 
946 void folio_memcg_lock(struct folio *folio);
947 void folio_memcg_unlock(struct folio *folio);
948 void lock_page_memcg(struct page *page);
949 void unlock_page_memcg(struct page *page);
950 
951 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
952 
953 /* idx can be of type enum memcg_stat_item or node_stat_item */
mod_memcg_state(struct mem_cgroup * memcg,int idx,int val)954 static inline void mod_memcg_state(struct mem_cgroup *memcg,
955 				   int idx, int val)
956 {
957 	unsigned long flags;
958 
959 	local_irq_save(flags);
960 	__mod_memcg_state(memcg, idx, val);
961 	local_irq_restore(flags);
962 }
963 
mod_memcg_page_state(struct page * page,int idx,int val)964 static inline void mod_memcg_page_state(struct page *page,
965 					int idx, int val)
966 {
967 	struct mem_cgroup *memcg;
968 
969 	if (mem_cgroup_disabled())
970 		return;
971 
972 	rcu_read_lock();
973 	memcg = page_memcg(page);
974 	if (memcg)
975 		mod_memcg_state(memcg, idx, val);
976 	rcu_read_unlock();
977 }
978 
memcg_page_state(struct mem_cgroup * memcg,int idx)979 static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
980 {
981 	long x = READ_ONCE(memcg->vmstats.state[idx]);
982 #ifdef CONFIG_SMP
983 	if (x < 0)
984 		x = 0;
985 #endif
986 	return x;
987 }
988 
lruvec_page_state(struct lruvec * lruvec,enum node_stat_item idx)989 static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
990 					      enum node_stat_item idx)
991 {
992 	struct mem_cgroup_per_node *pn;
993 	long x;
994 
995 	if (mem_cgroup_disabled())
996 		return node_page_state(lruvec_pgdat(lruvec), idx);
997 
998 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
999 	x = READ_ONCE(pn->lruvec_stats.state[idx]);
1000 #ifdef CONFIG_SMP
1001 	if (x < 0)
1002 		x = 0;
1003 #endif
1004 	return x;
1005 }
1006 
lruvec_page_state_local(struct lruvec * lruvec,enum node_stat_item idx)1007 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
1008 						    enum node_stat_item idx)
1009 {
1010 	struct mem_cgroup_per_node *pn;
1011 	long x = 0;
1012 	int cpu;
1013 
1014 	if (mem_cgroup_disabled())
1015 		return node_page_state(lruvec_pgdat(lruvec), idx);
1016 
1017 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1018 	for_each_possible_cpu(cpu)
1019 		x += per_cpu(pn->lruvec_stats_percpu->state[idx], cpu);
1020 #ifdef CONFIG_SMP
1021 	if (x < 0)
1022 		x = 0;
1023 #endif
1024 	return x;
1025 }
1026 
1027 void mem_cgroup_flush_stats(void);
1028 void mem_cgroup_flush_stats_delayed(void);
1029 
1030 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
1031 			      int val);
1032 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val);
1033 
mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)1034 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1035 					 int val)
1036 {
1037 	unsigned long flags;
1038 
1039 	local_irq_save(flags);
1040 	__mod_lruvec_kmem_state(p, idx, val);
1041 	local_irq_restore(flags);
1042 }
1043 
mod_memcg_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)1044 static inline void mod_memcg_lruvec_state(struct lruvec *lruvec,
1045 					  enum node_stat_item idx, int val)
1046 {
1047 	unsigned long flags;
1048 
1049 	local_irq_save(flags);
1050 	__mod_memcg_lruvec_state(lruvec, idx, val);
1051 	local_irq_restore(flags);
1052 }
1053 
1054 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
1055 			  unsigned long count);
1056 
count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)1057 static inline void count_memcg_events(struct mem_cgroup *memcg,
1058 				      enum vm_event_item idx,
1059 				      unsigned long count)
1060 {
1061 	unsigned long flags;
1062 
1063 	local_irq_save(flags);
1064 	__count_memcg_events(memcg, idx, count);
1065 	local_irq_restore(flags);
1066 }
1067 
count_memcg_page_event(struct page * page,enum vm_event_item idx)1068 static inline void count_memcg_page_event(struct page *page,
1069 					  enum vm_event_item idx)
1070 {
1071 	struct mem_cgroup *memcg = page_memcg(page);
1072 
1073 	if (memcg)
1074 		count_memcg_events(memcg, idx, 1);
1075 }
1076 
count_memcg_folio_events(struct folio * folio,enum vm_event_item idx,unsigned long nr)1077 static inline void count_memcg_folio_events(struct folio *folio,
1078 		enum vm_event_item idx, unsigned long nr)
1079 {
1080 	struct mem_cgroup *memcg = folio_memcg(folio);
1081 
1082 	if (memcg)
1083 		count_memcg_events(memcg, idx, nr);
1084 }
1085 
count_memcg_event_mm(struct mm_struct * mm,enum vm_event_item idx)1086 static inline void count_memcg_event_mm(struct mm_struct *mm,
1087 					enum vm_event_item idx)
1088 {
1089 	struct mem_cgroup *memcg;
1090 
1091 	if (mem_cgroup_disabled())
1092 		return;
1093 
1094 	rcu_read_lock();
1095 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1096 	if (likely(memcg))
1097 		count_memcg_events(memcg, idx, 1);
1098 	rcu_read_unlock();
1099 }
1100 
memcg_memory_event(struct mem_cgroup * memcg,enum memcg_memory_event event)1101 static inline void memcg_memory_event(struct mem_cgroup *memcg,
1102 				      enum memcg_memory_event event)
1103 {
1104 	bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX ||
1105 			  event == MEMCG_SWAP_FAIL;
1106 
1107 	atomic_long_inc(&memcg->memory_events_local[event]);
1108 	if (!swap_event)
1109 		cgroup_file_notify(&memcg->events_local_file);
1110 
1111 	do {
1112 		atomic_long_inc(&memcg->memory_events[event]);
1113 		if (swap_event)
1114 			cgroup_file_notify(&memcg->swap_events_file);
1115 		else
1116 			cgroup_file_notify(&memcg->events_file);
1117 
1118 		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1119 			break;
1120 		if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS)
1121 			break;
1122 	} while ((memcg = parent_mem_cgroup(memcg)) &&
1123 		 !mem_cgroup_is_root(memcg));
1124 }
1125 
memcg_memory_event_mm(struct mm_struct * mm,enum memcg_memory_event event)1126 static inline void memcg_memory_event_mm(struct mm_struct *mm,
1127 					 enum memcg_memory_event event)
1128 {
1129 	struct mem_cgroup *memcg;
1130 
1131 	if (mem_cgroup_disabled())
1132 		return;
1133 
1134 	rcu_read_lock();
1135 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1136 	if (likely(memcg))
1137 		memcg_memory_event(memcg, event);
1138 	rcu_read_unlock();
1139 }
1140 
1141 void split_page_memcg(struct page *head, unsigned int nr);
1142 
1143 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1144 						gfp_t gfp_mask,
1145 						unsigned long *total_scanned);
1146 
1147 #else /* CONFIG_MEMCG */
1148 
1149 #define MEM_CGROUP_ID_SHIFT	0
1150 #define MEM_CGROUP_ID_MAX	0
1151 
folio_memcg(struct folio * folio)1152 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
1153 {
1154 	return NULL;
1155 }
1156 
page_memcg(struct page * page)1157 static inline struct mem_cgroup *page_memcg(struct page *page)
1158 {
1159 	return NULL;
1160 }
1161 
folio_memcg_rcu(struct folio * folio)1162 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
1163 {
1164 	WARN_ON_ONCE(!rcu_read_lock_held());
1165 	return NULL;
1166 }
1167 
page_memcg_check(struct page * page)1168 static inline struct mem_cgroup *page_memcg_check(struct page *page)
1169 {
1170 	return NULL;
1171 }
1172 
folio_memcg_kmem(struct folio * folio)1173 static inline bool folio_memcg_kmem(struct folio *folio)
1174 {
1175 	return false;
1176 }
1177 
PageMemcgKmem(struct page * page)1178 static inline bool PageMemcgKmem(struct page *page)
1179 {
1180 	return false;
1181 }
1182 
mem_cgroup_is_root(struct mem_cgroup * memcg)1183 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
1184 {
1185 	return true;
1186 }
1187 
mem_cgroup_disabled(void)1188 static inline bool mem_cgroup_disabled(void)
1189 {
1190 	return true;
1191 }
1192 
memcg_memory_event(struct mem_cgroup * memcg,enum memcg_memory_event event)1193 static inline void memcg_memory_event(struct mem_cgroup *memcg,
1194 				      enum memcg_memory_event event)
1195 {
1196 }
1197 
memcg_memory_event_mm(struct mm_struct * mm,enum memcg_memory_event event)1198 static inline void memcg_memory_event_mm(struct mm_struct *mm,
1199 					 enum memcg_memory_event event)
1200 {
1201 }
1202 
mem_cgroup_protection(struct mem_cgroup * root,struct mem_cgroup * memcg,unsigned long * min,unsigned long * low)1203 static inline void mem_cgroup_protection(struct mem_cgroup *root,
1204 					 struct mem_cgroup *memcg,
1205 					 unsigned long *min,
1206 					 unsigned long *low)
1207 {
1208 	*min = *low = 0;
1209 }
1210 
mem_cgroup_calculate_protection(struct mem_cgroup * root,struct mem_cgroup * memcg)1211 static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root,
1212 						   struct mem_cgroup *memcg)
1213 {
1214 }
1215 
mem_cgroup_below_low(struct mem_cgroup * memcg)1216 static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg)
1217 {
1218 	return false;
1219 }
1220 
mem_cgroup_below_min(struct mem_cgroup * memcg)1221 static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
1222 {
1223 	return false;
1224 }
1225 
mem_cgroup_charge(struct folio * folio,struct mm_struct * mm,gfp_t gfp)1226 static inline int mem_cgroup_charge(struct folio *folio,
1227 		struct mm_struct *mm, gfp_t gfp)
1228 {
1229 	return 0;
1230 }
1231 
mem_cgroup_swapin_charge_page(struct page * page,struct mm_struct * mm,gfp_t gfp,swp_entry_t entry)1232 static inline int mem_cgroup_swapin_charge_page(struct page *page,
1233 			struct mm_struct *mm, gfp_t gfp, swp_entry_t entry)
1234 {
1235 	return 0;
1236 }
1237 
mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)1238 static inline void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
1239 {
1240 }
1241 
mem_cgroup_uncharge(struct folio * folio)1242 static inline void mem_cgroup_uncharge(struct folio *folio)
1243 {
1244 }
1245 
mem_cgroup_uncharge_list(struct list_head * page_list)1246 static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
1247 {
1248 }
1249 
mem_cgroup_migrate(struct folio * old,struct folio * new)1250 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)
1251 {
1252 }
1253 
mem_cgroup_lruvec(struct mem_cgroup * memcg,struct pglist_data * pgdat)1254 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
1255 					       struct pglist_data *pgdat)
1256 {
1257 	return &pgdat->__lruvec;
1258 }
1259 
folio_lruvec(struct folio * folio)1260 static inline struct lruvec *folio_lruvec(struct folio *folio)
1261 {
1262 	struct pglist_data *pgdat = folio_pgdat(folio);
1263 	return &pgdat->__lruvec;
1264 }
1265 
1266 static inline
lruvec_memcg_debug(struct lruvec * lruvec,struct folio * folio)1267 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
1268 {
1269 }
1270 
parent_mem_cgroup(struct mem_cgroup * memcg)1271 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
1272 {
1273 	return NULL;
1274 }
1275 
mm_match_cgroup(struct mm_struct * mm,struct mem_cgroup * memcg)1276 static inline bool mm_match_cgroup(struct mm_struct *mm,
1277 		struct mem_cgroup *memcg)
1278 {
1279 	return true;
1280 }
1281 
get_mem_cgroup_from_mm(struct mm_struct * mm)1282 static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1283 {
1284 	return NULL;
1285 }
1286 
1287 static inline
mem_cgroup_from_css(struct cgroup_subsys_state * css)1288 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
1289 {
1290 	return NULL;
1291 }
1292 
obj_cgroup_put(struct obj_cgroup * objcg)1293 static inline void obj_cgroup_put(struct obj_cgroup *objcg)
1294 {
1295 }
1296 
mem_cgroup_put(struct mem_cgroup * memcg)1297 static inline void mem_cgroup_put(struct mem_cgroup *memcg)
1298 {
1299 }
1300 
folio_lruvec_lock(struct folio * folio)1301 static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
1302 {
1303 	struct pglist_data *pgdat = folio_pgdat(folio);
1304 
1305 	spin_lock(&pgdat->__lruvec.lru_lock);
1306 	return &pgdat->__lruvec;
1307 }
1308 
folio_lruvec_lock_irq(struct folio * folio)1309 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
1310 {
1311 	struct pglist_data *pgdat = folio_pgdat(folio);
1312 
1313 	spin_lock_irq(&pgdat->__lruvec.lru_lock);
1314 	return &pgdat->__lruvec;
1315 }
1316 
folio_lruvec_lock_irqsave(struct folio * folio,unsigned long * flagsp)1317 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
1318 		unsigned long *flagsp)
1319 {
1320 	struct pglist_data *pgdat = folio_pgdat(folio);
1321 
1322 	spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
1323 	return &pgdat->__lruvec;
1324 }
1325 
1326 static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup * root,struct mem_cgroup * prev,struct mem_cgroup_reclaim_cookie * reclaim)1327 mem_cgroup_iter(struct mem_cgroup *root,
1328 		struct mem_cgroup *prev,
1329 		struct mem_cgroup_reclaim_cookie *reclaim)
1330 {
1331 	return NULL;
1332 }
1333 
mem_cgroup_iter_break(struct mem_cgroup * root,struct mem_cgroup * prev)1334 static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
1335 					 struct mem_cgroup *prev)
1336 {
1337 }
1338 
mem_cgroup_scan_tasks(struct mem_cgroup * memcg,int (* fn)(struct task_struct *,void *),void * arg)1339 static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1340 		int (*fn)(struct task_struct *, void *), void *arg)
1341 {
1342 	return 0;
1343 }
1344 
mem_cgroup_id(struct mem_cgroup * memcg)1345 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
1346 {
1347 	return 0;
1348 }
1349 
mem_cgroup_from_id(unsigned short id)1350 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
1351 {
1352 	WARN_ON_ONCE(id);
1353 	/* XXX: This should always return root_mem_cgroup */
1354 	return NULL;
1355 }
1356 
mem_cgroup_from_seq(struct seq_file * m)1357 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
1358 {
1359 	return NULL;
1360 }
1361 
lruvec_memcg(struct lruvec * lruvec)1362 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
1363 {
1364 	return NULL;
1365 }
1366 
mem_cgroup_online(struct mem_cgroup * memcg)1367 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
1368 {
1369 	return true;
1370 }
1371 
1372 static inline
mem_cgroup_get_zone_lru_size(struct lruvec * lruvec,enum lru_list lru,int zone_idx)1373 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
1374 		enum lru_list lru, int zone_idx)
1375 {
1376 	return 0;
1377 }
1378 
mem_cgroup_get_max(struct mem_cgroup * memcg)1379 static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1380 {
1381 	return 0;
1382 }
1383 
mem_cgroup_size(struct mem_cgroup * memcg)1384 static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1385 {
1386 	return 0;
1387 }
1388 
1389 static inline void
mem_cgroup_print_oom_context(struct mem_cgroup * memcg,struct task_struct * p)1390 mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1391 {
1392 }
1393 
1394 static inline void
mem_cgroup_print_oom_meminfo(struct mem_cgroup * memcg)1395 mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1396 {
1397 }
1398 
lock_page_memcg(struct page * page)1399 static inline void lock_page_memcg(struct page *page)
1400 {
1401 }
1402 
unlock_page_memcg(struct page * page)1403 static inline void unlock_page_memcg(struct page *page)
1404 {
1405 }
1406 
folio_memcg_lock(struct folio * folio)1407 static inline void folio_memcg_lock(struct folio *folio)
1408 {
1409 }
1410 
folio_memcg_unlock(struct folio * folio)1411 static inline void folio_memcg_unlock(struct folio *folio)
1412 {
1413 }
1414 
mem_cgroup_handle_over_high(void)1415 static inline void mem_cgroup_handle_over_high(void)
1416 {
1417 }
1418 
mem_cgroup_enter_user_fault(void)1419 static inline void mem_cgroup_enter_user_fault(void)
1420 {
1421 }
1422 
mem_cgroup_exit_user_fault(void)1423 static inline void mem_cgroup_exit_user_fault(void)
1424 {
1425 }
1426 
task_in_memcg_oom(struct task_struct * p)1427 static inline bool task_in_memcg_oom(struct task_struct *p)
1428 {
1429 	return false;
1430 }
1431 
mem_cgroup_oom_synchronize(bool wait)1432 static inline bool mem_cgroup_oom_synchronize(bool wait)
1433 {
1434 	return false;
1435 }
1436 
mem_cgroup_get_oom_group(struct task_struct * victim,struct mem_cgroup * oom_domain)1437 static inline struct mem_cgroup *mem_cgroup_get_oom_group(
1438 	struct task_struct *victim, struct mem_cgroup *oom_domain)
1439 {
1440 	return NULL;
1441 }
1442 
mem_cgroup_print_oom_group(struct mem_cgroup * memcg)1443 static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1444 {
1445 }
1446 
__mod_memcg_state(struct mem_cgroup * memcg,int idx,int nr)1447 static inline void __mod_memcg_state(struct mem_cgroup *memcg,
1448 				     int idx,
1449 				     int nr)
1450 {
1451 }
1452 
mod_memcg_state(struct mem_cgroup * memcg,int idx,int nr)1453 static inline void mod_memcg_state(struct mem_cgroup *memcg,
1454 				   int idx,
1455 				   int nr)
1456 {
1457 }
1458 
mod_memcg_page_state(struct page * page,int idx,int val)1459 static inline void mod_memcg_page_state(struct page *page,
1460 					int idx, int val)
1461 {
1462 }
1463 
memcg_page_state(struct mem_cgroup * memcg,int idx)1464 static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
1465 {
1466 	return 0;
1467 }
1468 
lruvec_page_state(struct lruvec * lruvec,enum node_stat_item idx)1469 static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
1470 					      enum node_stat_item idx)
1471 {
1472 	return node_page_state(lruvec_pgdat(lruvec), idx);
1473 }
1474 
lruvec_page_state_local(struct lruvec * lruvec,enum node_stat_item idx)1475 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
1476 						    enum node_stat_item idx)
1477 {
1478 	return node_page_state(lruvec_pgdat(lruvec), idx);
1479 }
1480 
mem_cgroup_flush_stats(void)1481 static inline void mem_cgroup_flush_stats(void)
1482 {
1483 }
1484 
mem_cgroup_flush_stats_delayed(void)1485 static inline void mem_cgroup_flush_stats_delayed(void)
1486 {
1487 }
1488 
__mod_memcg_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)1489 static inline void __mod_memcg_lruvec_state(struct lruvec *lruvec,
1490 					    enum node_stat_item idx, int val)
1491 {
1492 }
1493 
__mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)1494 static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1495 					   int val)
1496 {
1497 	struct page *page = virt_to_head_page(p);
1498 
1499 	__mod_node_page_state(page_pgdat(page), idx, val);
1500 }
1501 
mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)1502 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1503 					 int val)
1504 {
1505 	struct page *page = virt_to_head_page(p);
1506 
1507 	mod_node_page_state(page_pgdat(page), idx, val);
1508 }
1509 
count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)1510 static inline void count_memcg_events(struct mem_cgroup *memcg,
1511 				      enum vm_event_item idx,
1512 				      unsigned long count)
1513 {
1514 }
1515 
__count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)1516 static inline void __count_memcg_events(struct mem_cgroup *memcg,
1517 					enum vm_event_item idx,
1518 					unsigned long count)
1519 {
1520 }
1521 
count_memcg_page_event(struct page * page,int idx)1522 static inline void count_memcg_page_event(struct page *page,
1523 					  int idx)
1524 {
1525 }
1526 
count_memcg_folio_events(struct folio * folio,enum vm_event_item idx,unsigned long nr)1527 static inline void count_memcg_folio_events(struct folio *folio,
1528 		enum vm_event_item idx, unsigned long nr)
1529 {
1530 }
1531 
1532 static inline
count_memcg_event_mm(struct mm_struct * mm,enum vm_event_item idx)1533 void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
1534 {
1535 }
1536 
split_page_memcg(struct page * head,unsigned int nr)1537 static inline void split_page_memcg(struct page *head, unsigned int nr)
1538 {
1539 }
1540 
1541 static inline
mem_cgroup_soft_limit_reclaim(pg_data_t * pgdat,int order,gfp_t gfp_mask,unsigned long * total_scanned)1542 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1543 					    gfp_t gfp_mask,
1544 					    unsigned long *total_scanned)
1545 {
1546 	return 0;
1547 }
1548 #endif /* CONFIG_MEMCG */
1549 
__inc_lruvec_kmem_state(void * p,enum node_stat_item idx)1550 static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx)
1551 {
1552 	__mod_lruvec_kmem_state(p, idx, 1);
1553 }
1554 
__dec_lruvec_kmem_state(void * p,enum node_stat_item idx)1555 static inline void __dec_lruvec_kmem_state(void *p, enum node_stat_item idx)
1556 {
1557 	__mod_lruvec_kmem_state(p, idx, -1);
1558 }
1559 
parent_lruvec(struct lruvec * lruvec)1560 static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
1561 {
1562 	struct mem_cgroup *memcg;
1563 
1564 	memcg = lruvec_memcg(lruvec);
1565 	if (!memcg)
1566 		return NULL;
1567 	memcg = parent_mem_cgroup(memcg);
1568 	if (!memcg)
1569 		return NULL;
1570 	return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
1571 }
1572 
unlock_page_lruvec(struct lruvec * lruvec)1573 static inline void unlock_page_lruvec(struct lruvec *lruvec)
1574 {
1575 	spin_unlock(&lruvec->lru_lock);
1576 }
1577 
unlock_page_lruvec_irq(struct lruvec * lruvec)1578 static inline void unlock_page_lruvec_irq(struct lruvec *lruvec)
1579 {
1580 	spin_unlock_irq(&lruvec->lru_lock);
1581 }
1582 
unlock_page_lruvec_irqrestore(struct lruvec * lruvec,unsigned long flags)1583 static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec,
1584 		unsigned long flags)
1585 {
1586 	spin_unlock_irqrestore(&lruvec->lru_lock, flags);
1587 }
1588 
1589 /* Test requires a stable page->memcg binding, see page_memcg() */
folio_matches_lruvec(struct folio * folio,struct lruvec * lruvec)1590 static inline bool folio_matches_lruvec(struct folio *folio,
1591 		struct lruvec *lruvec)
1592 {
1593 	return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&
1594 	       lruvec_memcg(lruvec) == folio_memcg(folio);
1595 }
1596 
1597 /* Don't lock again iff page's lruvec locked */
folio_lruvec_relock_irq(struct folio * folio,struct lruvec * locked_lruvec)1598 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
1599 		struct lruvec *locked_lruvec)
1600 {
1601 	if (locked_lruvec) {
1602 		if (folio_matches_lruvec(folio, locked_lruvec))
1603 			return locked_lruvec;
1604 
1605 		unlock_page_lruvec_irq(locked_lruvec);
1606 	}
1607 
1608 	return folio_lruvec_lock_irq(folio);
1609 }
1610 
1611 /* Don't lock again iff page's lruvec locked */
folio_lruvec_relock_irqsave(struct folio * folio,struct lruvec * locked_lruvec,unsigned long * flags)1612 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio,
1613 		struct lruvec *locked_lruvec, unsigned long *flags)
1614 {
1615 	if (locked_lruvec) {
1616 		if (folio_matches_lruvec(folio, locked_lruvec))
1617 			return locked_lruvec;
1618 
1619 		unlock_page_lruvec_irqrestore(locked_lruvec, *flags);
1620 	}
1621 
1622 	return folio_lruvec_lock_irqsave(folio, flags);
1623 }
1624 
1625 #ifdef CONFIG_CGROUP_WRITEBACK
1626 
1627 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
1628 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
1629 			 unsigned long *pheadroom, unsigned long *pdirty,
1630 			 unsigned long *pwriteback);
1631 
1632 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1633 					     struct bdi_writeback *wb);
1634 
mem_cgroup_track_foreign_dirty(struct folio * folio,struct bdi_writeback * wb)1635 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
1636 						  struct bdi_writeback *wb)
1637 {
1638 	if (mem_cgroup_disabled())
1639 		return;
1640 
1641 	if (unlikely(&folio_memcg(folio)->css != wb->memcg_css))
1642 		mem_cgroup_track_foreign_dirty_slowpath(folio, wb);
1643 }
1644 
1645 void mem_cgroup_flush_foreign(struct bdi_writeback *wb);
1646 
1647 #else	/* CONFIG_CGROUP_WRITEBACK */
1648 
mem_cgroup_wb_domain(struct bdi_writeback * wb)1649 static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
1650 {
1651 	return NULL;
1652 }
1653 
mem_cgroup_wb_stats(struct bdi_writeback * wb,unsigned long * pfilepages,unsigned long * pheadroom,unsigned long * pdirty,unsigned long * pwriteback)1654 static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
1655 				       unsigned long *pfilepages,
1656 				       unsigned long *pheadroom,
1657 				       unsigned long *pdirty,
1658 				       unsigned long *pwriteback)
1659 {
1660 }
1661 
mem_cgroup_track_foreign_dirty(struct folio * folio,struct bdi_writeback * wb)1662 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
1663 						  struct bdi_writeback *wb)
1664 {
1665 }
1666 
mem_cgroup_flush_foreign(struct bdi_writeback * wb)1667 static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
1668 {
1669 }
1670 
1671 #endif	/* CONFIG_CGROUP_WRITEBACK */
1672 
1673 struct sock;
1674 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
1675 			     gfp_t gfp_mask);
1676 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1677 #ifdef CONFIG_MEMCG
1678 extern struct static_key_false memcg_sockets_enabled_key;
1679 #define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
1680 void mem_cgroup_sk_alloc(struct sock *sk);
1681 void mem_cgroup_sk_free(struct sock *sk);
mem_cgroup_under_socket_pressure(struct mem_cgroup * memcg)1682 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1683 {
1684 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
1685 		return true;
1686 	do {
1687 		if (time_before(jiffies, READ_ONCE(memcg->socket_pressure)))
1688 			return true;
1689 	} while ((memcg = parent_mem_cgroup(memcg)));
1690 	return false;
1691 }
1692 
1693 int alloc_shrinker_info(struct mem_cgroup *memcg);
1694 void free_shrinker_info(struct mem_cgroup *memcg);
1695 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);
1696 void reparent_shrinker_deferred(struct mem_cgroup *memcg);
1697 #else
1698 #define mem_cgroup_sockets_enabled 0
mem_cgroup_sk_alloc(struct sock * sk)1699 static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
mem_cgroup_sk_free(struct sock * sk)1700 static inline void mem_cgroup_sk_free(struct sock *sk) { };
mem_cgroup_under_socket_pressure(struct mem_cgroup * memcg)1701 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1702 {
1703 	return false;
1704 }
1705 
set_shrinker_bit(struct mem_cgroup * memcg,int nid,int shrinker_id)1706 static inline void set_shrinker_bit(struct mem_cgroup *memcg,
1707 				    int nid, int shrinker_id)
1708 {
1709 }
1710 #endif
1711 
1712 #ifdef CONFIG_MEMCG_KMEM
1713 bool mem_cgroup_kmem_disabled(void);
1714 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order);
1715 void __memcg_kmem_uncharge_page(struct page *page, int order);
1716 
1717 struct obj_cgroup *get_obj_cgroup_from_current(void);
1718 struct obj_cgroup *get_obj_cgroup_from_page(struct page *page);
1719 
1720 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size);
1721 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size);
1722 
1723 extern struct static_key_false memcg_kmem_enabled_key;
1724 
memcg_kmem_enabled(void)1725 static inline bool memcg_kmem_enabled(void)
1726 {
1727 	return static_branch_likely(&memcg_kmem_enabled_key);
1728 }
1729 
memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)1730 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1731 					 int order)
1732 {
1733 	if (memcg_kmem_enabled())
1734 		return __memcg_kmem_charge_page(page, gfp, order);
1735 	return 0;
1736 }
1737 
memcg_kmem_uncharge_page(struct page * page,int order)1738 static inline void memcg_kmem_uncharge_page(struct page *page, int order)
1739 {
1740 	if (memcg_kmem_enabled())
1741 		__memcg_kmem_uncharge_page(page, order);
1742 }
1743 
1744 /*
1745  * A helper for accessing memcg's kmem_id, used for getting
1746  * corresponding LRU lists.
1747  */
memcg_kmem_id(struct mem_cgroup * memcg)1748 static inline int memcg_kmem_id(struct mem_cgroup *memcg)
1749 {
1750 	return memcg ? memcg->kmemcg_id : -1;
1751 }
1752 
1753 struct mem_cgroup *mem_cgroup_from_obj(void *p);
1754 
count_objcg_event(struct obj_cgroup * objcg,enum vm_event_item idx)1755 static inline void count_objcg_event(struct obj_cgroup *objcg,
1756 				     enum vm_event_item idx)
1757 {
1758 	struct mem_cgroup *memcg;
1759 
1760 	if (mem_cgroup_kmem_disabled())
1761 		return;
1762 
1763 	rcu_read_lock();
1764 	memcg = obj_cgroup_memcg(objcg);
1765 	count_memcg_events(memcg, idx, 1);
1766 	rcu_read_unlock();
1767 }
1768 
1769 #else
mem_cgroup_kmem_disabled(void)1770 static inline bool mem_cgroup_kmem_disabled(void)
1771 {
1772 	return true;
1773 }
1774 
memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)1775 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1776 					 int order)
1777 {
1778 	return 0;
1779 }
1780 
memcg_kmem_uncharge_page(struct page * page,int order)1781 static inline void memcg_kmem_uncharge_page(struct page *page, int order)
1782 {
1783 }
1784 
__memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)1785 static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1786 					   int order)
1787 {
1788 	return 0;
1789 }
1790 
__memcg_kmem_uncharge_page(struct page * page,int order)1791 static inline void __memcg_kmem_uncharge_page(struct page *page, int order)
1792 {
1793 }
1794 
get_obj_cgroup_from_page(struct page * page)1795 static inline struct obj_cgroup *get_obj_cgroup_from_page(struct page *page)
1796 {
1797 	return NULL;
1798 }
1799 
memcg_kmem_enabled(void)1800 static inline bool memcg_kmem_enabled(void)
1801 {
1802 	return false;
1803 }
1804 
memcg_kmem_id(struct mem_cgroup * memcg)1805 static inline int memcg_kmem_id(struct mem_cgroup *memcg)
1806 {
1807 	return -1;
1808 }
1809 
mem_cgroup_from_obj(void * p)1810 static inline struct mem_cgroup *mem_cgroup_from_obj(void *p)
1811 {
1812        return NULL;
1813 }
1814 
count_objcg_event(struct obj_cgroup * objcg,enum vm_event_item idx)1815 static inline void count_objcg_event(struct obj_cgroup *objcg,
1816 				     enum vm_event_item idx)
1817 {
1818 }
1819 
1820 #endif /* CONFIG_MEMCG_KMEM */
1821 
1822 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
1823 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg);
1824 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size);
1825 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size);
1826 #else
obj_cgroup_may_zswap(struct obj_cgroup * objcg)1827 static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
1828 {
1829 	return true;
1830 }
obj_cgroup_charge_zswap(struct obj_cgroup * objcg,size_t size)1831 static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg,
1832 					   size_t size)
1833 {
1834 }
obj_cgroup_uncharge_zswap(struct obj_cgroup * objcg,size_t size)1835 static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg,
1836 					     size_t size)
1837 {
1838 }
1839 #endif
1840 
1841 #endif /* _LINUX_MEMCONTROL_H */
1842