33 __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx,  in __hugetlb_cgroup_counter_from_cgroup()  argument
37 		return &h_cg->rsvd_hugepage[idx];  in __hugetlb_cgroup_counter_from_cgroup()
38 	return &h_cg->hugepage[idx];  in __hugetlb_cgroup_counter_from_cgroup()
42 hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx)  in hugetlb_cgroup_counter_from_cgroup()  argument
44 	return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false);  in hugetlb_cgroup_counter_from_cgroup()
48 hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx)  in hugetlb_cgroup_counter_from_cgroup_rsvd()  argument
50 	return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true);  in hugetlb_cgroup_counter_from_cgroup_rsvd()
91 	int idx;  in hugetlb_cgroup_init()  local
93 	for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) {  in hugetlb_cgroup_init()
101 				parent_h_cgroup, idx);  in hugetlb_cgroup_init()
103 				parent_h_cgroup, idx);  in hugetlb_cgroup_init()
106 								     idx),  in hugetlb_cgroup_init()
109 			hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),  in hugetlb_cgroup_init()
113 				   pages_per_huge_page(&hstates[idx]));  in hugetlb_cgroup_init()
116 			hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx),  in hugetlb_cgroup_init()
120 			hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),  in hugetlb_cgroup_init()
187 static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_move_parent()  argument
209 		page_counter_charge(&parent->hugepage[idx], nr_pages);  in hugetlb_cgroup_move_parent()
211 	counter = &h_cg->hugepage[idx];  in hugetlb_cgroup_move_parent()
242 static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx,  in hugetlb_event()  argument
245 	atomic_long_inc(&hugetlb->events_local[idx][event]);  in hugetlb_event()
246 	cgroup_file_notify(&hugetlb->events_local_file[idx]);  in hugetlb_event()
249 		atomic_long_inc(&hugetlb->events[idx][event]);  in hugetlb_event()
250 		cgroup_file_notify(&hugetlb->events_file[idx]);  in hugetlb_event()
255 static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,  in __hugetlb_cgroup_charge_cgroup()  argument
269 	if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)  in __hugetlb_cgroup_charge_cgroup()
281 		    __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),  in __hugetlb_cgroup_charge_cgroup()
284 		hugetlb_event(h_cg, idx, HUGETLB_MAX);  in __hugetlb_cgroup_charge_cgroup()
298 int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,  in hugetlb_cgroup_charge_cgroup()  argument
301 	return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false);  in hugetlb_cgroup_charge_cgroup()
304 int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages,  in hugetlb_cgroup_charge_cgroup_rsvd()  argument
307 	return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true);  in hugetlb_cgroup_charge_cgroup_rsvd()
311 static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,  in __hugetlb_cgroup_commit_charge()  argument
321 			h_cg->nodeinfo[folio_nid(folio)]->usage[idx];  in __hugetlb_cgroup_commit_charge()
327 		WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx],  in __hugetlb_cgroup_commit_charge()
332 void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,  in hugetlb_cgroup_commit_charge()  argument
336 	__hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, false);  in hugetlb_cgroup_commit_charge()
339 void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,  in hugetlb_cgroup_commit_charge_rsvd()  argument
343 	__hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, true);  in hugetlb_cgroup_commit_charge_rsvd()
349 static void __hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages,  in __hugetlb_cgroup_uncharge_folio()  argument
362 	page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,  in __hugetlb_cgroup_uncharge_folio()
370 			h_cg->nodeinfo[folio_nid(folio)]->usage[idx];  in __hugetlb_cgroup_uncharge_folio()
376 		WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx],  in __hugetlb_cgroup_uncharge_folio()
381 void hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages,  in hugetlb_cgroup_uncharge_folio()  argument
384 	__hugetlb_cgroup_uncharge_folio(idx, nr_pages, folio, false);  in hugetlb_cgroup_uncharge_folio()
387 void hugetlb_cgroup_uncharge_folio_rsvd(int idx, unsigned long nr_pages,  in hugetlb_cgroup_uncharge_folio_rsvd()  argument
390 	__hugetlb_cgroup_uncharge_folio(idx, nr_pages, folio, true);  in hugetlb_cgroup_uncharge_folio_rsvd()
393 static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,  in __hugetlb_cgroup_uncharge_cgroup()  argument
400 	if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)  in __hugetlb_cgroup_uncharge_cgroup()
403 	page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,  in __hugetlb_cgroup_uncharge_cgroup()
411 void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,  in hugetlb_cgroup_uncharge_cgroup()  argument
414 	__hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false);  in hugetlb_cgroup_uncharge_cgroup()
417 void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages,  in hugetlb_cgroup_uncharge_cgroup_rsvd()  argument
420 	__hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true);  in hugetlb_cgroup_uncharge_cgroup_rsvd()
471 	int idx = MEMFILE_IDX(cft->private);  in hugetlb_cgroup_read_numa_stat()  local
481 			usage += READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]);  in hugetlb_cgroup_read_numa_stat()
487 				   READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]) *  in hugetlb_cgroup_read_numa_stat()
497 		   page_counter_read(&h_cg->hugepage[idx]) * PAGE_SIZE);  in hugetlb_cgroup_read_numa_stat()
509 						   ->usage[idx]);  in hugetlb_cgroup_read_numa_stat()
554 	int idx;  in hugetlb_cgroup_read_u64_max()  local
561 	idx = MEMFILE_IDX(cft->private);  in hugetlb_cgroup_read_u64_max()
562 	counter = &h_cg->hugepage[idx];  in hugetlb_cgroup_read_u64_max()
565 			   pages_per_huge_page(&hstates[idx]));  in hugetlb_cgroup_read_u64_max()
569 		counter = &h_cg->rsvd_hugepage[idx];  in hugetlb_cgroup_read_u64_max()
576 		counter = &h_cg->rsvd_hugepage[idx];  in hugetlb_cgroup_read_u64_max()
598 	int ret, idx;  in hugetlb_cgroup_write()  local
611 	idx = MEMFILE_IDX(of_cft(of)->private);  in hugetlb_cgroup_write()
612 	nr_pages = round_down(nr_pages, pages_per_huge_page(&hstates[idx]));  in hugetlb_cgroup_write()
621 			__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),  in hugetlb_cgroup_write()
687 	int idx;  in __hugetlb_events_show()  local
692 	idx = MEMFILE_IDX(cft->private);  in __hugetlb_events_show()
695 		max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]);  in __hugetlb_events_show()
697 		max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]);  in __hugetlb_events_show()
714 static void __init __hugetlb_cgroup_file_dfl_init(int idx)  in __hugetlb_cgroup_file_dfl_init()  argument
718 	struct hstate *h = &hstates[idx];  in __hugetlb_cgroup_file_dfl_init()
726 	cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);  in __hugetlb_cgroup_file_dfl_init()
734 	cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT);  in __hugetlb_cgroup_file_dfl_init()
742 	cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);  in __hugetlb_cgroup_file_dfl_init()
749 	cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE);  in __hugetlb_cgroup_file_dfl_init()
756 	cft->private = MEMFILE_PRIVATE(idx, 0);  in __hugetlb_cgroup_file_dfl_init()
758 	cft->file_offset = offsetof(struct hugetlb_cgroup, events_file[idx]);  in __hugetlb_cgroup_file_dfl_init()
764 	cft->private = MEMFILE_PRIVATE(idx, 0);  in __hugetlb_cgroup_file_dfl_init()
767 				    events_local_file[idx]);  in __hugetlb_cgroup_file_dfl_init()
773 	cft->private = MEMFILE_PRIVATE(idx, 0);  in __hugetlb_cgroup_file_dfl_init()
785 static void __init __hugetlb_cgroup_file_legacy_init(int idx)  in __hugetlb_cgroup_file_legacy_init()  argument
789 	struct hstate *h = &hstates[idx];  in __hugetlb_cgroup_file_legacy_init()
797 	cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);  in __hugetlb_cgroup_file_legacy_init()
804 	cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT);  in __hugetlb_cgroup_file_legacy_init()
811 	cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);  in __hugetlb_cgroup_file_legacy_init()
817 	cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE);  in __hugetlb_cgroup_file_legacy_init()
823 	cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);  in __hugetlb_cgroup_file_legacy_init()
830 	cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_MAX_USAGE);  in __hugetlb_cgroup_file_legacy_init()
837 	cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);  in __hugetlb_cgroup_file_legacy_init()
844 	cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_FAILCNT);  in __hugetlb_cgroup_file_legacy_init()
851 	cft->private = MEMFILE_PRIVATE(idx, 1);  in __hugetlb_cgroup_file_legacy_init()
862 static void __init __hugetlb_cgroup_file_init(int idx)  in __hugetlb_cgroup_file_init()  argument
864 	__hugetlb_cgroup_file_dfl_init(idx);  in __hugetlb_cgroup_file_init()
865 	__hugetlb_cgroup_file_legacy_init(idx);  in __hugetlb_cgroup_file_init()