1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  Housekeeping management. Manage the targets for routine code that can run on
4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5  *
6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8  *
9  */
10 
11 enum hk_flags {
12 	HK_FLAG_TIMER		= BIT(HK_TYPE_TIMER),
13 	HK_FLAG_RCU		= BIT(HK_TYPE_RCU),
14 	HK_FLAG_MISC		= BIT(HK_TYPE_MISC),
15 	HK_FLAG_SCHED		= BIT(HK_TYPE_SCHED),
16 	HK_FLAG_TICK		= BIT(HK_TYPE_TICK),
17 	HK_FLAG_DOMAIN		= BIT(HK_TYPE_DOMAIN),
18 	HK_FLAG_WQ		= BIT(HK_TYPE_WQ),
19 	HK_FLAG_MANAGED_IRQ	= BIT(HK_TYPE_MANAGED_IRQ),
20 	HK_FLAG_KTHREAD		= BIT(HK_TYPE_KTHREAD),
21 };
22 
23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
24 EXPORT_SYMBOL_GPL(housekeeping_overridden);
25 
26 struct housekeeping {
27 	cpumask_var_t cpumasks[HK_TYPE_MAX];
28 	unsigned long flags;
29 };
30 
31 static struct housekeeping housekeeping;
32 
housekeeping_enabled(enum hk_type type)33 bool housekeeping_enabled(enum hk_type type)
34 {
35 	return !!(housekeeping.flags & BIT(type));
36 }
37 EXPORT_SYMBOL_GPL(housekeeping_enabled);
38 
housekeeping_any_cpu(enum hk_type type)39 int housekeeping_any_cpu(enum hk_type type)
40 {
41 	int cpu;
42 
43 	if (static_branch_unlikely(&housekeeping_overridden)) {
44 		if (housekeeping.flags & BIT(type)) {
45 			cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
46 			if (cpu < nr_cpu_ids)
47 				return cpu;
48 
49 			return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
50 		}
51 	}
52 	return smp_processor_id();
53 }
54 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
55 
housekeeping_cpumask(enum hk_type type)56 const struct cpumask *housekeeping_cpumask(enum hk_type type)
57 {
58 	if (static_branch_unlikely(&housekeeping_overridden))
59 		if (housekeeping.flags & BIT(type))
60 			return housekeeping.cpumasks[type];
61 	return cpu_possible_mask;
62 }
63 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
64 
housekeeping_affine(struct task_struct * t,enum hk_type type)65 void housekeeping_affine(struct task_struct *t, enum hk_type type)
66 {
67 	if (static_branch_unlikely(&housekeeping_overridden))
68 		if (housekeeping.flags & BIT(type))
69 			set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
70 }
71 EXPORT_SYMBOL_GPL(housekeeping_affine);
72 
housekeeping_test_cpu(int cpu,enum hk_type type)73 bool housekeeping_test_cpu(int cpu, enum hk_type type)
74 {
75 	if (static_branch_unlikely(&housekeeping_overridden))
76 		if (housekeeping.flags & BIT(type))
77 			return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
78 	return true;
79 }
80 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
81 
housekeeping_init(void)82 void __init housekeeping_init(void)
83 {
84 	enum hk_type type;
85 
86 	if (!housekeeping.flags)
87 		return;
88 
89 	static_branch_enable(&housekeeping_overridden);
90 
91 	if (housekeeping.flags & HK_FLAG_TICK)
92 		sched_tick_offload_init();
93 
94 	for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
95 		/* We need at least one CPU to handle housekeeping work */
96 		WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
97 	}
98 }
99 
housekeeping_setup_type(enum hk_type type,cpumask_var_t housekeeping_staging)100 static void __init housekeeping_setup_type(enum hk_type type,
101 					   cpumask_var_t housekeeping_staging)
102 {
103 
104 	alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
105 	cpumask_copy(housekeeping.cpumasks[type],
106 		     housekeeping_staging);
107 }
108 
housekeeping_setup(char * str,unsigned long flags)109 static int __init housekeeping_setup(char *str, unsigned long flags)
110 {
111 	cpumask_var_t non_housekeeping_mask, housekeeping_staging;
112 	int err = 0;
113 
114 	if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
115 		if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
116 			pr_warn("Housekeeping: nohz unsupported."
117 				" Build with CONFIG_NO_HZ_FULL\n");
118 			return 0;
119 		}
120 	}
121 
122 	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
123 	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
124 		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
125 		goto free_non_housekeeping_mask;
126 	}
127 
128 	alloc_bootmem_cpumask_var(&housekeeping_staging);
129 	cpumask_andnot(housekeeping_staging,
130 		       cpu_possible_mask, non_housekeeping_mask);
131 
132 	if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
133 		__cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
134 		__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
135 		if (!housekeeping.flags) {
136 			pr_warn("Housekeeping: must include one present CPU, "
137 				"using boot CPU:%d\n", smp_processor_id());
138 		}
139 	}
140 
141 	if (!housekeeping.flags) {
142 		/* First setup call ("nohz_full=" or "isolcpus=") */
143 		enum hk_type type;
144 
145 		for_each_set_bit(type, &flags, HK_TYPE_MAX)
146 			housekeeping_setup_type(type, housekeeping_staging);
147 	} else {
148 		/* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
149 		enum hk_type type;
150 		unsigned long iter_flags = flags & housekeeping.flags;
151 
152 		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
153 			if (!cpumask_equal(housekeeping_staging,
154 					   housekeeping.cpumasks[type])) {
155 				pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
156 				goto free_housekeeping_staging;
157 			}
158 		}
159 
160 		iter_flags = flags & ~housekeeping.flags;
161 
162 		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
163 			housekeeping_setup_type(type, housekeeping_staging);
164 	}
165 
166 	if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
167 		tick_nohz_full_setup(non_housekeeping_mask);
168 
169 	housekeeping.flags |= flags;
170 	err = 1;
171 
172 free_housekeeping_staging:
173 	free_bootmem_cpumask_var(housekeeping_staging);
174 free_non_housekeeping_mask:
175 	free_bootmem_cpumask_var(non_housekeeping_mask);
176 
177 	return err;
178 }
179 
housekeeping_nohz_full_setup(char * str)180 static int __init housekeeping_nohz_full_setup(char *str)
181 {
182 	unsigned long flags;
183 
184 	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
185 		HK_FLAG_MISC | HK_FLAG_KTHREAD;
186 
187 	return housekeeping_setup(str, flags);
188 }
189 __setup("nohz_full=", housekeeping_nohz_full_setup);
190 
housekeeping_isolcpus_setup(char * str)191 static int __init housekeeping_isolcpus_setup(char *str)
192 {
193 	unsigned long flags = 0;
194 	bool illegal = false;
195 	char *par;
196 	int len;
197 
198 	while (isalpha(*str)) {
199 		if (!strncmp(str, "nohz,", 5)) {
200 			str += 5;
201 			flags |= HK_FLAG_TICK;
202 			continue;
203 		}
204 
205 		if (!strncmp(str, "domain,", 7)) {
206 			str += 7;
207 			flags |= HK_FLAG_DOMAIN;
208 			continue;
209 		}
210 
211 		if (!strncmp(str, "managed_irq,", 12)) {
212 			str += 12;
213 			flags |= HK_FLAG_MANAGED_IRQ;
214 			continue;
215 		}
216 
217 		/*
218 		 * Skip unknown sub-parameter and validate that it is not
219 		 * containing an invalid character.
220 		 */
221 		for (par = str, len = 0; *str && *str != ','; str++, len++) {
222 			if (!isalpha(*str) && *str != '_')
223 				illegal = true;
224 		}
225 
226 		if (illegal) {
227 			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
228 			return 0;
229 		}
230 
231 		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
232 		str++;
233 	}
234 
235 	/* Default behaviour for isolcpus without flags */
236 	if (!flags)
237 		flags |= HK_FLAG_DOMAIN;
238 
239 	return housekeeping_setup(str, flags);
240 }
241 __setup("isolcpus=", housekeeping_isolcpus_setup);
242