1 /* CPU control.
2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
3  *
4  * This code is licenced under the GPL.
5  */
6 #include <linux/sched/mm.h>
7 #include <linux/proc_fs.h>
8 #include <linux/smp.h>
9 #include <linux/init.h>
10 #include <linux/notifier.h>
11 #include <linux/sched/signal.h>
12 #include <linux/sched/hotplug.h>
13 #include <linux/sched/isolation.h>
14 #include <linux/sched/task.h>
15 #include <linux/sched/smt.h>
16 #include <linux/unistd.h>
17 #include <linux/cpu.h>
18 #include <linux/oom.h>
19 #include <linux/rcupdate.h>
20 #include <linux/export.h>
21 #include <linux/bug.h>
22 #include <linux/kthread.h>
23 #include <linux/stop_machine.h>
24 #include <linux/mutex.h>
25 #include <linux/gfp.h>
26 #include <linux/suspend.h>
27 #include <linux/lockdep.h>
28 #include <linux/tick.h>
29 #include <linux/irq.h>
30 #include <linux/nmi.h>
31 #include <linux/smpboot.h>
32 #include <linux/relay.h>
33 #include <linux/slab.h>
34 #include <linux/scs.h>
35 #include <linux/percpu-rwsem.h>
36 #include <linux/cpuset.h>
37 #include <linux/random.h>
38 #include <linux/cc_platform.h>
39 
40 #include <trace/events/power.h>
41 #define CREATE_TRACE_POINTS
42 #include <trace/events/cpuhp.h>
43 
44 #include "smpboot.h"
45 
46 /**
47  * struct cpuhp_cpu_state - Per cpu hotplug state storage
48  * @state:	The current cpu state
49  * @target:	The target state
50  * @fail:	Current CPU hotplug callback state
51  * @thread:	Pointer to the hotplug thread
52  * @should_run:	Thread should execute
53  * @rollback:	Perform a rollback
54  * @single:	Single callback invocation
55  * @bringup:	Single callback bringup or teardown selector
56  * @cpu:	CPU number
57  * @node:	Remote CPU node; for multi-instance, do a
58  *		single entry callback for install/remove
59  * @last:	For multi-instance rollback, remember how far we got
60  * @cb_state:	The state for a single callback (install/uninstall)
61  * @result:	Result of the operation
62  * @done_up:	Signal completion to the issuer of the task for cpu-up
63  * @done_down:	Signal completion to the issuer of the task for cpu-down
64  */
65 struct cpuhp_cpu_state {
66 	enum cpuhp_state	state;
67 	enum cpuhp_state	target;
68 	enum cpuhp_state	fail;
69 #ifdef CONFIG_SMP
70 	struct task_struct	*thread;
71 	bool			should_run;
72 	bool			rollback;
73 	bool			single;
74 	bool			bringup;
75 	struct hlist_node	*node;
76 	struct hlist_node	*last;
77 	enum cpuhp_state	cb_state;
78 	int			result;
79 	struct completion	done_up;
80 	struct completion	done_down;
81 #endif
82 };
83 
84 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
85 	.fail = CPUHP_INVALID,
86 };
87 
88 #ifdef CONFIG_SMP
89 cpumask_t cpus_booted_once_mask;
90 #endif
91 
92 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
93 static struct lockdep_map cpuhp_state_up_map =
94 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
95 static struct lockdep_map cpuhp_state_down_map =
96 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
97 
98 
cpuhp_lock_acquire(bool bringup)99 static inline void cpuhp_lock_acquire(bool bringup)
100 {
101 	lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
102 }
103 
cpuhp_lock_release(bool bringup)104 static inline void cpuhp_lock_release(bool bringup)
105 {
106 	lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
107 }
108 #else
109 
cpuhp_lock_acquire(bool bringup)110 static inline void cpuhp_lock_acquire(bool bringup) { }
cpuhp_lock_release(bool bringup)111 static inline void cpuhp_lock_release(bool bringup) { }
112 
113 #endif
114 
115 /**
116  * struct cpuhp_step - Hotplug state machine step
117  * @name:	Name of the step
118  * @startup:	Startup function of the step
119  * @teardown:	Teardown function of the step
120  * @cant_stop:	Bringup/teardown can't be stopped at this step
121  * @multi_instance:	State has multiple instances which get added afterwards
122  */
123 struct cpuhp_step {
124 	const char		*name;
125 	union {
126 		int		(*single)(unsigned int cpu);
127 		int		(*multi)(unsigned int cpu,
128 					 struct hlist_node *node);
129 	} startup;
130 	union {
131 		int		(*single)(unsigned int cpu);
132 		int		(*multi)(unsigned int cpu,
133 					 struct hlist_node *node);
134 	} teardown;
135 	/* private: */
136 	struct hlist_head	list;
137 	/* public: */
138 	bool			cant_stop;
139 	bool			multi_instance;
140 };
141 
142 static DEFINE_MUTEX(cpuhp_state_mutex);
143 static struct cpuhp_step cpuhp_hp_states[];
144 
cpuhp_get_step(enum cpuhp_state state)145 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
146 {
147 	return cpuhp_hp_states + state;
148 }
149 
cpuhp_step_empty(bool bringup,struct cpuhp_step * step)150 static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
151 {
152 	return bringup ? !step->startup.single : !step->teardown.single;
153 }
154 
155 /**
156  * cpuhp_invoke_callback - Invoke the callbacks for a given state
157  * @cpu:	The cpu for which the callback should be invoked
158  * @state:	The state to do callbacks for
159  * @bringup:	True if the bringup callback should be invoked
160  * @node:	For multi-instance, do a single entry callback for install/remove
161  * @lastp:	For multi-instance rollback, remember how far we got
162  *
163  * Called from cpu hotplug and from the state register machinery.
164  *
165  * Return: %0 on success or a negative errno code
166  */
cpuhp_invoke_callback(unsigned int cpu,enum cpuhp_state state,bool bringup,struct hlist_node * node,struct hlist_node ** lastp)167 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
168 				 bool bringup, struct hlist_node *node,
169 				 struct hlist_node **lastp)
170 {
171 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
172 	struct cpuhp_step *step = cpuhp_get_step(state);
173 	int (*cbm)(unsigned int cpu, struct hlist_node *node);
174 	int (*cb)(unsigned int cpu);
175 	int ret, cnt;
176 
177 	if (st->fail == state) {
178 		st->fail = CPUHP_INVALID;
179 		return -EAGAIN;
180 	}
181 
182 	if (cpuhp_step_empty(bringup, step)) {
183 		WARN_ON_ONCE(1);
184 		return 0;
185 	}
186 
187 	if (!step->multi_instance) {
188 		WARN_ON_ONCE(lastp && *lastp);
189 		cb = bringup ? step->startup.single : step->teardown.single;
190 
191 		trace_cpuhp_enter(cpu, st->target, state, cb);
192 		ret = cb(cpu);
193 		trace_cpuhp_exit(cpu, st->state, state, ret);
194 		return ret;
195 	}
196 	cbm = bringup ? step->startup.multi : step->teardown.multi;
197 
198 	/* Single invocation for instance add/remove */
199 	if (node) {
200 		WARN_ON_ONCE(lastp && *lastp);
201 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
202 		ret = cbm(cpu, node);
203 		trace_cpuhp_exit(cpu, st->state, state, ret);
204 		return ret;
205 	}
206 
207 	/* State transition. Invoke on all instances */
208 	cnt = 0;
209 	hlist_for_each(node, &step->list) {
210 		if (lastp && node == *lastp)
211 			break;
212 
213 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
214 		ret = cbm(cpu, node);
215 		trace_cpuhp_exit(cpu, st->state, state, ret);
216 		if (ret) {
217 			if (!lastp)
218 				goto err;
219 
220 			*lastp = node;
221 			return ret;
222 		}
223 		cnt++;
224 	}
225 	if (lastp)
226 		*lastp = NULL;
227 	return 0;
228 err:
229 	/* Rollback the instances if one failed */
230 	cbm = !bringup ? step->startup.multi : step->teardown.multi;
231 	if (!cbm)
232 		return ret;
233 
234 	hlist_for_each(node, &step->list) {
235 		if (!cnt--)
236 			break;
237 
238 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
239 		ret = cbm(cpu, node);
240 		trace_cpuhp_exit(cpu, st->state, state, ret);
241 		/*
242 		 * Rollback must not fail,
243 		 */
244 		WARN_ON_ONCE(ret);
245 	}
246 	return ret;
247 }
248 
249 #ifdef CONFIG_SMP
cpuhp_is_ap_state(enum cpuhp_state state)250 static bool cpuhp_is_ap_state(enum cpuhp_state state)
251 {
252 	/*
253 	 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
254 	 * purposes as that state is handled explicitly in cpu_down.
255 	 */
256 	return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
257 }
258 
wait_for_ap_thread(struct cpuhp_cpu_state * st,bool bringup)259 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
260 {
261 	struct completion *done = bringup ? &st->done_up : &st->done_down;
262 	wait_for_completion(done);
263 }
264 
complete_ap_thread(struct cpuhp_cpu_state * st,bool bringup)265 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
266 {
267 	struct completion *done = bringup ? &st->done_up : &st->done_down;
268 	complete(done);
269 }
270 
271 /*
272  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
273  */
cpuhp_is_atomic_state(enum cpuhp_state state)274 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
275 {
276 	return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
277 }
278 
279 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
280 static DEFINE_MUTEX(cpu_add_remove_lock);
281 bool cpuhp_tasks_frozen;
282 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
283 
284 /*
285  * The following two APIs (cpu_maps_update_begin/done) must be used when
286  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
287  */
cpu_maps_update_begin(void)288 void cpu_maps_update_begin(void)
289 {
290 	mutex_lock(&cpu_add_remove_lock);
291 }
292 
cpu_maps_update_done(void)293 void cpu_maps_update_done(void)
294 {
295 	mutex_unlock(&cpu_add_remove_lock);
296 }
297 
298 /*
299  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
300  * Should always be manipulated under cpu_add_remove_lock
301  */
302 static int cpu_hotplug_disabled;
303 
304 #ifdef CONFIG_HOTPLUG_CPU
305 
306 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
307 
cpus_read_lock(void)308 void cpus_read_lock(void)
309 {
310 	percpu_down_read(&cpu_hotplug_lock);
311 }
312 EXPORT_SYMBOL_GPL(cpus_read_lock);
313 
cpus_read_trylock(void)314 int cpus_read_trylock(void)
315 {
316 	return percpu_down_read_trylock(&cpu_hotplug_lock);
317 }
318 EXPORT_SYMBOL_GPL(cpus_read_trylock);
319 
cpus_read_unlock(void)320 void cpus_read_unlock(void)
321 {
322 	percpu_up_read(&cpu_hotplug_lock);
323 }
324 EXPORT_SYMBOL_GPL(cpus_read_unlock);
325 
cpus_write_lock(void)326 void cpus_write_lock(void)
327 {
328 	percpu_down_write(&cpu_hotplug_lock);
329 }
330 
cpus_write_unlock(void)331 void cpus_write_unlock(void)
332 {
333 	percpu_up_write(&cpu_hotplug_lock);
334 }
335 
lockdep_assert_cpus_held(void)336 void lockdep_assert_cpus_held(void)
337 {
338 	/*
339 	 * We can't have hotplug operations before userspace starts running,
340 	 * and some init codepaths will knowingly not take the hotplug lock.
341 	 * This is all valid, so mute lockdep until it makes sense to report
342 	 * unheld locks.
343 	 */
344 	if (system_state < SYSTEM_RUNNING)
345 		return;
346 
347 	percpu_rwsem_assert_held(&cpu_hotplug_lock);
348 }
349 
350 #ifdef CONFIG_LOCKDEP
lockdep_is_cpus_held(void)351 int lockdep_is_cpus_held(void)
352 {
353 	return percpu_rwsem_is_held(&cpu_hotplug_lock);
354 }
355 #endif
356 
lockdep_acquire_cpus_lock(void)357 static void lockdep_acquire_cpus_lock(void)
358 {
359 	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
360 }
361 
lockdep_release_cpus_lock(void)362 static void lockdep_release_cpus_lock(void)
363 {
364 	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
365 }
366 
367 /*
368  * Wait for currently running CPU hotplug operations to complete (if any) and
369  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
370  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
371  * hotplug path before performing hotplug operations. So acquiring that lock
372  * guarantees mutual exclusion from any currently running hotplug operations.
373  */
cpu_hotplug_disable(void)374 void cpu_hotplug_disable(void)
375 {
376 	cpu_maps_update_begin();
377 	cpu_hotplug_disabled++;
378 	cpu_maps_update_done();
379 }
380 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
381 
__cpu_hotplug_enable(void)382 static void __cpu_hotplug_enable(void)
383 {
384 	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
385 		return;
386 	cpu_hotplug_disabled--;
387 }
388 
cpu_hotplug_enable(void)389 void cpu_hotplug_enable(void)
390 {
391 	cpu_maps_update_begin();
392 	__cpu_hotplug_enable();
393 	cpu_maps_update_done();
394 }
395 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
396 
397 #else
398 
lockdep_acquire_cpus_lock(void)399 static void lockdep_acquire_cpus_lock(void)
400 {
401 }
402 
lockdep_release_cpus_lock(void)403 static void lockdep_release_cpus_lock(void)
404 {
405 }
406 
407 #endif	/* CONFIG_HOTPLUG_CPU */
408 
409 /*
410  * Architectures that need SMT-specific errata handling during SMT hotplug
411  * should override this.
412  */
arch_smt_update(void)413 void __weak arch_smt_update(void) { }
414 
415 #ifdef CONFIG_HOTPLUG_SMT
416 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
417 
cpu_smt_disable(bool force)418 void __init cpu_smt_disable(bool force)
419 {
420 	if (!cpu_smt_possible())
421 		return;
422 
423 	if (force) {
424 		pr_info("SMT: Force disabled\n");
425 		cpu_smt_control = CPU_SMT_FORCE_DISABLED;
426 	} else {
427 		pr_info("SMT: disabled\n");
428 		cpu_smt_control = CPU_SMT_DISABLED;
429 	}
430 }
431 
432 /*
433  * The decision whether SMT is supported can only be done after the full
434  * CPU identification. Called from architecture code.
435  */
cpu_smt_check_topology(void)436 void __init cpu_smt_check_topology(void)
437 {
438 	if (!topology_smt_supported())
439 		cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
440 }
441 
smt_cmdline_disable(char * str)442 static int __init smt_cmdline_disable(char *str)
443 {
444 	cpu_smt_disable(str && !strcmp(str, "force"));
445 	return 0;
446 }
447 early_param("nosmt", smt_cmdline_disable);
448 
cpu_smt_allowed(unsigned int cpu)449 static inline bool cpu_smt_allowed(unsigned int cpu)
450 {
451 	if (cpu_smt_control == CPU_SMT_ENABLED)
452 		return true;
453 
454 	if (topology_is_primary_thread(cpu))
455 		return true;
456 
457 	/*
458 	 * On x86 it's required to boot all logical CPUs at least once so
459 	 * that the init code can get a chance to set CR4.MCE on each
460 	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
461 	 * core will shutdown the machine.
462 	 */
463 	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
464 }
465 
466 /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
cpu_smt_possible(void)467 bool cpu_smt_possible(void)
468 {
469 	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
470 		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
471 }
472 EXPORT_SYMBOL_GPL(cpu_smt_possible);
473 #else
cpu_smt_allowed(unsigned int cpu)474 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
475 #endif
476 
477 static inline enum cpuhp_state
cpuhp_set_state(int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target)478 cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
479 {
480 	enum cpuhp_state prev_state = st->state;
481 	bool bringup = st->state < target;
482 
483 	st->rollback = false;
484 	st->last = NULL;
485 
486 	st->target = target;
487 	st->single = false;
488 	st->bringup = bringup;
489 	if (cpu_dying(cpu) != !bringup)
490 		set_cpu_dying(cpu, !bringup);
491 
492 	return prev_state;
493 }
494 
495 static inline void
cpuhp_reset_state(int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state prev_state)496 cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
497 		  enum cpuhp_state prev_state)
498 {
499 	bool bringup = !st->bringup;
500 
501 	st->target = prev_state;
502 
503 	/*
504 	 * Already rolling back. No need invert the bringup value or to change
505 	 * the current state.
506 	 */
507 	if (st->rollback)
508 		return;
509 
510 	st->rollback = true;
511 
512 	/*
513 	 * If we have st->last we need to undo partial multi_instance of this
514 	 * state first. Otherwise start undo at the previous state.
515 	 */
516 	if (!st->last) {
517 		if (st->bringup)
518 			st->state--;
519 		else
520 			st->state++;
521 	}
522 
523 	st->bringup = bringup;
524 	if (cpu_dying(cpu) != !bringup)
525 		set_cpu_dying(cpu, !bringup);
526 }
527 
528 /* Regular hotplug invocation of the AP hotplug thread */
__cpuhp_kick_ap(struct cpuhp_cpu_state * st)529 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
530 {
531 	if (!st->single && st->state == st->target)
532 		return;
533 
534 	st->result = 0;
535 	/*
536 	 * Make sure the above stores are visible before should_run becomes
537 	 * true. Paired with the mb() above in cpuhp_thread_fun()
538 	 */
539 	smp_mb();
540 	st->should_run = true;
541 	wake_up_process(st->thread);
542 	wait_for_ap_thread(st, st->bringup);
543 }
544 
cpuhp_kick_ap(int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target)545 static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
546 			 enum cpuhp_state target)
547 {
548 	enum cpuhp_state prev_state;
549 	int ret;
550 
551 	prev_state = cpuhp_set_state(cpu, st, target);
552 	__cpuhp_kick_ap(st);
553 	if ((ret = st->result)) {
554 		cpuhp_reset_state(cpu, st, prev_state);
555 		__cpuhp_kick_ap(st);
556 	}
557 
558 	return ret;
559 }
560 
bringup_wait_for_ap(unsigned int cpu)561 static int bringup_wait_for_ap(unsigned int cpu)
562 {
563 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
564 
565 	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
566 	wait_for_ap_thread(st, true);
567 	if (WARN_ON_ONCE((!cpu_online(cpu))))
568 		return -ECANCELED;
569 
570 	/* Unpark the hotplug thread of the target cpu */
571 	kthread_unpark(st->thread);
572 
573 	/*
574 	 * SMT soft disabling on X86 requires to bring the CPU out of the
575 	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
576 	 * CPU marked itself as booted_once in notify_cpu_starting() so the
577 	 * cpu_smt_allowed() check will now return false if this is not the
578 	 * primary sibling.
579 	 */
580 	if (!cpu_smt_allowed(cpu))
581 		return -ECANCELED;
582 
583 	if (st->target <= CPUHP_AP_ONLINE_IDLE)
584 		return 0;
585 
586 	return cpuhp_kick_ap(cpu, st, st->target);
587 }
588 
bringup_cpu(unsigned int cpu)589 static int bringup_cpu(unsigned int cpu)
590 {
591 	struct task_struct *idle = idle_thread_get(cpu);
592 	int ret;
593 
594 	/*
595 	 * Reset stale stack state from the last time this CPU was online.
596 	 */
597 	scs_task_reset(idle);
598 	kasan_unpoison_task_stack(idle);
599 
600 	/*
601 	 * Some architectures have to walk the irq descriptors to
602 	 * setup the vector space for the cpu which comes online.
603 	 * Prevent irq alloc/free across the bringup.
604 	 */
605 	irq_lock_sparse();
606 
607 	/* Arch-specific enabling code. */
608 	ret = __cpu_up(cpu, idle);
609 	irq_unlock_sparse();
610 	if (ret)
611 		return ret;
612 	return bringup_wait_for_ap(cpu);
613 }
614 
finish_cpu(unsigned int cpu)615 static int finish_cpu(unsigned int cpu)
616 {
617 	struct task_struct *idle = idle_thread_get(cpu);
618 	struct mm_struct *mm = idle->active_mm;
619 
620 	/*
621 	 * idle_task_exit() will have switched to &init_mm, now
622 	 * clean up any remaining active_mm state.
623 	 */
624 	if (mm != &init_mm)
625 		idle->active_mm = &init_mm;
626 	mmdrop(mm);
627 	return 0;
628 }
629 
630 /*
631  * Hotplug state machine related functions
632  */
633 
634 /*
635  * Get the next state to run. Empty ones will be skipped. Returns true if a
636  * state must be run.
637  *
638  * st->state will be modified ahead of time, to match state_to_run, as if it
639  * has already ran.
640  */
cpuhp_next_state(bool bringup,enum cpuhp_state * state_to_run,struct cpuhp_cpu_state * st,enum cpuhp_state target)641 static bool cpuhp_next_state(bool bringup,
642 			     enum cpuhp_state *state_to_run,
643 			     struct cpuhp_cpu_state *st,
644 			     enum cpuhp_state target)
645 {
646 	do {
647 		if (bringup) {
648 			if (st->state >= target)
649 				return false;
650 
651 			*state_to_run = ++st->state;
652 		} else {
653 			if (st->state <= target)
654 				return false;
655 
656 			*state_to_run = st->state--;
657 		}
658 
659 		if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
660 			break;
661 	} while (true);
662 
663 	return true;
664 }
665 
__cpuhp_invoke_callback_range(bool bringup,unsigned int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target,bool nofail)666 static int __cpuhp_invoke_callback_range(bool bringup,
667 					 unsigned int cpu,
668 					 struct cpuhp_cpu_state *st,
669 					 enum cpuhp_state target,
670 					 bool nofail)
671 {
672 	enum cpuhp_state state;
673 	int ret = 0;
674 
675 	while (cpuhp_next_state(bringup, &state, st, target)) {
676 		int err;
677 
678 		err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
679 		if (!err)
680 			continue;
681 
682 		if (nofail) {
683 			pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
684 				cpu, bringup ? "UP" : "DOWN",
685 				cpuhp_get_step(st->state)->name,
686 				st->state, err);
687 			ret = -1;
688 		} else {
689 			ret = err;
690 			break;
691 		}
692 	}
693 
694 	return ret;
695 }
696 
cpuhp_invoke_callback_range(bool bringup,unsigned int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target)697 static inline int cpuhp_invoke_callback_range(bool bringup,
698 					      unsigned int cpu,
699 					      struct cpuhp_cpu_state *st,
700 					      enum cpuhp_state target)
701 {
702 	return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);
703 }
704 
cpuhp_invoke_callback_range_nofail(bool bringup,unsigned int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target)705 static inline void cpuhp_invoke_callback_range_nofail(bool bringup,
706 						      unsigned int cpu,
707 						      struct cpuhp_cpu_state *st,
708 						      enum cpuhp_state target)
709 {
710 	__cpuhp_invoke_callback_range(bringup, cpu, st, target, true);
711 }
712 
can_rollback_cpu(struct cpuhp_cpu_state * st)713 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
714 {
715 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
716 		return true;
717 	/*
718 	 * When CPU hotplug is disabled, then taking the CPU down is not
719 	 * possible because takedown_cpu() and the architecture and
720 	 * subsystem specific mechanisms are not available. So the CPU
721 	 * which would be completely unplugged again needs to stay around
722 	 * in the current state.
723 	 */
724 	return st->state <= CPUHP_BRINGUP_CPU;
725 }
726 
cpuhp_up_callbacks(unsigned int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target)727 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
728 			      enum cpuhp_state target)
729 {
730 	enum cpuhp_state prev_state = st->state;
731 	int ret = 0;
732 
733 	ret = cpuhp_invoke_callback_range(true, cpu, st, target);
734 	if (ret) {
735 		pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
736 			 ret, cpu, cpuhp_get_step(st->state)->name,
737 			 st->state);
738 
739 		cpuhp_reset_state(cpu, st, prev_state);
740 		if (can_rollback_cpu(st))
741 			WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
742 							    prev_state));
743 	}
744 	return ret;
745 }
746 
747 /*
748  * The cpu hotplug threads manage the bringup and teardown of the cpus
749  */
cpuhp_should_run(unsigned int cpu)750 static int cpuhp_should_run(unsigned int cpu)
751 {
752 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
753 
754 	return st->should_run;
755 }
756 
757 /*
758  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
759  * callbacks when a state gets [un]installed at runtime.
760  *
761  * Each invocation of this function by the smpboot thread does a single AP
762  * state callback.
763  *
764  * It has 3 modes of operation:
765  *  - single: runs st->cb_state
766  *  - up:     runs ++st->state, while st->state < st->target
767  *  - down:   runs st->state--, while st->state > st->target
768  *
769  * When complete or on error, should_run is cleared and the completion is fired.
770  */
cpuhp_thread_fun(unsigned int cpu)771 static void cpuhp_thread_fun(unsigned int cpu)
772 {
773 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
774 	bool bringup = st->bringup;
775 	enum cpuhp_state state;
776 
777 	if (WARN_ON_ONCE(!st->should_run))
778 		return;
779 
780 	/*
781 	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
782 	 * that if we see ->should_run we also see the rest of the state.
783 	 */
784 	smp_mb();
785 
786 	/*
787 	 * The BP holds the hotplug lock, but we're now running on the AP,
788 	 * ensure that anybody asserting the lock is held, will actually find
789 	 * it so.
790 	 */
791 	lockdep_acquire_cpus_lock();
792 	cpuhp_lock_acquire(bringup);
793 
794 	if (st->single) {
795 		state = st->cb_state;
796 		st->should_run = false;
797 	} else {
798 		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
799 		if (!st->should_run)
800 			goto end;
801 	}
802 
803 	WARN_ON_ONCE(!cpuhp_is_ap_state(state));
804 
805 	if (cpuhp_is_atomic_state(state)) {
806 		local_irq_disable();
807 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
808 		local_irq_enable();
809 
810 		/*
811 		 * STARTING/DYING must not fail!
812 		 */
813 		WARN_ON_ONCE(st->result);
814 	} else {
815 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
816 	}
817 
818 	if (st->result) {
819 		/*
820 		 * If we fail on a rollback, we're up a creek without no
821 		 * paddle, no way forward, no way back. We loose, thanks for
822 		 * playing.
823 		 */
824 		WARN_ON_ONCE(st->rollback);
825 		st->should_run = false;
826 	}
827 
828 end:
829 	cpuhp_lock_release(bringup);
830 	lockdep_release_cpus_lock();
831 
832 	if (!st->should_run)
833 		complete_ap_thread(st, bringup);
834 }
835 
836 /* Invoke a single callback on a remote cpu */
837 static int
cpuhp_invoke_ap_callback(int cpu,enum cpuhp_state state,bool bringup,struct hlist_node * node)838 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
839 			 struct hlist_node *node)
840 {
841 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
842 	int ret;
843 
844 	if (!cpu_online(cpu))
845 		return 0;
846 
847 	cpuhp_lock_acquire(false);
848 	cpuhp_lock_release(false);
849 
850 	cpuhp_lock_acquire(true);
851 	cpuhp_lock_release(true);
852 
853 	/*
854 	 * If we are up and running, use the hotplug thread. For early calls
855 	 * we invoke the thread function directly.
856 	 */
857 	if (!st->thread)
858 		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
859 
860 	st->rollback = false;
861 	st->last = NULL;
862 
863 	st->node = node;
864 	st->bringup = bringup;
865 	st->cb_state = state;
866 	st->single = true;
867 
868 	__cpuhp_kick_ap(st);
869 
870 	/*
871 	 * If we failed and did a partial, do a rollback.
872 	 */
873 	if ((ret = st->result) && st->last) {
874 		st->rollback = true;
875 		st->bringup = !bringup;
876 
877 		__cpuhp_kick_ap(st);
878 	}
879 
880 	/*
881 	 * Clean up the leftovers so the next hotplug operation wont use stale
882 	 * data.
883 	 */
884 	st->node = st->last = NULL;
885 	return ret;
886 }
887 
cpuhp_kick_ap_work(unsigned int cpu)888 static int cpuhp_kick_ap_work(unsigned int cpu)
889 {
890 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
891 	enum cpuhp_state prev_state = st->state;
892 	int ret;
893 
894 	cpuhp_lock_acquire(false);
895 	cpuhp_lock_release(false);
896 
897 	cpuhp_lock_acquire(true);
898 	cpuhp_lock_release(true);
899 
900 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
901 	ret = cpuhp_kick_ap(cpu, st, st->target);
902 	trace_cpuhp_exit(cpu, st->state, prev_state, ret);
903 
904 	return ret;
905 }
906 
907 static struct smp_hotplug_thread cpuhp_threads = {
908 	.store			= &cpuhp_state.thread,
909 	.thread_should_run	= cpuhp_should_run,
910 	.thread_fn		= cpuhp_thread_fun,
911 	.thread_comm		= "cpuhp/%u",
912 	.selfparking		= true,
913 };
914 
cpuhp_init_state(void)915 static __init void cpuhp_init_state(void)
916 {
917 	struct cpuhp_cpu_state *st;
918 	int cpu;
919 
920 	for_each_possible_cpu(cpu) {
921 		st = per_cpu_ptr(&cpuhp_state, cpu);
922 		init_completion(&st->done_up);
923 		init_completion(&st->done_down);
924 	}
925 }
926 
cpuhp_threads_init(void)927 void __init cpuhp_threads_init(void)
928 {
929 	cpuhp_init_state();
930 	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
931 	kthread_unpark(this_cpu_read(cpuhp_state.thread));
932 }
933 
934 /*
935  *
936  * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
937  * protected region.
938  *
939  * The operation is still serialized against concurrent CPU hotplug via
940  * cpu_add_remove_lock, i.e. CPU map protection.  But it is _not_
941  * serialized against other hotplug related activity like adding or
942  * removing of state callbacks and state instances, which invoke either the
943  * startup or the teardown callback of the affected state.
944  *
945  * This is required for subsystems which are unfixable vs. CPU hotplug and
946  * evade lock inversion problems by scheduling work which has to be
947  * completed _before_ cpu_up()/_cpu_down() returns.
948  *
949  * Don't even think about adding anything to this for any new code or even
950  * drivers. It's only purpose is to keep existing lock order trainwrecks
951  * working.
952  *
953  * For cpu_down() there might be valid reasons to finish cleanups which are
954  * not required to be done under cpu_hotplug_lock, but that's a different
955  * story and would be not invoked via this.
956  */
cpu_up_down_serialize_trainwrecks(bool tasks_frozen)957 static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
958 {
959 	/*
960 	 * cpusets delegate hotplug operations to a worker to "solve" the
961 	 * lock order problems. Wait for the worker, but only if tasks are
962 	 * _not_ frozen (suspend, hibernate) as that would wait forever.
963 	 *
964 	 * The wait is required because otherwise the hotplug operation
965 	 * returns with inconsistent state, which could even be observed in
966 	 * user space when a new CPU is brought up. The CPU plug uevent
967 	 * would be delivered and user space reacting on it would fail to
968 	 * move tasks to the newly plugged CPU up to the point where the
969 	 * work has finished because up to that point the newly plugged CPU
970 	 * is not assignable in cpusets/cgroups. On unplug that's not
971 	 * necessarily a visible issue, but it is still inconsistent state,
972 	 * which is the real problem which needs to be "fixed". This can't
973 	 * prevent the transient state between scheduling the work and
974 	 * returning from waiting for it.
975 	 */
976 	if (!tasks_frozen)
977 		cpuset_wait_for_hotplug();
978 }
979 
980 #ifdef CONFIG_HOTPLUG_CPU
981 #ifndef arch_clear_mm_cpumask_cpu
982 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
983 #endif
984 
985 /**
986  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
987  * @cpu: a CPU id
988  *
989  * This function walks all processes, finds a valid mm struct for each one and
990  * then clears a corresponding bit in mm's cpumask.  While this all sounds
991  * trivial, there are various non-obvious corner cases, which this function
992  * tries to solve in a safe manner.
993  *
994  * Also note that the function uses a somewhat relaxed locking scheme, so it may
995  * be called only for an already offlined CPU.
996  */
clear_tasks_mm_cpumask(int cpu)997 void clear_tasks_mm_cpumask(int cpu)
998 {
999 	struct task_struct *p;
1000 
1001 	/*
1002 	 * This function is called after the cpu is taken down and marked
1003 	 * offline, so its not like new tasks will ever get this cpu set in
1004 	 * their mm mask. -- Peter Zijlstra
1005 	 * Thus, we may use rcu_read_lock() here, instead of grabbing
1006 	 * full-fledged tasklist_lock.
1007 	 */
1008 	WARN_ON(cpu_online(cpu));
1009 	rcu_read_lock();
1010 	for_each_process(p) {
1011 		struct task_struct *t;
1012 
1013 		/*
1014 		 * Main thread might exit, but other threads may still have
1015 		 * a valid mm. Find one.
1016 		 */
1017 		t = find_lock_task_mm(p);
1018 		if (!t)
1019 			continue;
1020 		arch_clear_mm_cpumask_cpu(cpu, t->mm);
1021 		task_unlock(t);
1022 	}
1023 	rcu_read_unlock();
1024 }
1025 
1026 /* Take this CPU down. */
take_cpu_down(void * _param)1027 static int take_cpu_down(void *_param)
1028 {
1029 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1030 	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
1031 	int err, cpu = smp_processor_id();
1032 
1033 	/* Ensure this CPU doesn't handle any more interrupts. */
1034 	err = __cpu_disable();
1035 	if (err < 0)
1036 		return err;
1037 
1038 	/*
1039 	 * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
1040 	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
1041 	 */
1042 	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
1043 
1044 	/*
1045 	 * Invoke the former CPU_DYING callbacks. DYING must not fail!
1046 	 */
1047 	cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
1048 
1049 	/* Give up timekeeping duties */
1050 	tick_handover_do_timer();
1051 	/* Remove CPU from timer broadcasting */
1052 	tick_offline_cpu(cpu);
1053 	/* Park the stopper thread */
1054 	stop_machine_park(cpu);
1055 	return 0;
1056 }
1057 
takedown_cpu(unsigned int cpu)1058 static int takedown_cpu(unsigned int cpu)
1059 {
1060 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1061 	int err;
1062 
1063 	/* Park the smpboot threads */
1064 	kthread_park(st->thread);
1065 
1066 	/*
1067 	 * Prevent irq alloc/free while the dying cpu reorganizes the
1068 	 * interrupt affinities.
1069 	 */
1070 	irq_lock_sparse();
1071 
1072 	/*
1073 	 * So now all preempt/rcu users must observe !cpu_active().
1074 	 */
1075 	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
1076 	if (err) {
1077 		/* CPU refused to die */
1078 		irq_unlock_sparse();
1079 		/* Unpark the hotplug thread so we can rollback there */
1080 		kthread_unpark(st->thread);
1081 		return err;
1082 	}
1083 	BUG_ON(cpu_online(cpu));
1084 
1085 	/*
1086 	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
1087 	 * all runnable tasks from the CPU, there's only the idle task left now
1088 	 * that the migration thread is done doing the stop_machine thing.
1089 	 *
1090 	 * Wait for the stop thread to go away.
1091 	 */
1092 	wait_for_ap_thread(st, false);
1093 	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
1094 
1095 	/* Interrupts are moved away from the dying cpu, reenable alloc/free */
1096 	irq_unlock_sparse();
1097 
1098 	hotplug_cpu__broadcast_tick_pull(cpu);
1099 	/* This actually kills the CPU. */
1100 	__cpu_die(cpu);
1101 
1102 	tick_cleanup_dead_cpu(cpu);
1103 	rcutree_migrate_callbacks(cpu);
1104 	return 0;
1105 }
1106 
cpuhp_complete_idle_dead(void * arg)1107 static void cpuhp_complete_idle_dead(void *arg)
1108 {
1109 	struct cpuhp_cpu_state *st = arg;
1110 
1111 	complete_ap_thread(st, false);
1112 }
1113 
cpuhp_report_idle_dead(void)1114 void cpuhp_report_idle_dead(void)
1115 {
1116 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1117 
1118 	BUG_ON(st->state != CPUHP_AP_OFFLINE);
1119 	rcu_report_dead(smp_processor_id());
1120 	st->state = CPUHP_AP_IDLE_DEAD;
1121 	/*
1122 	 * We cannot call complete after rcu_report_dead() so we delegate it
1123 	 * to an online cpu.
1124 	 */
1125 	smp_call_function_single(cpumask_first(cpu_online_mask),
1126 				 cpuhp_complete_idle_dead, st, 0);
1127 }
1128 
cpuhp_down_callbacks(unsigned int cpu,struct cpuhp_cpu_state * st,enum cpuhp_state target)1129 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1130 				enum cpuhp_state target)
1131 {
1132 	enum cpuhp_state prev_state = st->state;
1133 	int ret = 0;
1134 
1135 	ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1136 	if (ret) {
1137 		pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
1138 			 ret, cpu, cpuhp_get_step(st->state)->name,
1139 			 st->state);
1140 
1141 		cpuhp_reset_state(cpu, st, prev_state);
1142 
1143 		if (st->state < prev_state)
1144 			WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
1145 							    prev_state));
1146 	}
1147 
1148 	return ret;
1149 }
1150 
1151 /* Requires cpu_add_remove_lock to be held */
_cpu_down(unsigned int cpu,int tasks_frozen,enum cpuhp_state target)1152 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1153 			   enum cpuhp_state target)
1154 {
1155 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1156 	int prev_state, ret = 0;
1157 
1158 	if (num_online_cpus() == 1)
1159 		return -EBUSY;
1160 
1161 	if (!cpu_present(cpu))
1162 		return -EINVAL;
1163 
1164 	cpus_write_lock();
1165 
1166 	cpuhp_tasks_frozen = tasks_frozen;
1167 
1168 	prev_state = cpuhp_set_state(cpu, st, target);
1169 	/*
1170 	 * If the current CPU state is in the range of the AP hotplug thread,
1171 	 * then we need to kick the thread.
1172 	 */
1173 	if (st->state > CPUHP_TEARDOWN_CPU) {
1174 		st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1175 		ret = cpuhp_kick_ap_work(cpu);
1176 		/*
1177 		 * The AP side has done the error rollback already. Just
1178 		 * return the error code..
1179 		 */
1180 		if (ret)
1181 			goto out;
1182 
1183 		/*
1184 		 * We might have stopped still in the range of the AP hotplug
1185 		 * thread. Nothing to do anymore.
1186 		 */
1187 		if (st->state > CPUHP_TEARDOWN_CPU)
1188 			goto out;
1189 
1190 		st->target = target;
1191 	}
1192 	/*
1193 	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1194 	 * to do the further cleanups.
1195 	 */
1196 	ret = cpuhp_down_callbacks(cpu, st, target);
1197 	if (ret && st->state < prev_state) {
1198 		if (st->state == CPUHP_TEARDOWN_CPU) {
1199 			cpuhp_reset_state(cpu, st, prev_state);
1200 			__cpuhp_kick_ap(st);
1201 		} else {
1202 			WARN(1, "DEAD callback error for CPU%d", cpu);
1203 		}
1204 	}
1205 
1206 out:
1207 	cpus_write_unlock();
1208 	/*
1209 	 * Do post unplug cleanup. This is still protected against
1210 	 * concurrent CPU hotplug via cpu_add_remove_lock.
1211 	 */
1212 	lockup_detector_cleanup();
1213 	arch_smt_update();
1214 	cpu_up_down_serialize_trainwrecks(tasks_frozen);
1215 	return ret;
1216 }
1217 
cpu_down_maps_locked(unsigned int cpu,enum cpuhp_state target)1218 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1219 {
1220 	/*
1221 	 * If the platform does not support hotplug, report it explicitly to
1222 	 * differentiate it from a transient offlining failure.
1223 	 */
1224 	if (cc_platform_has(CC_ATTR_HOTPLUG_DISABLED))
1225 		return -EOPNOTSUPP;
1226 	if (cpu_hotplug_disabled)
1227 		return -EBUSY;
1228 	return _cpu_down(cpu, 0, target);
1229 }
1230 
cpu_down(unsigned int cpu,enum cpuhp_state target)1231 static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1232 {
1233 	int err;
1234 
1235 	cpu_maps_update_begin();
1236 	err = cpu_down_maps_locked(cpu, target);
1237 	cpu_maps_update_done();
1238 	return err;
1239 }
1240 
1241 /**
1242  * cpu_device_down - Bring down a cpu device
1243  * @dev: Pointer to the cpu device to offline
1244  *
1245  * This function is meant to be used by device core cpu subsystem only.
1246  *
1247  * Other subsystems should use remove_cpu() instead.
1248  *
1249  * Return: %0 on success or a negative errno code
1250  */
cpu_device_down(struct device * dev)1251 int cpu_device_down(struct device *dev)
1252 {
1253 	return cpu_down(dev->id, CPUHP_OFFLINE);
1254 }
1255 
remove_cpu(unsigned int cpu)1256 int remove_cpu(unsigned int cpu)
1257 {
1258 	int ret;
1259 
1260 	lock_device_hotplug();
1261 	ret = device_offline(get_cpu_device(cpu));
1262 	unlock_device_hotplug();
1263 
1264 	return ret;
1265 }
1266 EXPORT_SYMBOL_GPL(remove_cpu);
1267 
smp_shutdown_nonboot_cpus(unsigned int primary_cpu)1268 void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1269 {
1270 	unsigned int cpu;
1271 	int error;
1272 
1273 	cpu_maps_update_begin();
1274 
1275 	/*
1276 	 * Make certain the cpu I'm about to reboot on is online.
1277 	 *
1278 	 * This is inline to what migrate_to_reboot_cpu() already do.
1279 	 */
1280 	if (!cpu_online(primary_cpu))
1281 		primary_cpu = cpumask_first(cpu_online_mask);
1282 
1283 	for_each_online_cpu(cpu) {
1284 		if (cpu == primary_cpu)
1285 			continue;
1286 
1287 		error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1288 		if (error) {
1289 			pr_err("Failed to offline CPU%d - error=%d",
1290 				cpu, error);
1291 			break;
1292 		}
1293 	}
1294 
1295 	/*
1296 	 * Ensure all but the reboot CPU are offline.
1297 	 */
1298 	BUG_ON(num_online_cpus() > 1);
1299 
1300 	/*
1301 	 * Make sure the CPUs won't be enabled by someone else after this
1302 	 * point. Kexec will reboot to a new kernel shortly resetting
1303 	 * everything along the way.
1304 	 */
1305 	cpu_hotplug_disabled++;
1306 
1307 	cpu_maps_update_done();
1308 }
1309 
1310 #else
1311 #define takedown_cpu		NULL
1312 #endif /*CONFIG_HOTPLUG_CPU*/
1313 
1314 /**
1315  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1316  * @cpu: cpu that just started
1317  *
1318  * It must be called by the arch code on the new cpu, before the new cpu
1319  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1320  */
notify_cpu_starting(unsigned int cpu)1321 void notify_cpu_starting(unsigned int cpu)
1322 {
1323 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1324 	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1325 
1326 	rcu_cpu_starting(cpu);	/* Enables RCU usage on this CPU. */
1327 	cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1328 
1329 	/*
1330 	 * STARTING must not fail!
1331 	 */
1332 	cpuhp_invoke_callback_range_nofail(true, cpu, st, target);
1333 }
1334 
1335 /*
1336  * Called from the idle task. Wake up the controlling task which brings the
1337  * hotplug thread of the upcoming CPU up and then delegates the rest of the
1338  * online bringup to the hotplug thread.
1339  */
cpuhp_online_idle(enum cpuhp_state state)1340 void cpuhp_online_idle(enum cpuhp_state state)
1341 {
1342 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1343 
1344 	/* Happens for the boot cpu */
1345 	if (state != CPUHP_AP_ONLINE_IDLE)
1346 		return;
1347 
1348 	/*
1349 	 * Unpart the stopper thread before we start the idle loop (and start
1350 	 * scheduling); this ensures the stopper task is always available.
1351 	 */
1352 	stop_machine_unpark(smp_processor_id());
1353 
1354 	st->state = CPUHP_AP_ONLINE_IDLE;
1355 	complete_ap_thread(st, true);
1356 }
1357 
1358 /* Requires cpu_add_remove_lock to be held */
_cpu_up(unsigned int cpu,int tasks_frozen,enum cpuhp_state target)1359 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1360 {
1361 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1362 	struct task_struct *idle;
1363 	int ret = 0;
1364 
1365 	cpus_write_lock();
1366 
1367 	if (!cpu_present(cpu)) {
1368 		ret = -EINVAL;
1369 		goto out;
1370 	}
1371 
1372 	/*
1373 	 * The caller of cpu_up() might have raced with another
1374 	 * caller. Nothing to do.
1375 	 */
1376 	if (st->state >= target)
1377 		goto out;
1378 
1379 	if (st->state == CPUHP_OFFLINE) {
1380 		/* Let it fail before we try to bring the cpu up */
1381 		idle = idle_thread_get(cpu);
1382 		if (IS_ERR(idle)) {
1383 			ret = PTR_ERR(idle);
1384 			goto out;
1385 		}
1386 	}
1387 
1388 	cpuhp_tasks_frozen = tasks_frozen;
1389 
1390 	cpuhp_set_state(cpu, st, target);
1391 	/*
1392 	 * If the current CPU state is in the range of the AP hotplug thread,
1393 	 * then we need to kick the thread once more.
1394 	 */
1395 	if (st->state > CPUHP_BRINGUP_CPU) {
1396 		ret = cpuhp_kick_ap_work(cpu);
1397 		/*
1398 		 * The AP side has done the error rollback already. Just
1399 		 * return the error code..
1400 		 */
1401 		if (ret)
1402 			goto out;
1403 	}
1404 
1405 	/*
1406 	 * Try to reach the target state. We max out on the BP at
1407 	 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1408 	 * responsible for bringing it up to the target state.
1409 	 */
1410 	target = min((int)target, CPUHP_BRINGUP_CPU);
1411 	ret = cpuhp_up_callbacks(cpu, st, target);
1412 out:
1413 	cpus_write_unlock();
1414 	arch_smt_update();
1415 	cpu_up_down_serialize_trainwrecks(tasks_frozen);
1416 	return ret;
1417 }
1418 
cpu_up(unsigned int cpu,enum cpuhp_state target)1419 static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1420 {
1421 	int err = 0;
1422 
1423 	if (!cpu_possible(cpu)) {
1424 		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1425 		       cpu);
1426 #if defined(CONFIG_IA64)
1427 		pr_err("please check additional_cpus= boot parameter\n");
1428 #endif
1429 		return -EINVAL;
1430 	}
1431 
1432 	err = try_online_node(cpu_to_node(cpu));
1433 	if (err)
1434 		return err;
1435 
1436 	cpu_maps_update_begin();
1437 
1438 	if (cpu_hotplug_disabled) {
1439 		err = -EBUSY;
1440 		goto out;
1441 	}
1442 	if (!cpu_smt_allowed(cpu)) {
1443 		err = -EPERM;
1444 		goto out;
1445 	}
1446 
1447 	err = _cpu_up(cpu, 0, target);
1448 out:
1449 	cpu_maps_update_done();
1450 	return err;
1451 }
1452 
1453 /**
1454  * cpu_device_up - Bring up a cpu device
1455  * @dev: Pointer to the cpu device to online
1456  *
1457  * This function is meant to be used by device core cpu subsystem only.
1458  *
1459  * Other subsystems should use add_cpu() instead.
1460  *
1461  * Return: %0 on success or a negative errno code
1462  */
cpu_device_up(struct device * dev)1463 int cpu_device_up(struct device *dev)
1464 {
1465 	return cpu_up(dev->id, CPUHP_ONLINE);
1466 }
1467 
add_cpu(unsigned int cpu)1468 int add_cpu(unsigned int cpu)
1469 {
1470 	int ret;
1471 
1472 	lock_device_hotplug();
1473 	ret = device_online(get_cpu_device(cpu));
1474 	unlock_device_hotplug();
1475 
1476 	return ret;
1477 }
1478 EXPORT_SYMBOL_GPL(add_cpu);
1479 
1480 /**
1481  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1482  * @sleep_cpu: The cpu we hibernated on and should be brought up.
1483  *
1484  * On some architectures like arm64, we can hibernate on any CPU, but on
1485  * wake up the CPU we hibernated on might be offline as a side effect of
1486  * using maxcpus= for example.
1487  *
1488  * Return: %0 on success or a negative errno code
1489  */
bringup_hibernate_cpu(unsigned int sleep_cpu)1490 int bringup_hibernate_cpu(unsigned int sleep_cpu)
1491 {
1492 	int ret;
1493 
1494 	if (!cpu_online(sleep_cpu)) {
1495 		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1496 		ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1497 		if (ret) {
1498 			pr_err("Failed to bring hibernate-CPU up!\n");
1499 			return ret;
1500 		}
1501 	}
1502 	return 0;
1503 }
1504 
bringup_nonboot_cpus(unsigned int setup_max_cpus)1505 void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1506 {
1507 	unsigned int cpu;
1508 
1509 	for_each_present_cpu(cpu) {
1510 		if (num_online_cpus() >= setup_max_cpus)
1511 			break;
1512 		if (!cpu_online(cpu))
1513 			cpu_up(cpu, CPUHP_ONLINE);
1514 	}
1515 }
1516 
1517 #ifdef CONFIG_PM_SLEEP_SMP
1518 static cpumask_var_t frozen_cpus;
1519 
freeze_secondary_cpus(int primary)1520 int freeze_secondary_cpus(int primary)
1521 {
1522 	int cpu, error = 0;
1523 
1524 	cpu_maps_update_begin();
1525 	if (primary == -1) {
1526 		primary = cpumask_first(cpu_online_mask);
1527 		if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
1528 			primary = housekeeping_any_cpu(HK_TYPE_TIMER);
1529 	} else {
1530 		if (!cpu_online(primary))
1531 			primary = cpumask_first(cpu_online_mask);
1532 	}
1533 
1534 	/*
1535 	 * We take down all of the non-boot CPUs in one shot to avoid races
1536 	 * with the userspace trying to use the CPU hotplug at the same time
1537 	 */
1538 	cpumask_clear(frozen_cpus);
1539 
1540 	pr_info("Disabling non-boot CPUs ...\n");
1541 	for_each_online_cpu(cpu) {
1542 		if (cpu == primary)
1543 			continue;
1544 
1545 		if (pm_wakeup_pending()) {
1546 			pr_info("Wakeup pending. Abort CPU freeze\n");
1547 			error = -EBUSY;
1548 			break;
1549 		}
1550 
1551 		trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1552 		error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1553 		trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1554 		if (!error)
1555 			cpumask_set_cpu(cpu, frozen_cpus);
1556 		else {
1557 			pr_err("Error taking CPU%d down: %d\n", cpu, error);
1558 			break;
1559 		}
1560 	}
1561 
1562 	if (!error)
1563 		BUG_ON(num_online_cpus() > 1);
1564 	else
1565 		pr_err("Non-boot CPUs are not disabled\n");
1566 
1567 	/*
1568 	 * Make sure the CPUs won't be enabled by someone else. We need to do
1569 	 * this even in case of failure as all freeze_secondary_cpus() users are
1570 	 * supposed to do thaw_secondary_cpus() on the failure path.
1571 	 */
1572 	cpu_hotplug_disabled++;
1573 
1574 	cpu_maps_update_done();
1575 	return error;
1576 }
1577 
arch_thaw_secondary_cpus_begin(void)1578 void __weak arch_thaw_secondary_cpus_begin(void)
1579 {
1580 }
1581 
arch_thaw_secondary_cpus_end(void)1582 void __weak arch_thaw_secondary_cpus_end(void)
1583 {
1584 }
1585 
thaw_secondary_cpus(void)1586 void thaw_secondary_cpus(void)
1587 {
1588 	int cpu, error;
1589 
1590 	/* Allow everyone to use the CPU hotplug again */
1591 	cpu_maps_update_begin();
1592 	__cpu_hotplug_enable();
1593 	if (cpumask_empty(frozen_cpus))
1594 		goto out;
1595 
1596 	pr_info("Enabling non-boot CPUs ...\n");
1597 
1598 	arch_thaw_secondary_cpus_begin();
1599 
1600 	for_each_cpu(cpu, frozen_cpus) {
1601 		trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1602 		error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1603 		trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1604 		if (!error) {
1605 			pr_info("CPU%d is up\n", cpu);
1606 			continue;
1607 		}
1608 		pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1609 	}
1610 
1611 	arch_thaw_secondary_cpus_end();
1612 
1613 	cpumask_clear(frozen_cpus);
1614 out:
1615 	cpu_maps_update_done();
1616 }
1617 
alloc_frozen_cpus(void)1618 static int __init alloc_frozen_cpus(void)
1619 {
1620 	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1621 		return -ENOMEM;
1622 	return 0;
1623 }
1624 core_initcall(alloc_frozen_cpus);
1625 
1626 /*
1627  * When callbacks for CPU hotplug notifications are being executed, we must
1628  * ensure that the state of the system with respect to the tasks being frozen
1629  * or not, as reported by the notification, remains unchanged *throughout the
1630  * duration* of the execution of the callbacks.
1631  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1632  *
1633  * This synchronization is implemented by mutually excluding regular CPU
1634  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1635  * Hibernate notifications.
1636  */
1637 static int
cpu_hotplug_pm_callback(struct notifier_block * nb,unsigned long action,void * ptr)1638 cpu_hotplug_pm_callback(struct notifier_block *nb,
1639 			unsigned long action, void *ptr)
1640 {
1641 	switch (action) {
1642 
1643 	case PM_SUSPEND_PREPARE:
1644 	case PM_HIBERNATION_PREPARE:
1645 		cpu_hotplug_disable();
1646 		break;
1647 
1648 	case PM_POST_SUSPEND:
1649 	case PM_POST_HIBERNATION:
1650 		cpu_hotplug_enable();
1651 		break;
1652 
1653 	default:
1654 		return NOTIFY_DONE;
1655 	}
1656 
1657 	return NOTIFY_OK;
1658 }
1659 
1660 
cpu_hotplug_pm_sync_init(void)1661 static int __init cpu_hotplug_pm_sync_init(void)
1662 {
1663 	/*
1664 	 * cpu_hotplug_pm_callback has higher priority than x86
1665 	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1666 	 * to disable cpu hotplug to avoid cpu hotplug race.
1667 	 */
1668 	pm_notifier(cpu_hotplug_pm_callback, 0);
1669 	return 0;
1670 }
1671 core_initcall(cpu_hotplug_pm_sync_init);
1672 
1673 #endif /* CONFIG_PM_SLEEP_SMP */
1674 
1675 int __boot_cpu_id;
1676 
1677 #endif /* CONFIG_SMP */
1678 
1679 /* Boot processor state steps */
1680 static struct cpuhp_step cpuhp_hp_states[] = {
1681 	[CPUHP_OFFLINE] = {
1682 		.name			= "offline",
1683 		.startup.single		= NULL,
1684 		.teardown.single	= NULL,
1685 	},
1686 #ifdef CONFIG_SMP
1687 	[CPUHP_CREATE_THREADS]= {
1688 		.name			= "threads:prepare",
1689 		.startup.single		= smpboot_create_threads,
1690 		.teardown.single	= NULL,
1691 		.cant_stop		= true,
1692 	},
1693 	[CPUHP_PERF_PREPARE] = {
1694 		.name			= "perf:prepare",
1695 		.startup.single		= perf_event_init_cpu,
1696 		.teardown.single	= perf_event_exit_cpu,
1697 	},
1698 	[CPUHP_RANDOM_PREPARE] = {
1699 		.name			= "random:prepare",
1700 		.startup.single		= random_prepare_cpu,
1701 		.teardown.single	= NULL,
1702 	},
1703 	[CPUHP_WORKQUEUE_PREP] = {
1704 		.name			= "workqueue:prepare",
1705 		.startup.single		= workqueue_prepare_cpu,
1706 		.teardown.single	= NULL,
1707 	},
1708 	[CPUHP_HRTIMERS_PREPARE] = {
1709 		.name			= "hrtimers:prepare",
1710 		.startup.single		= hrtimers_prepare_cpu,
1711 		.teardown.single	= hrtimers_dead_cpu,
1712 	},
1713 	[CPUHP_SMPCFD_PREPARE] = {
1714 		.name			= "smpcfd:prepare",
1715 		.startup.single		= smpcfd_prepare_cpu,
1716 		.teardown.single	= smpcfd_dead_cpu,
1717 	},
1718 	[CPUHP_RELAY_PREPARE] = {
1719 		.name			= "relay:prepare",
1720 		.startup.single		= relay_prepare_cpu,
1721 		.teardown.single	= NULL,
1722 	},
1723 	[CPUHP_SLAB_PREPARE] = {
1724 		.name			= "slab:prepare",
1725 		.startup.single		= slab_prepare_cpu,
1726 		.teardown.single	= slab_dead_cpu,
1727 	},
1728 	[CPUHP_RCUTREE_PREP] = {
1729 		.name			= "RCU/tree:prepare",
1730 		.startup.single		= rcutree_prepare_cpu,
1731 		.teardown.single	= rcutree_dead_cpu,
1732 	},
1733 	/*
1734 	 * On the tear-down path, timers_dead_cpu() must be invoked
1735 	 * before blk_mq_queue_reinit_notify() from notify_dead(),
1736 	 * otherwise a RCU stall occurs.
1737 	 */
1738 	[CPUHP_TIMERS_PREPARE] = {
1739 		.name			= "timers:prepare",
1740 		.startup.single		= timers_prepare_cpu,
1741 		.teardown.single	= timers_dead_cpu,
1742 	},
1743 	/* Kicks the plugged cpu into life */
1744 	[CPUHP_BRINGUP_CPU] = {
1745 		.name			= "cpu:bringup",
1746 		.startup.single		= bringup_cpu,
1747 		.teardown.single	= finish_cpu,
1748 		.cant_stop		= true,
1749 	},
1750 	/* Final state before CPU kills itself */
1751 	[CPUHP_AP_IDLE_DEAD] = {
1752 		.name			= "idle:dead",
1753 	},
1754 	/*
1755 	 * Last state before CPU enters the idle loop to die. Transient state
1756 	 * for synchronization.
1757 	 */
1758 	[CPUHP_AP_OFFLINE] = {
1759 		.name			= "ap:offline",
1760 		.cant_stop		= true,
1761 	},
1762 	/* First state is scheduler control. Interrupts are disabled */
1763 	[CPUHP_AP_SCHED_STARTING] = {
1764 		.name			= "sched:starting",
1765 		.startup.single		= sched_cpu_starting,
1766 		.teardown.single	= sched_cpu_dying,
1767 	},
1768 	[CPUHP_AP_RCUTREE_DYING] = {
1769 		.name			= "RCU/tree:dying",
1770 		.startup.single		= NULL,
1771 		.teardown.single	= rcutree_dying_cpu,
1772 	},
1773 	[CPUHP_AP_SMPCFD_DYING] = {
1774 		.name			= "smpcfd:dying",
1775 		.startup.single		= NULL,
1776 		.teardown.single	= smpcfd_dying_cpu,
1777 	},
1778 	/* Entry state on starting. Interrupts enabled from here on. Transient
1779 	 * state for synchronsization */
1780 	[CPUHP_AP_ONLINE] = {
1781 		.name			= "ap:online",
1782 	},
1783 	/*
1784 	 * Handled on control processor until the plugged processor manages
1785 	 * this itself.
1786 	 */
1787 	[CPUHP_TEARDOWN_CPU] = {
1788 		.name			= "cpu:teardown",
1789 		.startup.single		= NULL,
1790 		.teardown.single	= takedown_cpu,
1791 		.cant_stop		= true,
1792 	},
1793 
1794 	[CPUHP_AP_SCHED_WAIT_EMPTY] = {
1795 		.name			= "sched:waitempty",
1796 		.startup.single		= NULL,
1797 		.teardown.single	= sched_cpu_wait_empty,
1798 	},
1799 
1800 	/* Handle smpboot threads park/unpark */
1801 	[CPUHP_AP_SMPBOOT_THREADS] = {
1802 		.name			= "smpboot/threads:online",
1803 		.startup.single		= smpboot_unpark_threads,
1804 		.teardown.single	= smpboot_park_threads,
1805 	},
1806 	[CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1807 		.name			= "irq/affinity:online",
1808 		.startup.single		= irq_affinity_online_cpu,
1809 		.teardown.single	= NULL,
1810 	},
1811 	[CPUHP_AP_PERF_ONLINE] = {
1812 		.name			= "perf:online",
1813 		.startup.single		= perf_event_init_cpu,
1814 		.teardown.single	= perf_event_exit_cpu,
1815 	},
1816 	[CPUHP_AP_WATCHDOG_ONLINE] = {
1817 		.name			= "lockup_detector:online",
1818 		.startup.single		= lockup_detector_online_cpu,
1819 		.teardown.single	= lockup_detector_offline_cpu,
1820 	},
1821 	[CPUHP_AP_WORKQUEUE_ONLINE] = {
1822 		.name			= "workqueue:online",
1823 		.startup.single		= workqueue_online_cpu,
1824 		.teardown.single	= workqueue_offline_cpu,
1825 	},
1826 	[CPUHP_AP_RANDOM_ONLINE] = {
1827 		.name			= "random:online",
1828 		.startup.single		= random_online_cpu,
1829 		.teardown.single	= NULL,
1830 	},
1831 	[CPUHP_AP_RCUTREE_ONLINE] = {
1832 		.name			= "RCU/tree:online",
1833 		.startup.single		= rcutree_online_cpu,
1834 		.teardown.single	= rcutree_offline_cpu,
1835 	},
1836 #endif
1837 	/*
1838 	 * The dynamically registered state space is here
1839 	 */
1840 
1841 #ifdef CONFIG_SMP
1842 	/* Last state is scheduler control setting the cpu active */
1843 	[CPUHP_AP_ACTIVE] = {
1844 		.name			= "sched:active",
1845 		.startup.single		= sched_cpu_activate,
1846 		.teardown.single	= sched_cpu_deactivate,
1847 	},
1848 #endif
1849 
1850 	/* CPU is fully up and running. */
1851 	[CPUHP_ONLINE] = {
1852 		.name			= "online",
1853 		.startup.single		= NULL,
1854 		.teardown.single	= NULL,
1855 	},
1856 };
1857 
1858 /* Sanity check for callbacks */
cpuhp_cb_check(enum cpuhp_state state)1859 static int cpuhp_cb_check(enum cpuhp_state state)
1860 {
1861 	if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1862 		return -EINVAL;
1863 	return 0;
1864 }
1865 
1866 /*
1867  * Returns a free for dynamic slot assignment of the Online state. The states
1868  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1869  * by having no name assigned.
1870  */
cpuhp_reserve_state(enum cpuhp_state state)1871 static int cpuhp_reserve_state(enum cpuhp_state state)
1872 {
1873 	enum cpuhp_state i, end;
1874 	struct cpuhp_step *step;
1875 
1876 	switch (state) {
1877 	case CPUHP_AP_ONLINE_DYN:
1878 		step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1879 		end = CPUHP_AP_ONLINE_DYN_END;
1880 		break;
1881 	case CPUHP_BP_PREPARE_DYN:
1882 		step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1883 		end = CPUHP_BP_PREPARE_DYN_END;
1884 		break;
1885 	default:
1886 		return -EINVAL;
1887 	}
1888 
1889 	for (i = state; i <= end; i++, step++) {
1890 		if (!step->name)
1891 			return i;
1892 	}
1893 	WARN(1, "No more dynamic states available for CPU hotplug\n");
1894 	return -ENOSPC;
1895 }
1896 
cpuhp_store_callbacks(enum cpuhp_state state,const char * name,int (* startup)(unsigned int cpu),int (* teardown)(unsigned int cpu),bool multi_instance)1897 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1898 				 int (*startup)(unsigned int cpu),
1899 				 int (*teardown)(unsigned int cpu),
1900 				 bool multi_instance)
1901 {
1902 	/* (Un)Install the callbacks for further cpu hotplug operations */
1903 	struct cpuhp_step *sp;
1904 	int ret = 0;
1905 
1906 	/*
1907 	 * If name is NULL, then the state gets removed.
1908 	 *
1909 	 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1910 	 * the first allocation from these dynamic ranges, so the removal
1911 	 * would trigger a new allocation and clear the wrong (already
1912 	 * empty) state, leaving the callbacks of the to be cleared state
1913 	 * dangling, which causes wreckage on the next hotplug operation.
1914 	 */
1915 	if (name && (state == CPUHP_AP_ONLINE_DYN ||
1916 		     state == CPUHP_BP_PREPARE_DYN)) {
1917 		ret = cpuhp_reserve_state(state);
1918 		if (ret < 0)
1919 			return ret;
1920 		state = ret;
1921 	}
1922 	sp = cpuhp_get_step(state);
1923 	if (name && sp->name)
1924 		return -EBUSY;
1925 
1926 	sp->startup.single = startup;
1927 	sp->teardown.single = teardown;
1928 	sp->name = name;
1929 	sp->multi_instance = multi_instance;
1930 	INIT_HLIST_HEAD(&sp->list);
1931 	return ret;
1932 }
1933 
cpuhp_get_teardown_cb(enum cpuhp_state state)1934 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1935 {
1936 	return cpuhp_get_step(state)->teardown.single;
1937 }
1938 
1939 /*
1940  * Call the startup/teardown function for a step either on the AP or
1941  * on the current CPU.
1942  */
cpuhp_issue_call(int cpu,enum cpuhp_state state,bool bringup,struct hlist_node * node)1943 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1944 			    struct hlist_node *node)
1945 {
1946 	struct cpuhp_step *sp = cpuhp_get_step(state);
1947 	int ret;
1948 
1949 	/*
1950 	 * If there's nothing to do, we done.
1951 	 * Relies on the union for multi_instance.
1952 	 */
1953 	if (cpuhp_step_empty(bringup, sp))
1954 		return 0;
1955 	/*
1956 	 * The non AP bound callbacks can fail on bringup. On teardown
1957 	 * e.g. module removal we crash for now.
1958 	 */
1959 #ifdef CONFIG_SMP
1960 	if (cpuhp_is_ap_state(state))
1961 		ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1962 	else
1963 		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1964 #else
1965 	ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1966 #endif
1967 	BUG_ON(ret && !bringup);
1968 	return ret;
1969 }
1970 
1971 /*
1972  * Called from __cpuhp_setup_state on a recoverable failure.
1973  *
1974  * Note: The teardown callbacks for rollback are not allowed to fail!
1975  */
cpuhp_rollback_install(int failedcpu,enum cpuhp_state state,struct hlist_node * node)1976 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1977 				   struct hlist_node *node)
1978 {
1979 	int cpu;
1980 
1981 	/* Roll back the already executed steps on the other cpus */
1982 	for_each_present_cpu(cpu) {
1983 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1984 		int cpustate = st->state;
1985 
1986 		if (cpu >= failedcpu)
1987 			break;
1988 
1989 		/* Did we invoke the startup call on that cpu ? */
1990 		if (cpustate >= state)
1991 			cpuhp_issue_call(cpu, state, false, node);
1992 	}
1993 }
1994 
__cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,struct hlist_node * node,bool invoke)1995 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1996 					  struct hlist_node *node,
1997 					  bool invoke)
1998 {
1999 	struct cpuhp_step *sp;
2000 	int cpu;
2001 	int ret;
2002 
2003 	lockdep_assert_cpus_held();
2004 
2005 	sp = cpuhp_get_step(state);
2006 	if (sp->multi_instance == false)
2007 		return -EINVAL;
2008 
2009 	mutex_lock(&cpuhp_state_mutex);
2010 
2011 	if (!invoke || !sp->startup.multi)
2012 		goto add_node;
2013 
2014 	/*
2015 	 * Try to call the startup callback for each present cpu
2016 	 * depending on the hotplug state of the cpu.
2017 	 */
2018 	for_each_present_cpu(cpu) {
2019 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2020 		int cpustate = st->state;
2021 
2022 		if (cpustate < state)
2023 			continue;
2024 
2025 		ret = cpuhp_issue_call(cpu, state, true, node);
2026 		if (ret) {
2027 			if (sp->teardown.multi)
2028 				cpuhp_rollback_install(cpu, state, node);
2029 			goto unlock;
2030 		}
2031 	}
2032 add_node:
2033 	ret = 0;
2034 	hlist_add_head(node, &sp->list);
2035 unlock:
2036 	mutex_unlock(&cpuhp_state_mutex);
2037 	return ret;
2038 }
2039 
__cpuhp_state_add_instance(enum cpuhp_state state,struct hlist_node * node,bool invoke)2040 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
2041 			       bool invoke)
2042 {
2043 	int ret;
2044 
2045 	cpus_read_lock();
2046 	ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
2047 	cpus_read_unlock();
2048 	return ret;
2049 }
2050 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
2051 
2052 /**
2053  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2054  * @state:		The state to setup
2055  * @name:		Name of the step
2056  * @invoke:		If true, the startup function is invoked for cpus where
2057  *			cpu state >= @state
2058  * @startup:		startup callback function
2059  * @teardown:		teardown callback function
2060  * @multi_instance:	State is set up for multiple instances which get
2061  *			added afterwards.
2062  *
2063  * The caller needs to hold cpus read locked while calling this function.
2064  * Return:
2065  *   On success:
2066  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN;
2067  *      0 for all other states
2068  *   On failure: proper (negative) error code
2069  */
__cpuhp_setup_state_cpuslocked(enum cpuhp_state state,const char * name,bool invoke,int (* startup)(unsigned int cpu),int (* teardown)(unsigned int cpu),bool multi_instance)2070 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
2071 				   const char *name, bool invoke,
2072 				   int (*startup)(unsigned int cpu),
2073 				   int (*teardown)(unsigned int cpu),
2074 				   bool multi_instance)
2075 {
2076 	int cpu, ret = 0;
2077 	bool dynstate;
2078 
2079 	lockdep_assert_cpus_held();
2080 
2081 	if (cpuhp_cb_check(state) || !name)
2082 		return -EINVAL;
2083 
2084 	mutex_lock(&cpuhp_state_mutex);
2085 
2086 	ret = cpuhp_store_callbacks(state, name, startup, teardown,
2087 				    multi_instance);
2088 
2089 	dynstate = state == CPUHP_AP_ONLINE_DYN;
2090 	if (ret > 0 && dynstate) {
2091 		state = ret;
2092 		ret = 0;
2093 	}
2094 
2095 	if (ret || !invoke || !startup)
2096 		goto out;
2097 
2098 	/*
2099 	 * Try to call the startup callback for each present cpu
2100 	 * depending on the hotplug state of the cpu.
2101 	 */
2102 	for_each_present_cpu(cpu) {
2103 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2104 		int cpustate = st->state;
2105 
2106 		if (cpustate < state)
2107 			continue;
2108 
2109 		ret = cpuhp_issue_call(cpu, state, true, NULL);
2110 		if (ret) {
2111 			if (teardown)
2112 				cpuhp_rollback_install(cpu, state, NULL);
2113 			cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2114 			goto out;
2115 		}
2116 	}
2117 out:
2118 	mutex_unlock(&cpuhp_state_mutex);
2119 	/*
2120 	 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
2121 	 * dynamically allocated state in case of success.
2122 	 */
2123 	if (!ret && dynstate)
2124 		return state;
2125 	return ret;
2126 }
2127 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
2128 
__cpuhp_setup_state(enum cpuhp_state state,const char * name,bool invoke,int (* startup)(unsigned int cpu),int (* teardown)(unsigned int cpu),bool multi_instance)2129 int __cpuhp_setup_state(enum cpuhp_state state,
2130 			const char *name, bool invoke,
2131 			int (*startup)(unsigned int cpu),
2132 			int (*teardown)(unsigned int cpu),
2133 			bool multi_instance)
2134 {
2135 	int ret;
2136 
2137 	cpus_read_lock();
2138 	ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
2139 					     teardown, multi_instance);
2140 	cpus_read_unlock();
2141 	return ret;
2142 }
2143 EXPORT_SYMBOL(__cpuhp_setup_state);
2144 
__cpuhp_state_remove_instance(enum cpuhp_state state,struct hlist_node * node,bool invoke)2145 int __cpuhp_state_remove_instance(enum cpuhp_state state,
2146 				  struct hlist_node *node, bool invoke)
2147 {
2148 	struct cpuhp_step *sp = cpuhp_get_step(state);
2149 	int cpu;
2150 
2151 	BUG_ON(cpuhp_cb_check(state));
2152 
2153 	if (!sp->multi_instance)
2154 		return -EINVAL;
2155 
2156 	cpus_read_lock();
2157 	mutex_lock(&cpuhp_state_mutex);
2158 
2159 	if (!invoke || !cpuhp_get_teardown_cb(state))
2160 		goto remove;
2161 	/*
2162 	 * Call the teardown callback for each present cpu depending
2163 	 * on the hotplug state of the cpu. This function is not
2164 	 * allowed to fail currently!
2165 	 */
2166 	for_each_present_cpu(cpu) {
2167 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2168 		int cpustate = st->state;
2169 
2170 		if (cpustate >= state)
2171 			cpuhp_issue_call(cpu, state, false, node);
2172 	}
2173 
2174 remove:
2175 	hlist_del(node);
2176 	mutex_unlock(&cpuhp_state_mutex);
2177 	cpus_read_unlock();
2178 
2179 	return 0;
2180 }
2181 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2182 
2183 /**
2184  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2185  * @state:	The state to remove
2186  * @invoke:	If true, the teardown function is invoked for cpus where
2187  *		cpu state >= @state
2188  *
2189  * The caller needs to hold cpus read locked while calling this function.
2190  * The teardown callback is currently not allowed to fail. Think
2191  * about module removal!
2192  */
__cpuhp_remove_state_cpuslocked(enum cpuhp_state state,bool invoke)2193 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2194 {
2195 	struct cpuhp_step *sp = cpuhp_get_step(state);
2196 	int cpu;
2197 
2198 	BUG_ON(cpuhp_cb_check(state));
2199 
2200 	lockdep_assert_cpus_held();
2201 
2202 	mutex_lock(&cpuhp_state_mutex);
2203 	if (sp->multi_instance) {
2204 		WARN(!hlist_empty(&sp->list),
2205 		     "Error: Removing state %d which has instances left.\n",
2206 		     state);
2207 		goto remove;
2208 	}
2209 
2210 	if (!invoke || !cpuhp_get_teardown_cb(state))
2211 		goto remove;
2212 
2213 	/*
2214 	 * Call the teardown callback for each present cpu depending
2215 	 * on the hotplug state of the cpu. This function is not
2216 	 * allowed to fail currently!
2217 	 */
2218 	for_each_present_cpu(cpu) {
2219 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2220 		int cpustate = st->state;
2221 
2222 		if (cpustate >= state)
2223 			cpuhp_issue_call(cpu, state, false, NULL);
2224 	}
2225 remove:
2226 	cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2227 	mutex_unlock(&cpuhp_state_mutex);
2228 }
2229 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2230 
__cpuhp_remove_state(enum cpuhp_state state,bool invoke)2231 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2232 {
2233 	cpus_read_lock();
2234 	__cpuhp_remove_state_cpuslocked(state, invoke);
2235 	cpus_read_unlock();
2236 }
2237 EXPORT_SYMBOL(__cpuhp_remove_state);
2238 
2239 #ifdef CONFIG_HOTPLUG_SMT
cpuhp_offline_cpu_device(unsigned int cpu)2240 static void cpuhp_offline_cpu_device(unsigned int cpu)
2241 {
2242 	struct device *dev = get_cpu_device(cpu);
2243 
2244 	dev->offline = true;
2245 	/* Tell user space about the state change */
2246 	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2247 }
2248 
cpuhp_online_cpu_device(unsigned int cpu)2249 static void cpuhp_online_cpu_device(unsigned int cpu)
2250 {
2251 	struct device *dev = get_cpu_device(cpu);
2252 
2253 	dev->offline = false;
2254 	/* Tell user space about the state change */
2255 	kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2256 }
2257 
cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)2258 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2259 {
2260 	int cpu, ret = 0;
2261 
2262 	cpu_maps_update_begin();
2263 	for_each_online_cpu(cpu) {
2264 		if (topology_is_primary_thread(cpu))
2265 			continue;
2266 		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2267 		if (ret)
2268 			break;
2269 		/*
2270 		 * As this needs to hold the cpu maps lock it's impossible
2271 		 * to call device_offline() because that ends up calling
2272 		 * cpu_down() which takes cpu maps lock. cpu maps lock
2273 		 * needs to be held as this might race against in kernel
2274 		 * abusers of the hotplug machinery (thermal management).
2275 		 *
2276 		 * So nothing would update device:offline state. That would
2277 		 * leave the sysfs entry stale and prevent onlining after
2278 		 * smt control has been changed to 'off' again. This is
2279 		 * called under the sysfs hotplug lock, so it is properly
2280 		 * serialized against the regular offline usage.
2281 		 */
2282 		cpuhp_offline_cpu_device(cpu);
2283 	}
2284 	if (!ret)
2285 		cpu_smt_control = ctrlval;
2286 	cpu_maps_update_done();
2287 	return ret;
2288 }
2289 
cpuhp_smt_enable(void)2290 int cpuhp_smt_enable(void)
2291 {
2292 	int cpu, ret = 0;
2293 
2294 	cpu_maps_update_begin();
2295 	cpu_smt_control = CPU_SMT_ENABLED;
2296 	for_each_present_cpu(cpu) {
2297 		/* Skip online CPUs and CPUs on offline nodes */
2298 		if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2299 			continue;
2300 		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2301 		if (ret)
2302 			break;
2303 		/* See comment in cpuhp_smt_disable() */
2304 		cpuhp_online_cpu_device(cpu);
2305 	}
2306 	cpu_maps_update_done();
2307 	return ret;
2308 }
2309 #endif
2310 
2311 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
state_show(struct device * dev,struct device_attribute * attr,char * buf)2312 static ssize_t state_show(struct device *dev,
2313 			  struct device_attribute *attr, char *buf)
2314 {
2315 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2316 
2317 	return sprintf(buf, "%d\n", st->state);
2318 }
2319 static DEVICE_ATTR_RO(state);
2320 
target_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2321 static ssize_t target_store(struct device *dev, struct device_attribute *attr,
2322 			    const char *buf, size_t count)
2323 {
2324 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2325 	struct cpuhp_step *sp;
2326 	int target, ret;
2327 
2328 	ret = kstrtoint(buf, 10, &target);
2329 	if (ret)
2330 		return ret;
2331 
2332 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2333 	if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2334 		return -EINVAL;
2335 #else
2336 	if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2337 		return -EINVAL;
2338 #endif
2339 
2340 	ret = lock_device_hotplug_sysfs();
2341 	if (ret)
2342 		return ret;
2343 
2344 	mutex_lock(&cpuhp_state_mutex);
2345 	sp = cpuhp_get_step(target);
2346 	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2347 	mutex_unlock(&cpuhp_state_mutex);
2348 	if (ret)
2349 		goto out;
2350 
2351 	if (st->state < target)
2352 		ret = cpu_up(dev->id, target);
2353 	else if (st->state > target)
2354 		ret = cpu_down(dev->id, target);
2355 	else if (WARN_ON(st->target != target))
2356 		st->target = target;
2357 out:
2358 	unlock_device_hotplug();
2359 	return ret ? ret : count;
2360 }
2361 
target_show(struct device * dev,struct device_attribute * attr,char * buf)2362 static ssize_t target_show(struct device *dev,
2363 			   struct device_attribute *attr, char *buf)
2364 {
2365 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2366 
2367 	return sprintf(buf, "%d\n", st->target);
2368 }
2369 static DEVICE_ATTR_RW(target);
2370 
fail_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2371 static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
2372 			  const char *buf, size_t count)
2373 {
2374 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2375 	struct cpuhp_step *sp;
2376 	int fail, ret;
2377 
2378 	ret = kstrtoint(buf, 10, &fail);
2379 	if (ret)
2380 		return ret;
2381 
2382 	if (fail == CPUHP_INVALID) {
2383 		st->fail = fail;
2384 		return count;
2385 	}
2386 
2387 	if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2388 		return -EINVAL;
2389 
2390 	/*
2391 	 * Cannot fail STARTING/DYING callbacks.
2392 	 */
2393 	if (cpuhp_is_atomic_state(fail))
2394 		return -EINVAL;
2395 
2396 	/*
2397 	 * DEAD callbacks cannot fail...
2398 	 * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
2399 	 * triggering STARTING callbacks, a failure in this state would
2400 	 * hinder rollback.
2401 	 */
2402 	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
2403 		return -EINVAL;
2404 
2405 	/*
2406 	 * Cannot fail anything that doesn't have callbacks.
2407 	 */
2408 	mutex_lock(&cpuhp_state_mutex);
2409 	sp = cpuhp_get_step(fail);
2410 	if (!sp->startup.single && !sp->teardown.single)
2411 		ret = -EINVAL;
2412 	mutex_unlock(&cpuhp_state_mutex);
2413 	if (ret)
2414 		return ret;
2415 
2416 	st->fail = fail;
2417 
2418 	return count;
2419 }
2420 
fail_show(struct device * dev,struct device_attribute * attr,char * buf)2421 static ssize_t fail_show(struct device *dev,
2422 			 struct device_attribute *attr, char *buf)
2423 {
2424 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2425 
2426 	return sprintf(buf, "%d\n", st->fail);
2427 }
2428 
2429 static DEVICE_ATTR_RW(fail);
2430 
2431 static struct attribute *cpuhp_cpu_attrs[] = {
2432 	&dev_attr_state.attr,
2433 	&dev_attr_target.attr,
2434 	&dev_attr_fail.attr,
2435 	NULL
2436 };
2437 
2438 static const struct attribute_group cpuhp_cpu_attr_group = {
2439 	.attrs = cpuhp_cpu_attrs,
2440 	.name = "hotplug",
2441 	NULL
2442 };
2443 
states_show(struct device * dev,struct device_attribute * attr,char * buf)2444 static ssize_t states_show(struct device *dev,
2445 				 struct device_attribute *attr, char *buf)
2446 {
2447 	ssize_t cur, res = 0;
2448 	int i;
2449 
2450 	mutex_lock(&cpuhp_state_mutex);
2451 	for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2452 		struct cpuhp_step *sp = cpuhp_get_step(i);
2453 
2454 		if (sp->name) {
2455 			cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2456 			buf += cur;
2457 			res += cur;
2458 		}
2459 	}
2460 	mutex_unlock(&cpuhp_state_mutex);
2461 	return res;
2462 }
2463 static DEVICE_ATTR_RO(states);
2464 
2465 static struct attribute *cpuhp_cpu_root_attrs[] = {
2466 	&dev_attr_states.attr,
2467 	NULL
2468 };
2469 
2470 static const struct attribute_group cpuhp_cpu_root_attr_group = {
2471 	.attrs = cpuhp_cpu_root_attrs,
2472 	.name = "hotplug",
2473 	NULL
2474 };
2475 
2476 #ifdef CONFIG_HOTPLUG_SMT
2477 
2478 static ssize_t
__store_smt_control(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2479 __store_smt_control(struct device *dev, struct device_attribute *attr,
2480 		    const char *buf, size_t count)
2481 {
2482 	int ctrlval, ret;
2483 
2484 	if (sysfs_streq(buf, "on"))
2485 		ctrlval = CPU_SMT_ENABLED;
2486 	else if (sysfs_streq(buf, "off"))
2487 		ctrlval = CPU_SMT_DISABLED;
2488 	else if (sysfs_streq(buf, "forceoff"))
2489 		ctrlval = CPU_SMT_FORCE_DISABLED;
2490 	else
2491 		return -EINVAL;
2492 
2493 	if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2494 		return -EPERM;
2495 
2496 	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2497 		return -ENODEV;
2498 
2499 	ret = lock_device_hotplug_sysfs();
2500 	if (ret)
2501 		return ret;
2502 
2503 	if (ctrlval != cpu_smt_control) {
2504 		switch (ctrlval) {
2505 		case CPU_SMT_ENABLED:
2506 			ret = cpuhp_smt_enable();
2507 			break;
2508 		case CPU_SMT_DISABLED:
2509 		case CPU_SMT_FORCE_DISABLED:
2510 			ret = cpuhp_smt_disable(ctrlval);
2511 			break;
2512 		}
2513 	}
2514 
2515 	unlock_device_hotplug();
2516 	return ret ? ret : count;
2517 }
2518 
2519 #else /* !CONFIG_HOTPLUG_SMT */
2520 static ssize_t
__store_smt_control(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2521 __store_smt_control(struct device *dev, struct device_attribute *attr,
2522 		    const char *buf, size_t count)
2523 {
2524 	return -ENODEV;
2525 }
2526 #endif /* CONFIG_HOTPLUG_SMT */
2527 
2528 static const char *smt_states[] = {
2529 	[CPU_SMT_ENABLED]		= "on",
2530 	[CPU_SMT_DISABLED]		= "off",
2531 	[CPU_SMT_FORCE_DISABLED]	= "forceoff",
2532 	[CPU_SMT_NOT_SUPPORTED]		= "notsupported",
2533 	[CPU_SMT_NOT_IMPLEMENTED]	= "notimplemented",
2534 };
2535 
control_show(struct device * dev,struct device_attribute * attr,char * buf)2536 static ssize_t control_show(struct device *dev,
2537 			    struct device_attribute *attr, char *buf)
2538 {
2539 	const char *state = smt_states[cpu_smt_control];
2540 
2541 	return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2542 }
2543 
control_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2544 static ssize_t control_store(struct device *dev, struct device_attribute *attr,
2545 			     const char *buf, size_t count)
2546 {
2547 	return __store_smt_control(dev, attr, buf, count);
2548 }
2549 static DEVICE_ATTR_RW(control);
2550 
active_show(struct device * dev,struct device_attribute * attr,char * buf)2551 static ssize_t active_show(struct device *dev,
2552 			   struct device_attribute *attr, char *buf)
2553 {
2554 	return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2555 }
2556 static DEVICE_ATTR_RO(active);
2557 
2558 static struct attribute *cpuhp_smt_attrs[] = {
2559 	&dev_attr_control.attr,
2560 	&dev_attr_active.attr,
2561 	NULL
2562 };
2563 
2564 static const struct attribute_group cpuhp_smt_attr_group = {
2565 	.attrs = cpuhp_smt_attrs,
2566 	.name = "smt",
2567 	NULL
2568 };
2569 
cpu_smt_sysfs_init(void)2570 static int __init cpu_smt_sysfs_init(void)
2571 {
2572 	return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2573 				  &cpuhp_smt_attr_group);
2574 }
2575 
cpuhp_sysfs_init(void)2576 static int __init cpuhp_sysfs_init(void)
2577 {
2578 	int cpu, ret;
2579 
2580 	ret = cpu_smt_sysfs_init();
2581 	if (ret)
2582 		return ret;
2583 
2584 	ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2585 				 &cpuhp_cpu_root_attr_group);
2586 	if (ret)
2587 		return ret;
2588 
2589 	for_each_possible_cpu(cpu) {
2590 		struct device *dev = get_cpu_device(cpu);
2591 
2592 		if (!dev)
2593 			continue;
2594 		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2595 		if (ret)
2596 			return ret;
2597 	}
2598 	return 0;
2599 }
2600 device_initcall(cpuhp_sysfs_init);
2601 #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2602 
2603 /*
2604  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2605  * represents all NR_CPUS bits binary values of 1<<nr.
2606  *
2607  * It is used by cpumask_of() to get a constant address to a CPU
2608  * mask value that has a single bit set only.
2609  */
2610 
2611 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2612 #define MASK_DECLARE_1(x)	[x+1][0] = (1UL << (x))
2613 #define MASK_DECLARE_2(x)	MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2614 #define MASK_DECLARE_4(x)	MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2615 #define MASK_DECLARE_8(x)	MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2616 
2617 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2618 
2619 	MASK_DECLARE_8(0),	MASK_DECLARE_8(8),
2620 	MASK_DECLARE_8(16),	MASK_DECLARE_8(24),
2621 #if BITS_PER_LONG > 32
2622 	MASK_DECLARE_8(32),	MASK_DECLARE_8(40),
2623 	MASK_DECLARE_8(48),	MASK_DECLARE_8(56),
2624 #endif
2625 };
2626 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2627 
2628 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2629 EXPORT_SYMBOL(cpu_all_bits);
2630 
2631 #ifdef CONFIG_INIT_ALL_POSSIBLE
2632 struct cpumask __cpu_possible_mask __read_mostly
2633 	= {CPU_BITS_ALL};
2634 #else
2635 struct cpumask __cpu_possible_mask __read_mostly;
2636 #endif
2637 EXPORT_SYMBOL(__cpu_possible_mask);
2638 
2639 struct cpumask __cpu_online_mask __read_mostly;
2640 EXPORT_SYMBOL(__cpu_online_mask);
2641 
2642 struct cpumask __cpu_present_mask __read_mostly;
2643 EXPORT_SYMBOL(__cpu_present_mask);
2644 
2645 struct cpumask __cpu_active_mask __read_mostly;
2646 EXPORT_SYMBOL(__cpu_active_mask);
2647 
2648 struct cpumask __cpu_dying_mask __read_mostly;
2649 EXPORT_SYMBOL(__cpu_dying_mask);
2650 
2651 atomic_t __num_online_cpus __read_mostly;
2652 EXPORT_SYMBOL(__num_online_cpus);
2653 
init_cpu_present(const struct cpumask * src)2654 void init_cpu_present(const struct cpumask *src)
2655 {
2656 	cpumask_copy(&__cpu_present_mask, src);
2657 }
2658 
init_cpu_possible(const struct cpumask * src)2659 void init_cpu_possible(const struct cpumask *src)
2660 {
2661 	cpumask_copy(&__cpu_possible_mask, src);
2662 }
2663 
init_cpu_online(const struct cpumask * src)2664 void init_cpu_online(const struct cpumask *src)
2665 {
2666 	cpumask_copy(&__cpu_online_mask, src);
2667 }
2668 
set_cpu_online(unsigned int cpu,bool online)2669 void set_cpu_online(unsigned int cpu, bool online)
2670 {
2671 	/*
2672 	 * atomic_inc/dec() is required to handle the horrid abuse of this
2673 	 * function by the reboot and kexec code which invoke it from
2674 	 * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2675 	 * regular CPU hotplug is properly serialized.
2676 	 *
2677 	 * Note, that the fact that __num_online_cpus is of type atomic_t
2678 	 * does not protect readers which are not serialized against
2679 	 * concurrent hotplug operations.
2680 	 */
2681 	if (online) {
2682 		if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2683 			atomic_inc(&__num_online_cpus);
2684 	} else {
2685 		if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2686 			atomic_dec(&__num_online_cpus);
2687 	}
2688 }
2689 
2690 /*
2691  * Activate the first processor.
2692  */
boot_cpu_init(void)2693 void __init boot_cpu_init(void)
2694 {
2695 	int cpu = smp_processor_id();
2696 
2697 	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
2698 	set_cpu_online(cpu, true);
2699 	set_cpu_active(cpu, true);
2700 	set_cpu_present(cpu, true);
2701 	set_cpu_possible(cpu, true);
2702 
2703 #ifdef CONFIG_SMP
2704 	__boot_cpu_id = cpu;
2705 #endif
2706 }
2707 
2708 /*
2709  * Must be called _AFTER_ setting up the per_cpu areas
2710  */
boot_cpu_hotplug_init(void)2711 void __init boot_cpu_hotplug_init(void)
2712 {
2713 #ifdef CONFIG_SMP
2714 	cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2715 #endif
2716 	this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2717 }
2718 
2719 /*
2720  * These are used for a global "mitigations=" cmdline option for toggling
2721  * optional CPU mitigations.
2722  */
2723 enum cpu_mitigations {
2724 	CPU_MITIGATIONS_OFF,
2725 	CPU_MITIGATIONS_AUTO,
2726 	CPU_MITIGATIONS_AUTO_NOSMT,
2727 };
2728 
2729 static enum cpu_mitigations cpu_mitigations __ro_after_init =
2730 	CPU_MITIGATIONS_AUTO;
2731 
mitigations_parse_cmdline(char * arg)2732 static int __init mitigations_parse_cmdline(char *arg)
2733 {
2734 	if (!strcmp(arg, "off"))
2735 		cpu_mitigations = CPU_MITIGATIONS_OFF;
2736 	else if (!strcmp(arg, "auto"))
2737 		cpu_mitigations = CPU_MITIGATIONS_AUTO;
2738 	else if (!strcmp(arg, "auto,nosmt"))
2739 		cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2740 	else
2741 		pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2742 			arg);
2743 
2744 	return 0;
2745 }
2746 early_param("mitigations", mitigations_parse_cmdline);
2747 
2748 /* mitigations=off */
cpu_mitigations_off(void)2749 bool cpu_mitigations_off(void)
2750 {
2751 	return cpu_mitigations == CPU_MITIGATIONS_OFF;
2752 }
2753 EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2754 
2755 /* mitigations=auto,nosmt */
cpu_mitigations_auto_nosmt(void)2756 bool cpu_mitigations_auto_nosmt(void)
2757 {
2758 	return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2759 }
2760 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
2761