1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
4  * policies)
5  */
6 
7 int sched_rr_timeslice = RR_TIMESLICE;
8 /* More than 4 hours if BW_SHIFT equals 20. */
9 static const u64 max_rt_runtime = MAX_BW;
10 
11 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
12 
13 struct rt_bandwidth def_rt_bandwidth;
14 
15 /*
16  * period over which we measure -rt task CPU usage in us.
17  * default: 1s
18  */
19 unsigned int sysctl_sched_rt_period = 1000000;
20 
21 /*
22  * part of the period that we allow rt tasks to run in us.
23  * default: 0.95s
24  */
25 int sysctl_sched_rt_runtime = 950000;
26 
27 #ifdef CONFIG_SYSCTL
28 static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
29 static int sched_rt_handler(struct ctl_table *table, int write, void *buffer,
30 		size_t *lenp, loff_t *ppos);
31 static int sched_rr_handler(struct ctl_table *table, int write, void *buffer,
32 		size_t *lenp, loff_t *ppos);
33 static struct ctl_table sched_rt_sysctls[] = {
34 	{
35 		.procname       = "sched_rt_period_us",
36 		.data           = &sysctl_sched_rt_period,
37 		.maxlen         = sizeof(unsigned int),
38 		.mode           = 0644,
39 		.proc_handler   = sched_rt_handler,
40 		.extra1         = SYSCTL_ONE,
41 		.extra2         = SYSCTL_INT_MAX,
42 	},
43 	{
44 		.procname       = "sched_rt_runtime_us",
45 		.data           = &sysctl_sched_rt_runtime,
46 		.maxlen         = sizeof(int),
47 		.mode           = 0644,
48 		.proc_handler   = sched_rt_handler,
49 		.extra1         = SYSCTL_NEG_ONE,
50 		.extra2         = SYSCTL_INT_MAX,
51 	},
52 	{
53 		.procname       = "sched_rr_timeslice_ms",
54 		.data           = &sysctl_sched_rr_timeslice,
55 		.maxlen         = sizeof(int),
56 		.mode           = 0644,
57 		.proc_handler   = sched_rr_handler,
58 	},
59 	{}
60 };
61 
sched_rt_sysctl_init(void)62 static int __init sched_rt_sysctl_init(void)
63 {
64 	register_sysctl_init("kernel", sched_rt_sysctls);
65 	return 0;
66 }
67 late_initcall(sched_rt_sysctl_init);
68 #endif
69 
sched_rt_period_timer(struct hrtimer * timer)70 static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
71 {
72 	struct rt_bandwidth *rt_b =
73 		container_of(timer, struct rt_bandwidth, rt_period_timer);
74 	int idle = 0;
75 	int overrun;
76 
77 	raw_spin_lock(&rt_b->rt_runtime_lock);
78 	for (;;) {
79 		overrun = hrtimer_forward_now(timer, rt_b->rt_period);
80 		if (!overrun)
81 			break;
82 
83 		raw_spin_unlock(&rt_b->rt_runtime_lock);
84 		idle = do_sched_rt_period_timer(rt_b, overrun);
85 		raw_spin_lock(&rt_b->rt_runtime_lock);
86 	}
87 	if (idle)
88 		rt_b->rt_period_active = 0;
89 	raw_spin_unlock(&rt_b->rt_runtime_lock);
90 
91 	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
92 }
93 
init_rt_bandwidth(struct rt_bandwidth * rt_b,u64 period,u64 runtime)94 void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
95 {
96 	rt_b->rt_period = ns_to_ktime(period);
97 	rt_b->rt_runtime = runtime;
98 
99 	raw_spin_lock_init(&rt_b->rt_runtime_lock);
100 
101 	hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC,
102 		     HRTIMER_MODE_REL_HARD);
103 	rt_b->rt_period_timer.function = sched_rt_period_timer;
104 }
105 
do_start_rt_bandwidth(struct rt_bandwidth * rt_b)106 static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b)
107 {
108 	raw_spin_lock(&rt_b->rt_runtime_lock);
109 	if (!rt_b->rt_period_active) {
110 		rt_b->rt_period_active = 1;
111 		/*
112 		 * SCHED_DEADLINE updates the bandwidth, as a run away
113 		 * RT task with a DL task could hog a CPU. But DL does
114 		 * not reset the period. If a deadline task was running
115 		 * without an RT task running, it can cause RT tasks to
116 		 * throttle when they start up. Kick the timer right away
117 		 * to update the period.
118 		 */
119 		hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
120 		hrtimer_start_expires(&rt_b->rt_period_timer,
121 				      HRTIMER_MODE_ABS_PINNED_HARD);
122 	}
123 	raw_spin_unlock(&rt_b->rt_runtime_lock);
124 }
125 
start_rt_bandwidth(struct rt_bandwidth * rt_b)126 static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
127 {
128 	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
129 		return;
130 
131 	do_start_rt_bandwidth(rt_b);
132 }
133 
init_rt_rq(struct rt_rq * rt_rq)134 void init_rt_rq(struct rt_rq *rt_rq)
135 {
136 	struct rt_prio_array *array;
137 	int i;
138 
139 	array = &rt_rq->active;
140 	for (i = 0; i < MAX_RT_PRIO; i++) {
141 		INIT_LIST_HEAD(array->queue + i);
142 		__clear_bit(i, array->bitmap);
143 	}
144 	/* delimiter for bitsearch: */
145 	__set_bit(MAX_RT_PRIO, array->bitmap);
146 
147 #if defined CONFIG_SMP
148 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
149 	rt_rq->highest_prio.next = MAX_RT_PRIO-1;
150 	rt_rq->rt_nr_migratory = 0;
151 	rt_rq->overloaded = 0;
152 	plist_head_init(&rt_rq->pushable_tasks);
153 #endif /* CONFIG_SMP */
154 	/* We start is dequeued state, because no RT tasks are queued */
155 	rt_rq->rt_queued = 0;
156 
157 	rt_rq->rt_time = 0;
158 	rt_rq->rt_throttled = 0;
159 	rt_rq->rt_runtime = 0;
160 	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
161 }
162 
163 #ifdef CONFIG_RT_GROUP_SCHED
destroy_rt_bandwidth(struct rt_bandwidth * rt_b)164 static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
165 {
166 	hrtimer_cancel(&rt_b->rt_period_timer);
167 }
168 
169 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
170 
rt_task_of(struct sched_rt_entity * rt_se)171 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
172 {
173 #ifdef CONFIG_SCHED_DEBUG
174 	WARN_ON_ONCE(!rt_entity_is_task(rt_se));
175 #endif
176 	return container_of(rt_se, struct task_struct, rt);
177 }
178 
rq_of_rt_rq(struct rt_rq * rt_rq)179 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
180 {
181 	return rt_rq->rq;
182 }
183 
rt_rq_of_se(struct sched_rt_entity * rt_se)184 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
185 {
186 	return rt_se->rt_rq;
187 }
188 
rq_of_rt_se(struct sched_rt_entity * rt_se)189 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
190 {
191 	struct rt_rq *rt_rq = rt_se->rt_rq;
192 
193 	return rt_rq->rq;
194 }
195 
unregister_rt_sched_group(struct task_group * tg)196 void unregister_rt_sched_group(struct task_group *tg)
197 {
198 	if (tg->rt_se)
199 		destroy_rt_bandwidth(&tg->rt_bandwidth);
200 
201 }
202 
free_rt_sched_group(struct task_group * tg)203 void free_rt_sched_group(struct task_group *tg)
204 {
205 	int i;
206 
207 	for_each_possible_cpu(i) {
208 		if (tg->rt_rq)
209 			kfree(tg->rt_rq[i]);
210 		if (tg->rt_se)
211 			kfree(tg->rt_se[i]);
212 	}
213 
214 	kfree(tg->rt_rq);
215 	kfree(tg->rt_se);
216 }
217 
init_tg_rt_entry(struct task_group * tg,struct rt_rq * rt_rq,struct sched_rt_entity * rt_se,int cpu,struct sched_rt_entity * parent)218 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
219 		struct sched_rt_entity *rt_se, int cpu,
220 		struct sched_rt_entity *parent)
221 {
222 	struct rq *rq = cpu_rq(cpu);
223 
224 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
225 	rt_rq->rt_nr_boosted = 0;
226 	rt_rq->rq = rq;
227 	rt_rq->tg = tg;
228 
229 	tg->rt_rq[cpu] = rt_rq;
230 	tg->rt_se[cpu] = rt_se;
231 
232 	if (!rt_se)
233 		return;
234 
235 	if (!parent)
236 		rt_se->rt_rq = &rq->rt;
237 	else
238 		rt_se->rt_rq = parent->my_q;
239 
240 	rt_se->my_q = rt_rq;
241 	rt_se->parent = parent;
242 	INIT_LIST_HEAD(&rt_se->run_list);
243 }
244 
alloc_rt_sched_group(struct task_group * tg,struct task_group * parent)245 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
246 {
247 	struct rt_rq *rt_rq;
248 	struct sched_rt_entity *rt_se;
249 	int i;
250 
251 	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);
252 	if (!tg->rt_rq)
253 		goto err;
254 	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);
255 	if (!tg->rt_se)
256 		goto err;
257 
258 	init_rt_bandwidth(&tg->rt_bandwidth,
259 			ktime_to_ns(def_rt_bandwidth.rt_period), 0);
260 
261 	for_each_possible_cpu(i) {
262 		rt_rq = kzalloc_node(sizeof(struct rt_rq),
263 				     GFP_KERNEL, cpu_to_node(i));
264 		if (!rt_rq)
265 			goto err;
266 
267 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
268 				     GFP_KERNEL, cpu_to_node(i));
269 		if (!rt_se)
270 			goto err_free_rq;
271 
272 		init_rt_rq(rt_rq);
273 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
274 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
275 	}
276 
277 	return 1;
278 
279 err_free_rq:
280 	kfree(rt_rq);
281 err:
282 	return 0;
283 }
284 
285 #else /* CONFIG_RT_GROUP_SCHED */
286 
287 #define rt_entity_is_task(rt_se) (1)
288 
rt_task_of(struct sched_rt_entity * rt_se)289 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
290 {
291 	return container_of(rt_se, struct task_struct, rt);
292 }
293 
rq_of_rt_rq(struct rt_rq * rt_rq)294 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
295 {
296 	return container_of(rt_rq, struct rq, rt);
297 }
298 
rq_of_rt_se(struct sched_rt_entity * rt_se)299 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
300 {
301 	struct task_struct *p = rt_task_of(rt_se);
302 
303 	return task_rq(p);
304 }
305 
rt_rq_of_se(struct sched_rt_entity * rt_se)306 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
307 {
308 	struct rq *rq = rq_of_rt_se(rt_se);
309 
310 	return &rq->rt;
311 }
312 
unregister_rt_sched_group(struct task_group * tg)313 void unregister_rt_sched_group(struct task_group *tg) { }
314 
free_rt_sched_group(struct task_group * tg)315 void free_rt_sched_group(struct task_group *tg) { }
316 
alloc_rt_sched_group(struct task_group * tg,struct task_group * parent)317 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
318 {
319 	return 1;
320 }
321 #endif /* CONFIG_RT_GROUP_SCHED */
322 
323 #ifdef CONFIG_SMP
324 
need_pull_rt_task(struct rq * rq,struct task_struct * prev)325 static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
326 {
327 	/* Try to pull RT tasks here if we lower this rq's prio */
328 	return rq->online && rq->rt.highest_prio.curr > prev->prio;
329 }
330 
rt_overloaded(struct rq * rq)331 static inline int rt_overloaded(struct rq *rq)
332 {
333 	return atomic_read(&rq->rd->rto_count);
334 }
335 
rt_set_overload(struct rq * rq)336 static inline void rt_set_overload(struct rq *rq)
337 {
338 	if (!rq->online)
339 		return;
340 
341 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
342 	/*
343 	 * Make sure the mask is visible before we set
344 	 * the overload count. That is checked to determine
345 	 * if we should look at the mask. It would be a shame
346 	 * if we looked at the mask, but the mask was not
347 	 * updated yet.
348 	 *
349 	 * Matched by the barrier in pull_rt_task().
350 	 */
351 	smp_wmb();
352 	atomic_inc(&rq->rd->rto_count);
353 }
354 
rt_clear_overload(struct rq * rq)355 static inline void rt_clear_overload(struct rq *rq)
356 {
357 	if (!rq->online)
358 		return;
359 
360 	/* the order here really doesn't matter */
361 	atomic_dec(&rq->rd->rto_count);
362 	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
363 }
364 
update_rt_migration(struct rt_rq * rt_rq)365 static void update_rt_migration(struct rt_rq *rt_rq)
366 {
367 	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
368 		if (!rt_rq->overloaded) {
369 			rt_set_overload(rq_of_rt_rq(rt_rq));
370 			rt_rq->overloaded = 1;
371 		}
372 	} else if (rt_rq->overloaded) {
373 		rt_clear_overload(rq_of_rt_rq(rt_rq));
374 		rt_rq->overloaded = 0;
375 	}
376 }
377 
inc_rt_migration(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)378 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
379 {
380 	struct task_struct *p;
381 
382 	if (!rt_entity_is_task(rt_se))
383 		return;
384 
385 	p = rt_task_of(rt_se);
386 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;
387 
388 	rt_rq->rt_nr_total++;
389 	if (p->nr_cpus_allowed > 1)
390 		rt_rq->rt_nr_migratory++;
391 
392 	update_rt_migration(rt_rq);
393 }
394 
dec_rt_migration(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)395 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
396 {
397 	struct task_struct *p;
398 
399 	if (!rt_entity_is_task(rt_se))
400 		return;
401 
402 	p = rt_task_of(rt_se);
403 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;
404 
405 	rt_rq->rt_nr_total--;
406 	if (p->nr_cpus_allowed > 1)
407 		rt_rq->rt_nr_migratory--;
408 
409 	update_rt_migration(rt_rq);
410 }
411 
has_pushable_tasks(struct rq * rq)412 static inline int has_pushable_tasks(struct rq *rq)
413 {
414 	return !plist_head_empty(&rq->rt.pushable_tasks);
415 }
416 
417 static DEFINE_PER_CPU(struct balance_callback, rt_push_head);
418 static DEFINE_PER_CPU(struct balance_callback, rt_pull_head);
419 
420 static void push_rt_tasks(struct rq *);
421 static void pull_rt_task(struct rq *);
422 
rt_queue_push_tasks(struct rq * rq)423 static inline void rt_queue_push_tasks(struct rq *rq)
424 {
425 	if (!has_pushable_tasks(rq))
426 		return;
427 
428 	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);
429 }
430 
rt_queue_pull_task(struct rq * rq)431 static inline void rt_queue_pull_task(struct rq *rq)
432 {
433 	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);
434 }
435 
enqueue_pushable_task(struct rq * rq,struct task_struct * p)436 static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
437 {
438 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
439 	plist_node_init(&p->pushable_tasks, p->prio);
440 	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
441 
442 	/* Update the highest prio pushable task */
443 	if (p->prio < rq->rt.highest_prio.next)
444 		rq->rt.highest_prio.next = p->prio;
445 }
446 
dequeue_pushable_task(struct rq * rq,struct task_struct * p)447 static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
448 {
449 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
450 
451 	/* Update the new highest prio pushable task */
452 	if (has_pushable_tasks(rq)) {
453 		p = plist_first_entry(&rq->rt.pushable_tasks,
454 				      struct task_struct, pushable_tasks);
455 		rq->rt.highest_prio.next = p->prio;
456 	} else {
457 		rq->rt.highest_prio.next = MAX_RT_PRIO-1;
458 	}
459 }
460 
461 #else
462 
enqueue_pushable_task(struct rq * rq,struct task_struct * p)463 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
464 {
465 }
466 
dequeue_pushable_task(struct rq * rq,struct task_struct * p)467 static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
468 {
469 }
470 
471 static inline
inc_rt_migration(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)472 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
473 {
474 }
475 
476 static inline
dec_rt_migration(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)477 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
478 {
479 }
480 
rt_queue_push_tasks(struct rq * rq)481 static inline void rt_queue_push_tasks(struct rq *rq)
482 {
483 }
484 #endif /* CONFIG_SMP */
485 
486 static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
487 static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count);
488 
on_rt_rq(struct sched_rt_entity * rt_se)489 static inline int on_rt_rq(struct sched_rt_entity *rt_se)
490 {
491 	return rt_se->on_rq;
492 }
493 
494 #ifdef CONFIG_UCLAMP_TASK
495 /*
496  * Verify the fitness of task @p to run on @cpu taking into account the uclamp
497  * settings.
498  *
499  * This check is only important for heterogeneous systems where uclamp_min value
500  * is higher than the capacity of a @cpu. For non-heterogeneous system this
501  * function will always return true.
502  *
503  * The function will return true if the capacity of the @cpu is >= the
504  * uclamp_min and false otherwise.
505  *
506  * Note that uclamp_min will be clamped to uclamp_max if uclamp_min
507  * > uclamp_max.
508  */
rt_task_fits_capacity(struct task_struct * p,int cpu)509 static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
510 {
511 	unsigned int min_cap;
512 	unsigned int max_cap;
513 	unsigned int cpu_cap;
514 
515 	/* Only heterogeneous systems can benefit from this check */
516 	if (!sched_asym_cpucap_active())
517 		return true;
518 
519 	min_cap = uclamp_eff_value(p, UCLAMP_MIN);
520 	max_cap = uclamp_eff_value(p, UCLAMP_MAX);
521 
522 	cpu_cap = capacity_orig_of(cpu);
523 
524 	return cpu_cap >= min(min_cap, max_cap);
525 }
526 #else
rt_task_fits_capacity(struct task_struct * p,int cpu)527 static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
528 {
529 	return true;
530 }
531 #endif
532 
533 #ifdef CONFIG_RT_GROUP_SCHED
534 
sched_rt_runtime(struct rt_rq * rt_rq)535 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
536 {
537 	if (!rt_rq->tg)
538 		return RUNTIME_INF;
539 
540 	return rt_rq->rt_runtime;
541 }
542 
sched_rt_period(struct rt_rq * rt_rq)543 static inline u64 sched_rt_period(struct rt_rq *rt_rq)
544 {
545 	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
546 }
547 
548 typedef struct task_group *rt_rq_iter_t;
549 
next_task_group(struct task_group * tg)550 static inline struct task_group *next_task_group(struct task_group *tg)
551 {
552 	do {
553 		tg = list_entry_rcu(tg->list.next,
554 			typeof(struct task_group), list);
555 	} while (&tg->list != &task_groups && task_group_is_autogroup(tg));
556 
557 	if (&tg->list == &task_groups)
558 		tg = NULL;
559 
560 	return tg;
561 }
562 
563 #define for_each_rt_rq(rt_rq, iter, rq)					\
564 	for (iter = container_of(&task_groups, typeof(*iter), list);	\
565 		(iter = next_task_group(iter)) &&			\
566 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
567 
568 #define for_each_sched_rt_entity(rt_se) \
569 	for (; rt_se; rt_se = rt_se->parent)
570 
group_rt_rq(struct sched_rt_entity * rt_se)571 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
572 {
573 	return rt_se->my_q;
574 }
575 
576 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
577 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
578 
sched_rt_rq_enqueue(struct rt_rq * rt_rq)579 static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
580 {
581 	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
582 	struct rq *rq = rq_of_rt_rq(rt_rq);
583 	struct sched_rt_entity *rt_se;
584 
585 	int cpu = cpu_of(rq);
586 
587 	rt_se = rt_rq->tg->rt_se[cpu];
588 
589 	if (rt_rq->rt_nr_running) {
590 		if (!rt_se)
591 			enqueue_top_rt_rq(rt_rq);
592 		else if (!on_rt_rq(rt_se))
593 			enqueue_rt_entity(rt_se, 0);
594 
595 		if (rt_rq->highest_prio.curr < curr->prio)
596 			resched_curr(rq);
597 	}
598 }
599 
sched_rt_rq_dequeue(struct rt_rq * rt_rq)600 static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
601 {
602 	struct sched_rt_entity *rt_se;
603 	int cpu = cpu_of(rq_of_rt_rq(rt_rq));
604 
605 	rt_se = rt_rq->tg->rt_se[cpu];
606 
607 	if (!rt_se) {
608 		dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
609 		/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
610 		cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);
611 	}
612 	else if (on_rt_rq(rt_se))
613 		dequeue_rt_entity(rt_se, 0);
614 }
615 
rt_rq_throttled(struct rt_rq * rt_rq)616 static inline int rt_rq_throttled(struct rt_rq *rt_rq)
617 {
618 	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
619 }
620 
rt_se_boosted(struct sched_rt_entity * rt_se)621 static int rt_se_boosted(struct sched_rt_entity *rt_se)
622 {
623 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
624 	struct task_struct *p;
625 
626 	if (rt_rq)
627 		return !!rt_rq->rt_nr_boosted;
628 
629 	p = rt_task_of(rt_se);
630 	return p->prio != p->normal_prio;
631 }
632 
633 #ifdef CONFIG_SMP
sched_rt_period_mask(void)634 static inline const struct cpumask *sched_rt_period_mask(void)
635 {
636 	return this_rq()->rd->span;
637 }
638 #else
sched_rt_period_mask(void)639 static inline const struct cpumask *sched_rt_period_mask(void)
640 {
641 	return cpu_online_mask;
642 }
643 #endif
644 
645 static inline
sched_rt_period_rt_rq(struct rt_bandwidth * rt_b,int cpu)646 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
647 {
648 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
649 }
650 
sched_rt_bandwidth(struct rt_rq * rt_rq)651 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
652 {
653 	return &rt_rq->tg->rt_bandwidth;
654 }
655 
656 #else /* !CONFIG_RT_GROUP_SCHED */
657 
sched_rt_runtime(struct rt_rq * rt_rq)658 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
659 {
660 	return rt_rq->rt_runtime;
661 }
662 
sched_rt_period(struct rt_rq * rt_rq)663 static inline u64 sched_rt_period(struct rt_rq *rt_rq)
664 {
665 	return ktime_to_ns(def_rt_bandwidth.rt_period);
666 }
667 
668 typedef struct rt_rq *rt_rq_iter_t;
669 
670 #define for_each_rt_rq(rt_rq, iter, rq) \
671 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
672 
673 #define for_each_sched_rt_entity(rt_se) \
674 	for (; rt_se; rt_se = NULL)
675 
group_rt_rq(struct sched_rt_entity * rt_se)676 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
677 {
678 	return NULL;
679 }
680 
sched_rt_rq_enqueue(struct rt_rq * rt_rq)681 static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
682 {
683 	struct rq *rq = rq_of_rt_rq(rt_rq);
684 
685 	if (!rt_rq->rt_nr_running)
686 		return;
687 
688 	enqueue_top_rt_rq(rt_rq);
689 	resched_curr(rq);
690 }
691 
sched_rt_rq_dequeue(struct rt_rq * rt_rq)692 static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
693 {
694 	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
695 }
696 
rt_rq_throttled(struct rt_rq * rt_rq)697 static inline int rt_rq_throttled(struct rt_rq *rt_rq)
698 {
699 	return rt_rq->rt_throttled;
700 }
701 
sched_rt_period_mask(void)702 static inline const struct cpumask *sched_rt_period_mask(void)
703 {
704 	return cpu_online_mask;
705 }
706 
707 static inline
sched_rt_period_rt_rq(struct rt_bandwidth * rt_b,int cpu)708 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
709 {
710 	return &cpu_rq(cpu)->rt;
711 }
712 
sched_rt_bandwidth(struct rt_rq * rt_rq)713 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
714 {
715 	return &def_rt_bandwidth;
716 }
717 
718 #endif /* CONFIG_RT_GROUP_SCHED */
719 
sched_rt_bandwidth_account(struct rt_rq * rt_rq)720 bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
721 {
722 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
723 
724 	return (hrtimer_active(&rt_b->rt_period_timer) ||
725 		rt_rq->rt_time < rt_b->rt_runtime);
726 }
727 
728 #ifdef CONFIG_SMP
729 /*
730  * We ran out of runtime, see if we can borrow some from our neighbours.
731  */
do_balance_runtime(struct rt_rq * rt_rq)732 static void do_balance_runtime(struct rt_rq *rt_rq)
733 {
734 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
735 	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
736 	int i, weight;
737 	u64 rt_period;
738 
739 	weight = cpumask_weight(rd->span);
740 
741 	raw_spin_lock(&rt_b->rt_runtime_lock);
742 	rt_period = ktime_to_ns(rt_b->rt_period);
743 	for_each_cpu(i, rd->span) {
744 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
745 		s64 diff;
746 
747 		if (iter == rt_rq)
748 			continue;
749 
750 		raw_spin_lock(&iter->rt_runtime_lock);
751 		/*
752 		 * Either all rqs have inf runtime and there's nothing to steal
753 		 * or __disable_runtime() below sets a specific rq to inf to
754 		 * indicate its been disabled and disallow stealing.
755 		 */
756 		if (iter->rt_runtime == RUNTIME_INF)
757 			goto next;
758 
759 		/*
760 		 * From runqueues with spare time, take 1/n part of their
761 		 * spare time, but no more than our period.
762 		 */
763 		diff = iter->rt_runtime - iter->rt_time;
764 		if (diff > 0) {
765 			diff = div_u64((u64)diff, weight);
766 			if (rt_rq->rt_runtime + diff > rt_period)
767 				diff = rt_period - rt_rq->rt_runtime;
768 			iter->rt_runtime -= diff;
769 			rt_rq->rt_runtime += diff;
770 			if (rt_rq->rt_runtime == rt_period) {
771 				raw_spin_unlock(&iter->rt_runtime_lock);
772 				break;
773 			}
774 		}
775 next:
776 		raw_spin_unlock(&iter->rt_runtime_lock);
777 	}
778 	raw_spin_unlock(&rt_b->rt_runtime_lock);
779 }
780 
781 /*
782  * Ensure this RQ takes back all the runtime it lend to its neighbours.
783  */
__disable_runtime(struct rq * rq)784 static void __disable_runtime(struct rq *rq)
785 {
786 	struct root_domain *rd = rq->rd;
787 	rt_rq_iter_t iter;
788 	struct rt_rq *rt_rq;
789 
790 	if (unlikely(!scheduler_running))
791 		return;
792 
793 	for_each_rt_rq(rt_rq, iter, rq) {
794 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
795 		s64 want;
796 		int i;
797 
798 		raw_spin_lock(&rt_b->rt_runtime_lock);
799 		raw_spin_lock(&rt_rq->rt_runtime_lock);
800 		/*
801 		 * Either we're all inf and nobody needs to borrow, or we're
802 		 * already disabled and thus have nothing to do, or we have
803 		 * exactly the right amount of runtime to take out.
804 		 */
805 		if (rt_rq->rt_runtime == RUNTIME_INF ||
806 				rt_rq->rt_runtime == rt_b->rt_runtime)
807 			goto balanced;
808 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
809 
810 		/*
811 		 * Calculate the difference between what we started out with
812 		 * and what we current have, that's the amount of runtime
813 		 * we lend and now have to reclaim.
814 		 */
815 		want = rt_b->rt_runtime - rt_rq->rt_runtime;
816 
817 		/*
818 		 * Greedy reclaim, take back as much as we can.
819 		 */
820 		for_each_cpu(i, rd->span) {
821 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
822 			s64 diff;
823 
824 			/*
825 			 * Can't reclaim from ourselves or disabled runqueues.
826 			 */
827 			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
828 				continue;
829 
830 			raw_spin_lock(&iter->rt_runtime_lock);
831 			if (want > 0) {
832 				diff = min_t(s64, iter->rt_runtime, want);
833 				iter->rt_runtime -= diff;
834 				want -= diff;
835 			} else {
836 				iter->rt_runtime -= want;
837 				want -= want;
838 			}
839 			raw_spin_unlock(&iter->rt_runtime_lock);
840 
841 			if (!want)
842 				break;
843 		}
844 
845 		raw_spin_lock(&rt_rq->rt_runtime_lock);
846 		/*
847 		 * We cannot be left wanting - that would mean some runtime
848 		 * leaked out of the system.
849 		 */
850 		WARN_ON_ONCE(want);
851 balanced:
852 		/*
853 		 * Disable all the borrow logic by pretending we have inf
854 		 * runtime - in which case borrowing doesn't make sense.
855 		 */
856 		rt_rq->rt_runtime = RUNTIME_INF;
857 		rt_rq->rt_throttled = 0;
858 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
859 		raw_spin_unlock(&rt_b->rt_runtime_lock);
860 
861 		/* Make rt_rq available for pick_next_task() */
862 		sched_rt_rq_enqueue(rt_rq);
863 	}
864 }
865 
__enable_runtime(struct rq * rq)866 static void __enable_runtime(struct rq *rq)
867 {
868 	rt_rq_iter_t iter;
869 	struct rt_rq *rt_rq;
870 
871 	if (unlikely(!scheduler_running))
872 		return;
873 
874 	/*
875 	 * Reset each runqueue's bandwidth settings
876 	 */
877 	for_each_rt_rq(rt_rq, iter, rq) {
878 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
879 
880 		raw_spin_lock(&rt_b->rt_runtime_lock);
881 		raw_spin_lock(&rt_rq->rt_runtime_lock);
882 		rt_rq->rt_runtime = rt_b->rt_runtime;
883 		rt_rq->rt_time = 0;
884 		rt_rq->rt_throttled = 0;
885 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
886 		raw_spin_unlock(&rt_b->rt_runtime_lock);
887 	}
888 }
889 
balance_runtime(struct rt_rq * rt_rq)890 static void balance_runtime(struct rt_rq *rt_rq)
891 {
892 	if (!sched_feat(RT_RUNTIME_SHARE))
893 		return;
894 
895 	if (rt_rq->rt_time > rt_rq->rt_runtime) {
896 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
897 		do_balance_runtime(rt_rq);
898 		raw_spin_lock(&rt_rq->rt_runtime_lock);
899 	}
900 }
901 #else /* !CONFIG_SMP */
balance_runtime(struct rt_rq * rt_rq)902 static inline void balance_runtime(struct rt_rq *rt_rq) {}
903 #endif /* CONFIG_SMP */
904 
do_sched_rt_period_timer(struct rt_bandwidth * rt_b,int overrun)905 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
906 {
907 	int i, idle = 1, throttled = 0;
908 	const struct cpumask *span;
909 
910 	span = sched_rt_period_mask();
911 #ifdef CONFIG_RT_GROUP_SCHED
912 	/*
913 	 * FIXME: isolated CPUs should really leave the root task group,
914 	 * whether they are isolcpus or were isolated via cpusets, lest
915 	 * the timer run on a CPU which does not service all runqueues,
916 	 * potentially leaving other CPUs indefinitely throttled.  If
917 	 * isolation is really required, the user will turn the throttle
918 	 * off to kill the perturbations it causes anyway.  Meanwhile,
919 	 * this maintains functionality for boot and/or troubleshooting.
920 	 */
921 	if (rt_b == &root_task_group.rt_bandwidth)
922 		span = cpu_online_mask;
923 #endif
924 	for_each_cpu(i, span) {
925 		int enqueue = 0;
926 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
927 		struct rq *rq = rq_of_rt_rq(rt_rq);
928 		struct rq_flags rf;
929 		int skip;
930 
931 		/*
932 		 * When span == cpu_online_mask, taking each rq->lock
933 		 * can be time-consuming. Try to avoid it when possible.
934 		 */
935 		raw_spin_lock(&rt_rq->rt_runtime_lock);
936 		if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)
937 			rt_rq->rt_runtime = rt_b->rt_runtime;
938 		skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;
939 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
940 		if (skip)
941 			continue;
942 
943 		rq_lock(rq, &rf);
944 		update_rq_clock(rq);
945 
946 		if (rt_rq->rt_time) {
947 			u64 runtime;
948 
949 			raw_spin_lock(&rt_rq->rt_runtime_lock);
950 			if (rt_rq->rt_throttled)
951 				balance_runtime(rt_rq);
952 			runtime = rt_rq->rt_runtime;
953 			rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
954 			if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
955 				rt_rq->rt_throttled = 0;
956 				enqueue = 1;
957 
958 				/*
959 				 * When we're idle and a woken (rt) task is
960 				 * throttled check_preempt_curr() will set
961 				 * skip_update and the time between the wakeup
962 				 * and this unthrottle will get accounted as
963 				 * 'runtime'.
964 				 */
965 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)
966 					rq_clock_cancel_skipupdate(rq);
967 			}
968 			if (rt_rq->rt_time || rt_rq->rt_nr_running)
969 				idle = 0;
970 			raw_spin_unlock(&rt_rq->rt_runtime_lock);
971 		} else if (rt_rq->rt_nr_running) {
972 			idle = 0;
973 			if (!rt_rq_throttled(rt_rq))
974 				enqueue = 1;
975 		}
976 		if (rt_rq->rt_throttled)
977 			throttled = 1;
978 
979 		if (enqueue)
980 			sched_rt_rq_enqueue(rt_rq);
981 		rq_unlock(rq, &rf);
982 	}
983 
984 	if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
985 		return 1;
986 
987 	return idle;
988 }
989 
rt_se_prio(struct sched_rt_entity * rt_se)990 static inline int rt_se_prio(struct sched_rt_entity *rt_se)
991 {
992 #ifdef CONFIG_RT_GROUP_SCHED
993 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
994 
995 	if (rt_rq)
996 		return rt_rq->highest_prio.curr;
997 #endif
998 
999 	return rt_task_of(rt_se)->prio;
1000 }
1001 
sched_rt_runtime_exceeded(struct rt_rq * rt_rq)1002 static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
1003 {
1004 	u64 runtime = sched_rt_runtime(rt_rq);
1005 
1006 	if (rt_rq->rt_throttled)
1007 		return rt_rq_throttled(rt_rq);
1008 
1009 	if (runtime >= sched_rt_period(rt_rq))
1010 		return 0;
1011 
1012 	balance_runtime(rt_rq);
1013 	runtime = sched_rt_runtime(rt_rq);
1014 	if (runtime == RUNTIME_INF)
1015 		return 0;
1016 
1017 	if (rt_rq->rt_time > runtime) {
1018 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
1019 
1020 		/*
1021 		 * Don't actually throttle groups that have no runtime assigned
1022 		 * but accrue some time due to boosting.
1023 		 */
1024 		if (likely(rt_b->rt_runtime)) {
1025 			rt_rq->rt_throttled = 1;
1026 			printk_deferred_once("sched: RT throttling activated\n");
1027 		} else {
1028 			/*
1029 			 * In case we did anyway, make it go away,
1030 			 * replenishment is a joke, since it will replenish us
1031 			 * with exactly 0 ns.
1032 			 */
1033 			rt_rq->rt_time = 0;
1034 		}
1035 
1036 		if (rt_rq_throttled(rt_rq)) {
1037 			sched_rt_rq_dequeue(rt_rq);
1038 			return 1;
1039 		}
1040 	}
1041 
1042 	return 0;
1043 }
1044 
1045 /*
1046  * Update the current task's runtime statistics. Skip current tasks that
1047  * are not in our scheduling class.
1048  */
update_curr_rt(struct rq * rq)1049 static void update_curr_rt(struct rq *rq)
1050 {
1051 	struct task_struct *curr = rq->curr;
1052 	struct sched_rt_entity *rt_se = &curr->rt;
1053 	u64 delta_exec;
1054 	u64 now;
1055 
1056 	if (curr->sched_class != &rt_sched_class)
1057 		return;
1058 
1059 	now = rq_clock_task(rq);
1060 	delta_exec = now - curr->se.exec_start;
1061 	if (unlikely((s64)delta_exec <= 0))
1062 		return;
1063 
1064 	schedstat_set(curr->stats.exec_max,
1065 		      max(curr->stats.exec_max, delta_exec));
1066 
1067 	trace_sched_stat_runtime(curr, delta_exec, 0);
1068 
1069 	update_current_exec_runtime(curr, now, delta_exec);
1070 
1071 	if (!rt_bandwidth_enabled())
1072 		return;
1073 
1074 	for_each_sched_rt_entity(rt_se) {
1075 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1076 		int exceeded;
1077 
1078 		if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
1079 			raw_spin_lock(&rt_rq->rt_runtime_lock);
1080 			rt_rq->rt_time += delta_exec;
1081 			exceeded = sched_rt_runtime_exceeded(rt_rq);
1082 			if (exceeded)
1083 				resched_curr(rq);
1084 			raw_spin_unlock(&rt_rq->rt_runtime_lock);
1085 			if (exceeded)
1086 				do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
1087 		}
1088 	}
1089 }
1090 
1091 static void
dequeue_top_rt_rq(struct rt_rq * rt_rq,unsigned int count)1092 dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count)
1093 {
1094 	struct rq *rq = rq_of_rt_rq(rt_rq);
1095 
1096 	BUG_ON(&rq->rt != rt_rq);
1097 
1098 	if (!rt_rq->rt_queued)
1099 		return;
1100 
1101 	BUG_ON(!rq->nr_running);
1102 
1103 	sub_nr_running(rq, count);
1104 	rt_rq->rt_queued = 0;
1105 
1106 }
1107 
1108 static void
enqueue_top_rt_rq(struct rt_rq * rt_rq)1109 enqueue_top_rt_rq(struct rt_rq *rt_rq)
1110 {
1111 	struct rq *rq = rq_of_rt_rq(rt_rq);
1112 
1113 	BUG_ON(&rq->rt != rt_rq);
1114 
1115 	if (rt_rq->rt_queued)
1116 		return;
1117 
1118 	if (rt_rq_throttled(rt_rq))
1119 		return;
1120 
1121 	if (rt_rq->rt_nr_running) {
1122 		add_nr_running(rq, rt_rq->rt_nr_running);
1123 		rt_rq->rt_queued = 1;
1124 	}
1125 
1126 	/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
1127 	cpufreq_update_util(rq, 0);
1128 }
1129 
1130 #if defined CONFIG_SMP
1131 
1132 static void
inc_rt_prio_smp(struct rt_rq * rt_rq,int prio,int prev_prio)1133 inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1134 {
1135 	struct rq *rq = rq_of_rt_rq(rt_rq);
1136 
1137 #ifdef CONFIG_RT_GROUP_SCHED
1138 	/*
1139 	 * Change rq's cpupri only if rt_rq is the top queue.
1140 	 */
1141 	if (&rq->rt != rt_rq)
1142 		return;
1143 #endif
1144 	if (rq->online && prio < prev_prio)
1145 		cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
1146 }
1147 
1148 static void
dec_rt_prio_smp(struct rt_rq * rt_rq,int prio,int prev_prio)1149 dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1150 {
1151 	struct rq *rq = rq_of_rt_rq(rt_rq);
1152 
1153 #ifdef CONFIG_RT_GROUP_SCHED
1154 	/*
1155 	 * Change rq's cpupri only if rt_rq is the top queue.
1156 	 */
1157 	if (&rq->rt != rt_rq)
1158 		return;
1159 #endif
1160 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)
1161 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
1162 }
1163 
1164 #else /* CONFIG_SMP */
1165 
1166 static inline
inc_rt_prio_smp(struct rt_rq * rt_rq,int prio,int prev_prio)1167 void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1168 static inline
dec_rt_prio_smp(struct rt_rq * rt_rq,int prio,int prev_prio)1169 void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1170 
1171 #endif /* CONFIG_SMP */
1172 
1173 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
1174 static void
inc_rt_prio(struct rt_rq * rt_rq,int prio)1175 inc_rt_prio(struct rt_rq *rt_rq, int prio)
1176 {
1177 	int prev_prio = rt_rq->highest_prio.curr;
1178 
1179 	if (prio < prev_prio)
1180 		rt_rq->highest_prio.curr = prio;
1181 
1182 	inc_rt_prio_smp(rt_rq, prio, prev_prio);
1183 }
1184 
1185 static void
dec_rt_prio(struct rt_rq * rt_rq,int prio)1186 dec_rt_prio(struct rt_rq *rt_rq, int prio)
1187 {
1188 	int prev_prio = rt_rq->highest_prio.curr;
1189 
1190 	if (rt_rq->rt_nr_running) {
1191 
1192 		WARN_ON(prio < prev_prio);
1193 
1194 		/*
1195 		 * This may have been our highest task, and therefore
1196 		 * we may have some recomputation to do
1197 		 */
1198 		if (prio == prev_prio) {
1199 			struct rt_prio_array *array = &rt_rq->active;
1200 
1201 			rt_rq->highest_prio.curr =
1202 				sched_find_first_bit(array->bitmap);
1203 		}
1204 
1205 	} else {
1206 		rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
1207 	}
1208 
1209 	dec_rt_prio_smp(rt_rq, prio, prev_prio);
1210 }
1211 
1212 #else
1213 
inc_rt_prio(struct rt_rq * rt_rq,int prio)1214 static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
dec_rt_prio(struct rt_rq * rt_rq,int prio)1215 static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}
1216 
1217 #endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */
1218 
1219 #ifdef CONFIG_RT_GROUP_SCHED
1220 
1221 static void
inc_rt_group(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)1222 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1223 {
1224 	if (rt_se_boosted(rt_se))
1225 		rt_rq->rt_nr_boosted++;
1226 
1227 	if (rt_rq->tg)
1228 		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
1229 }
1230 
1231 static void
dec_rt_group(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)1232 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1233 {
1234 	if (rt_se_boosted(rt_se))
1235 		rt_rq->rt_nr_boosted--;
1236 
1237 	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
1238 }
1239 
1240 #else /* CONFIG_RT_GROUP_SCHED */
1241 
1242 static void
inc_rt_group(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)1243 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1244 {
1245 	start_rt_bandwidth(&def_rt_bandwidth);
1246 }
1247 
1248 static inline
dec_rt_group(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)1249 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
1250 
1251 #endif /* CONFIG_RT_GROUP_SCHED */
1252 
1253 static inline
rt_se_nr_running(struct sched_rt_entity * rt_se)1254 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
1255 {
1256 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1257 
1258 	if (group_rq)
1259 		return group_rq->rt_nr_running;
1260 	else
1261 		return 1;
1262 }
1263 
1264 static inline
rt_se_rr_nr_running(struct sched_rt_entity * rt_se)1265 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
1266 {
1267 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1268 	struct task_struct *tsk;
1269 
1270 	if (group_rq)
1271 		return group_rq->rr_nr_running;
1272 
1273 	tsk = rt_task_of(rt_se);
1274 
1275 	return (tsk->policy == SCHED_RR) ? 1 : 0;
1276 }
1277 
1278 static inline
inc_rt_tasks(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)1279 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1280 {
1281 	int prio = rt_se_prio(rt_se);
1282 
1283 	WARN_ON(!rt_prio(prio));
1284 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
1285 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);
1286 
1287 	inc_rt_prio(rt_rq, prio);
1288 	inc_rt_migration(rt_se, rt_rq);
1289 	inc_rt_group(rt_se, rt_rq);
1290 }
1291 
1292 static inline
dec_rt_tasks(struct sched_rt_entity * rt_se,struct rt_rq * rt_rq)1293 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1294 {
1295 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
1296 	WARN_ON(!rt_rq->rt_nr_running);
1297 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
1298 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);
1299 
1300 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));
1301 	dec_rt_migration(rt_se, rt_rq);
1302 	dec_rt_group(rt_se, rt_rq);
1303 }
1304 
1305 /*
1306  * Change rt_se->run_list location unless SAVE && !MOVE
1307  *
1308  * assumes ENQUEUE/DEQUEUE flags match
1309  */
move_entity(unsigned int flags)1310 static inline bool move_entity(unsigned int flags)
1311 {
1312 	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE)
1313 		return false;
1314 
1315 	return true;
1316 }
1317 
__delist_rt_entity(struct sched_rt_entity * rt_se,struct rt_prio_array * array)1318 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)
1319 {
1320 	list_del_init(&rt_se->run_list);
1321 
1322 	if (list_empty(array->queue + rt_se_prio(rt_se)))
1323 		__clear_bit(rt_se_prio(rt_se), array->bitmap);
1324 
1325 	rt_se->on_list = 0;
1326 }
1327 
1328 static inline struct sched_statistics *
__schedstats_from_rt_se(struct sched_rt_entity * rt_se)1329 __schedstats_from_rt_se(struct sched_rt_entity *rt_se)
1330 {
1331 #ifdef CONFIG_RT_GROUP_SCHED
1332 	/* schedstats is not supported for rt group. */
1333 	if (!rt_entity_is_task(rt_se))
1334 		return NULL;
1335 #endif
1336 
1337 	return &rt_task_of(rt_se)->stats;
1338 }
1339 
1340 static inline void
update_stats_wait_start_rt(struct rt_rq * rt_rq,struct sched_rt_entity * rt_se)1341 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1342 {
1343 	struct sched_statistics *stats;
1344 	struct task_struct *p = NULL;
1345 
1346 	if (!schedstat_enabled())
1347 		return;
1348 
1349 	if (rt_entity_is_task(rt_se))
1350 		p = rt_task_of(rt_se);
1351 
1352 	stats = __schedstats_from_rt_se(rt_se);
1353 	if (!stats)
1354 		return;
1355 
1356 	__update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats);
1357 }
1358 
1359 static inline void
update_stats_enqueue_sleeper_rt(struct rt_rq * rt_rq,struct sched_rt_entity * rt_se)1360 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1361 {
1362 	struct sched_statistics *stats;
1363 	struct task_struct *p = NULL;
1364 
1365 	if (!schedstat_enabled())
1366 		return;
1367 
1368 	if (rt_entity_is_task(rt_se))
1369 		p = rt_task_of(rt_se);
1370 
1371 	stats = __schedstats_from_rt_se(rt_se);
1372 	if (!stats)
1373 		return;
1374 
1375 	__update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats);
1376 }
1377 
1378 static inline void
update_stats_enqueue_rt(struct rt_rq * rt_rq,struct sched_rt_entity * rt_se,int flags)1379 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
1380 			int flags)
1381 {
1382 	if (!schedstat_enabled())
1383 		return;
1384 
1385 	if (flags & ENQUEUE_WAKEUP)
1386 		update_stats_enqueue_sleeper_rt(rt_rq, rt_se);
1387 }
1388 
1389 static inline void
update_stats_wait_end_rt(struct rt_rq * rt_rq,struct sched_rt_entity * rt_se)1390 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1391 {
1392 	struct sched_statistics *stats;
1393 	struct task_struct *p = NULL;
1394 
1395 	if (!schedstat_enabled())
1396 		return;
1397 
1398 	if (rt_entity_is_task(rt_se))
1399 		p = rt_task_of(rt_se);
1400 
1401 	stats = __schedstats_from_rt_se(rt_se);
1402 	if (!stats)
1403 		return;
1404 
1405 	__update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats);
1406 }
1407 
1408 static inline void
update_stats_dequeue_rt(struct rt_rq * rt_rq,struct sched_rt_entity * rt_se,int flags)1409 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
1410 			int flags)
1411 {
1412 	struct task_struct *p = NULL;
1413 
1414 	if (!schedstat_enabled())
1415 		return;
1416 
1417 	if (rt_entity_is_task(rt_se))
1418 		p = rt_task_of(rt_se);
1419 
1420 	if ((flags & DEQUEUE_SLEEP) && p) {
1421 		unsigned int state;
1422 
1423 		state = READ_ONCE(p->__state);
1424 		if (state & TASK_INTERRUPTIBLE)
1425 			__schedstat_set(p->stats.sleep_start,
1426 					rq_clock(rq_of_rt_rq(rt_rq)));
1427 
1428 		if (state & TASK_UNINTERRUPTIBLE)
1429 			__schedstat_set(p->stats.block_start,
1430 					rq_clock(rq_of_rt_rq(rt_rq)));
1431 	}
1432 }
1433 
__enqueue_rt_entity(struct sched_rt_entity * rt_se,unsigned int flags)1434 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1435 {
1436 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1437 	struct rt_prio_array *array = &rt_rq->active;
1438 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1439 	struct list_head *queue = array->queue + rt_se_prio(rt_se);
1440 
1441 	/*
1442 	 * Don't enqueue the group if its throttled, or when empty.
1443 	 * The latter is a consequence of the former when a child group
1444 	 * get throttled and the current group doesn't have any other
1445 	 * active members.
1446 	 */
1447 	if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) {
1448 		if (rt_se->on_list)
1449 			__delist_rt_entity(rt_se, array);
1450 		return;
1451 	}
1452 
1453 	if (move_entity(flags)) {
1454 		WARN_ON_ONCE(rt_se->on_list);
1455 		if (flags & ENQUEUE_HEAD)
1456 			list_add(&rt_se->run_list, queue);
1457 		else
1458 			list_add_tail(&rt_se->run_list, queue);
1459 
1460 		__set_bit(rt_se_prio(rt_se), array->bitmap);
1461 		rt_se->on_list = 1;
1462 	}
1463 	rt_se->on_rq = 1;
1464 
1465 	inc_rt_tasks(rt_se, rt_rq);
1466 }
1467 
__dequeue_rt_entity(struct sched_rt_entity * rt_se,unsigned int flags)1468 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1469 {
1470 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1471 	struct rt_prio_array *array = &rt_rq->active;
1472 
1473 	if (move_entity(flags)) {
1474 		WARN_ON_ONCE(!rt_se->on_list);
1475 		__delist_rt_entity(rt_se, array);
1476 	}
1477 	rt_se->on_rq = 0;
1478 
1479 	dec_rt_tasks(rt_se, rt_rq);
1480 }
1481 
1482 /*
1483  * Because the prio of an upper entry depends on the lower
1484  * entries, we must remove entries top - down.
1485  */
dequeue_rt_stack(struct sched_rt_entity * rt_se,unsigned int flags)1486 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
1487 {
1488 	struct sched_rt_entity *back = NULL;
1489 	unsigned int rt_nr_running;
1490 
1491 	for_each_sched_rt_entity(rt_se) {
1492 		rt_se->back = back;
1493 		back = rt_se;
1494 	}
1495 
1496 	rt_nr_running = rt_rq_of_se(back)->rt_nr_running;
1497 
1498 	for (rt_se = back; rt_se; rt_se = rt_se->back) {
1499 		if (on_rt_rq(rt_se))
1500 			__dequeue_rt_entity(rt_se, flags);
1501 	}
1502 
1503 	dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running);
1504 }
1505 
enqueue_rt_entity(struct sched_rt_entity * rt_se,unsigned int flags)1506 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1507 {
1508 	struct rq *rq = rq_of_rt_se(rt_se);
1509 
1510 	update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1511 
1512 	dequeue_rt_stack(rt_se, flags);
1513 	for_each_sched_rt_entity(rt_se)
1514 		__enqueue_rt_entity(rt_se, flags);
1515 	enqueue_top_rt_rq(&rq->rt);
1516 }
1517 
dequeue_rt_entity(struct sched_rt_entity * rt_se,unsigned int flags)1518 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1519 {
1520 	struct rq *rq = rq_of_rt_se(rt_se);
1521 
1522 	update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1523 
1524 	dequeue_rt_stack(rt_se, flags);
1525 
1526 	for_each_sched_rt_entity(rt_se) {
1527 		struct rt_rq *rt_rq = group_rt_rq(rt_se);
1528 
1529 		if (rt_rq && rt_rq->rt_nr_running)
1530 			__enqueue_rt_entity(rt_se, flags);
1531 	}
1532 	enqueue_top_rt_rq(&rq->rt);
1533 }
1534 
1535 /*
1536  * Adding/removing a task to/from a priority array:
1537  */
1538 static void
enqueue_task_rt(struct rq * rq,struct task_struct * p,int flags)1539 enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
1540 {
1541 	struct sched_rt_entity *rt_se = &p->rt;
1542 
1543 	if (flags & ENQUEUE_WAKEUP)
1544 		rt_se->timeout = 0;
1545 
1546 	check_schedstat_required();
1547 	update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se);
1548 
1549 	enqueue_rt_entity(rt_se, flags);
1550 
1551 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
1552 		enqueue_pushable_task(rq, p);
1553 }
1554 
dequeue_task_rt(struct rq * rq,struct task_struct * p,int flags)1555 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
1556 {
1557 	struct sched_rt_entity *rt_se = &p->rt;
1558 
1559 	update_curr_rt(rq);
1560 	dequeue_rt_entity(rt_se, flags);
1561 
1562 	dequeue_pushable_task(rq, p);
1563 }
1564 
1565 /*
1566  * Put task to the head or the end of the run list without the overhead of
1567  * dequeue followed by enqueue.
1568  */
1569 static void
requeue_rt_entity(struct rt_rq * rt_rq,struct sched_rt_entity * rt_se,int head)1570 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
1571 {
1572 	if (on_rt_rq(rt_se)) {
1573 		struct rt_prio_array *array = &rt_rq->active;
1574 		struct list_head *queue = array->queue + rt_se_prio(rt_se);
1575 
1576 		if (head)
1577 			list_move(&rt_se->run_list, queue);
1578 		else
1579 			list_move_tail(&rt_se->run_list, queue);
1580 	}
1581 }
1582 
requeue_task_rt(struct rq * rq,struct task_struct * p,int head)1583 static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
1584 {
1585 	struct sched_rt_entity *rt_se = &p->rt;
1586 	struct rt_rq *rt_rq;
1587 
1588 	for_each_sched_rt_entity(rt_se) {
1589 		rt_rq = rt_rq_of_se(rt_se);
1590 		requeue_rt_entity(rt_rq, rt_se, head);
1591 	}
1592 }
1593 
yield_task_rt(struct rq * rq)1594 static void yield_task_rt(struct rq *rq)
1595 {
1596 	requeue_task_rt(rq, rq->curr, 0);
1597 }
1598 
1599 #ifdef CONFIG_SMP
1600 static int find_lowest_rq(struct task_struct *task);
1601 
1602 static int
select_task_rq_rt(struct task_struct * p,int cpu,int flags)1603 select_task_rq_rt(struct task_struct *p, int cpu, int flags)
1604 {
1605 	struct task_struct *curr;
1606 	struct rq *rq;
1607 	bool test;
1608 
1609 	/* For anything but wake ups, just return the task_cpu */
1610 	if (!(flags & (WF_TTWU | WF_FORK)))
1611 		goto out;
1612 
1613 	rq = cpu_rq(cpu);
1614 
1615 	rcu_read_lock();
1616 	curr = READ_ONCE(rq->curr); /* unlocked access */
1617 
1618 	/*
1619 	 * If the current task on @p's runqueue is an RT task, then
1620 	 * try to see if we can wake this RT task up on another
1621 	 * runqueue. Otherwise simply start this RT task
1622 	 * on its current runqueue.
1623 	 *
1624 	 * We want to avoid overloading runqueues. If the woken
1625 	 * task is a higher priority, then it will stay on this CPU
1626 	 * and the lower prio task should be moved to another CPU.
1627 	 * Even though this will probably make the lower prio task
1628 	 * lose its cache, we do not want to bounce a higher task
1629 	 * around just because it gave up its CPU, perhaps for a
1630 	 * lock?
1631 	 *
1632 	 * For equal prio tasks, we just let the scheduler sort it out.
1633 	 *
1634 	 * Otherwise, just let it ride on the affined RQ and the
1635 	 * post-schedule router will push the preempted task away
1636 	 *
1637 	 * This test is optimistic, if we get it wrong the load-balancer
1638 	 * will have to sort it out.
1639 	 *
1640 	 * We take into account the capacity of the CPU to ensure it fits the
1641 	 * requirement of the task - which is only important on heterogeneous
1642 	 * systems like big.LITTLE.
1643 	 */
1644 	test = curr &&
1645 	       unlikely(rt_task(curr)) &&
1646 	       (curr->nr_cpus_allowed < 2 || curr->prio <= p->prio);
1647 
1648 	if (test || !rt_task_fits_capacity(p, cpu)) {
1649 		int target = find_lowest_rq(p);
1650 
1651 		/*
1652 		 * Bail out if we were forcing a migration to find a better
1653 		 * fitting CPU but our search failed.
1654 		 */
1655 		if (!test && target != -1 && !rt_task_fits_capacity(p, target))
1656 			goto out_unlock;
1657 
1658 		/*
1659 		 * Don't bother moving it if the destination CPU is
1660 		 * not running a lower priority task.
1661 		 */
1662 		if (target != -1 &&
1663 		    p->prio < cpu_rq(target)->rt.highest_prio.curr)
1664 			cpu = target;
1665 	}
1666 
1667 out_unlock:
1668 	rcu_read_unlock();
1669 
1670 out:
1671 	return cpu;
1672 }
1673 
check_preempt_equal_prio(struct rq * rq,struct task_struct * p)1674 static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
1675 {
1676 	/*
1677 	 * Current can't be migrated, useless to reschedule,
1678 	 * let's hope p can move out.
1679 	 */
1680 	if (rq->curr->nr_cpus_allowed == 1 ||
1681 	    !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
1682 		return;
1683 
1684 	/*
1685 	 * p is migratable, so let's not schedule it and
1686 	 * see if it is pushed or pulled somewhere else.
1687 	 */
1688 	if (p->nr_cpus_allowed != 1 &&
1689 	    cpupri_find(&rq->rd->cpupri, p, NULL))
1690 		return;
1691 
1692 	/*
1693 	 * There appear to be other CPUs that can accept
1694 	 * the current task but none can run 'p', so lets reschedule
1695 	 * to try and push the current task away:
1696 	 */
1697 	requeue_task_rt(rq, p, 1);
1698 	resched_curr(rq);
1699 }
1700 
balance_rt(struct rq * rq,struct task_struct * p,struct rq_flags * rf)1701 static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
1702 {
1703 	if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
1704 		/*
1705 		 * This is OK, because current is on_cpu, which avoids it being
1706 		 * picked for load-balance and preemption/IRQs are still
1707 		 * disabled avoiding further scheduler activity on it and we've
1708 		 * not yet started the picking loop.
1709 		 */
1710 		rq_unpin_lock(rq, rf);
1711 		pull_rt_task(rq);
1712 		rq_repin_lock(rq, rf);
1713 	}
1714 
1715 	return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq);
1716 }
1717 #endif /* CONFIG_SMP */
1718 
1719 /*
1720  * Preempt the current task with a newly woken task if needed:
1721  */
check_preempt_curr_rt(struct rq * rq,struct task_struct * p,int flags)1722 static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
1723 {
1724 	if (p->prio < rq->curr->prio) {
1725 		resched_curr(rq);
1726 		return;
1727 	}
1728 
1729 #ifdef CONFIG_SMP
1730 	/*
1731 	 * If:
1732 	 *
1733 	 * - the newly woken task is of equal priority to the current task
1734 	 * - the newly woken task is non-migratable while current is migratable
1735 	 * - current will be preempted on the next reschedule
1736 	 *
1737 	 * we should check to see if current can readily move to a different
1738 	 * cpu.  If so, we will reschedule to allow the push logic to try
1739 	 * to move current somewhere else, making room for our non-migratable
1740 	 * task.
1741 	 */
1742 	if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))
1743 		check_preempt_equal_prio(rq, p);
1744 #endif
1745 }
1746 
set_next_task_rt(struct rq * rq,struct task_struct * p,bool first)1747 static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
1748 {
1749 	struct sched_rt_entity *rt_se = &p->rt;
1750 	struct rt_rq *rt_rq = &rq->rt;
1751 
1752 	p->se.exec_start = rq_clock_task(rq);
1753 	if (on_rt_rq(&p->rt))
1754 		update_stats_wait_end_rt(rt_rq, rt_se);
1755 
1756 	/* The running task is never eligible for pushing */
1757 	dequeue_pushable_task(rq, p);
1758 
1759 	if (!first)
1760 		return;
1761 
1762 	/*
1763 	 * If prev task was rt, put_prev_task() has already updated the
1764 	 * utilization. We only care of the case where we start to schedule a
1765 	 * rt task
1766 	 */
1767 	if (rq->curr->sched_class != &rt_sched_class)
1768 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
1769 
1770 	rt_queue_push_tasks(rq);
1771 }
1772 
pick_next_rt_entity(struct rt_rq * rt_rq)1773 static struct sched_rt_entity *pick_next_rt_entity(struct rt_rq *rt_rq)
1774 {
1775 	struct rt_prio_array *array = &rt_rq->active;
1776 	struct sched_rt_entity *next = NULL;
1777 	struct list_head *queue;
1778 	int idx;
1779 
1780 	idx = sched_find_first_bit(array->bitmap);
1781 	BUG_ON(idx >= MAX_RT_PRIO);
1782 
1783 	queue = array->queue + idx;
1784 	if (SCHED_WARN_ON(list_empty(queue)))
1785 		return NULL;
1786 	next = list_entry(queue->next, struct sched_rt_entity, run_list);
1787 
1788 	return next;
1789 }
1790 
_pick_next_task_rt(struct rq * rq)1791 static struct task_struct *_pick_next_task_rt(struct rq *rq)
1792 {
1793 	struct sched_rt_entity *rt_se;
1794 	struct rt_rq *rt_rq  = &rq->rt;
1795 
1796 	do {
1797 		rt_se = pick_next_rt_entity(rt_rq);
1798 		if (unlikely(!rt_se))
1799 			return NULL;
1800 		rt_rq = group_rt_rq(rt_se);
1801 	} while (rt_rq);
1802 
1803 	return rt_task_of(rt_se);
1804 }
1805 
pick_task_rt(struct rq * rq)1806 static struct task_struct *pick_task_rt(struct rq *rq)
1807 {
1808 	struct task_struct *p;
1809 
1810 	if (!sched_rt_runnable(rq))
1811 		return NULL;
1812 
1813 	p = _pick_next_task_rt(rq);
1814 
1815 	return p;
1816 }
1817 
pick_next_task_rt(struct rq * rq)1818 static struct task_struct *pick_next_task_rt(struct rq *rq)
1819 {
1820 	struct task_struct *p = pick_task_rt(rq);
1821 
1822 	if (p)
1823 		set_next_task_rt(rq, p, true);
1824 
1825 	return p;
1826 }
1827 
put_prev_task_rt(struct rq * rq,struct task_struct * p)1828 static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
1829 {
1830 	struct sched_rt_entity *rt_se = &p->rt;
1831 	struct rt_rq *rt_rq = &rq->rt;
1832 
1833 	if (on_rt_rq(&p->rt))
1834 		update_stats_wait_start_rt(rt_rq, rt_se);
1835 
1836 	update_curr_rt(rq);
1837 
1838 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
1839 
1840 	/*
1841 	 * The previous task needs to be made eligible for pushing
1842 	 * if it is still active
1843 	 */
1844 	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
1845 		enqueue_pushable_task(rq, p);
1846 }
1847 
1848 #ifdef CONFIG_SMP
1849 
1850 /* Only try algorithms three times */
1851 #define RT_MAX_TRIES 3
1852 
pick_rt_task(struct rq * rq,struct task_struct * p,int cpu)1853 static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
1854 {
1855 	if (!task_on_cpu(rq, p) &&
1856 	    cpumask_test_cpu(cpu, &p->cpus_mask))
1857 		return 1;
1858 
1859 	return 0;
1860 }
1861 
1862 /*
1863  * Return the highest pushable rq's task, which is suitable to be executed
1864  * on the CPU, NULL otherwise
1865  */
pick_highest_pushable_task(struct rq * rq,int cpu)1866 static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
1867 {
1868 	struct plist_head *head = &rq->rt.pushable_tasks;
1869 	struct task_struct *p;
1870 
1871 	if (!has_pushable_tasks(rq))
1872 		return NULL;
1873 
1874 	plist_for_each_entry(p, head, pushable_tasks) {
1875 		if (pick_rt_task(rq, p, cpu))
1876 			return p;
1877 	}
1878 
1879 	return NULL;
1880 }
1881 
1882 static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
1883 
find_lowest_rq(struct task_struct * task)1884 static int find_lowest_rq(struct task_struct *task)
1885 {
1886 	struct sched_domain *sd;
1887 	struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
1888 	int this_cpu = smp_processor_id();
1889 	int cpu      = task_cpu(task);
1890 	int ret;
1891 
1892 	/* Make sure the mask is initialized first */
1893 	if (unlikely(!lowest_mask))
1894 		return -1;
1895 
1896 	if (task->nr_cpus_allowed == 1)
1897 		return -1; /* No other targets possible */
1898 
1899 	/*
1900 	 * If we're on asym system ensure we consider the different capacities
1901 	 * of the CPUs when searching for the lowest_mask.
1902 	 */
1903 	if (sched_asym_cpucap_active()) {
1904 
1905 		ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri,
1906 					  task, lowest_mask,
1907 					  rt_task_fits_capacity);
1908 	} else {
1909 
1910 		ret = cpupri_find(&task_rq(task)->rd->cpupri,
1911 				  task, lowest_mask);
1912 	}
1913 
1914 	if (!ret)
1915 		return -1; /* No targets found */
1916 
1917 	/*
1918 	 * At this point we have built a mask of CPUs representing the
1919 	 * lowest priority tasks in the system.  Now we want to elect
1920 	 * the best one based on our affinity and topology.
1921 	 *
1922 	 * We prioritize the last CPU that the task executed on since
1923 	 * it is most likely cache-hot in that location.
1924 	 */
1925 	if (cpumask_test_cpu(cpu, lowest_mask))
1926 		return cpu;
1927 
1928 	/*
1929 	 * Otherwise, we consult the sched_domains span maps to figure
1930 	 * out which CPU is logically closest to our hot cache data.
1931 	 */
1932 	if (!cpumask_test_cpu(this_cpu, lowest_mask))
1933 		this_cpu = -1; /* Skip this_cpu opt if not among lowest */
1934 
1935 	rcu_read_lock();
1936 	for_each_domain(cpu, sd) {
1937 		if (sd->flags & SD_WAKE_AFFINE) {
1938 			int best_cpu;
1939 
1940 			/*
1941 			 * "this_cpu" is cheaper to preempt than a
1942 			 * remote processor.
1943 			 */
1944 			if (this_cpu != -1 &&
1945 			    cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
1946 				rcu_read_unlock();
1947 				return this_cpu;
1948 			}
1949 
1950 			best_cpu = cpumask_any_and_distribute(lowest_mask,
1951 							      sched_domain_span(sd));
1952 			if (best_cpu < nr_cpu_ids) {
1953 				rcu_read_unlock();
1954 				return best_cpu;
1955 			}
1956 		}
1957 	}
1958 	rcu_read_unlock();
1959 
1960 	/*
1961 	 * And finally, if there were no matches within the domains
1962 	 * just give the caller *something* to work with from the compatible
1963 	 * locations.
1964 	 */
1965 	if (this_cpu != -1)
1966 		return this_cpu;
1967 
1968 	cpu = cpumask_any_distribute(lowest_mask);
1969 	if (cpu < nr_cpu_ids)
1970 		return cpu;
1971 
1972 	return -1;
1973 }
1974 
1975 /* Will lock the rq it finds */
find_lock_lowest_rq(struct task_struct * task,struct rq * rq)1976 static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
1977 {
1978 	struct rq *lowest_rq = NULL;
1979 	int tries;
1980 	int cpu;
1981 
1982 	for (tries = 0; tries < RT_MAX_TRIES; tries++) {
1983 		cpu = find_lowest_rq(task);
1984 
1985 		if ((cpu == -1) || (cpu == rq->cpu))
1986 			break;
1987 
1988 		lowest_rq = cpu_rq(cpu);
1989 
1990 		if (lowest_rq->rt.highest_prio.curr <= task->prio) {
1991 			/*
1992 			 * Target rq has tasks of equal or higher priority,
1993 			 * retrying does not release any lock and is unlikely
1994 			 * to yield a different result.
1995 			 */
1996 			lowest_rq = NULL;
1997 			break;
1998 		}
1999 
2000 		/* if the prio of this runqueue changed, try again */
2001 		if (double_lock_balance(rq, lowest_rq)) {
2002 			/*
2003 			 * We had to unlock the run queue. In
2004 			 * the mean time, task could have
2005 			 * migrated already or had its affinity changed.
2006 			 * Also make sure that it wasn't scheduled on its rq.
2007 			 * It is possible the task was scheduled, set
2008 			 * "migrate_disabled" and then got preempted, so we must
2009 			 * check the task migration disable flag here too.
2010 			 */
2011 			if (unlikely(task_rq(task) != rq ||
2012 				     !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
2013 				     task_on_cpu(rq, task) ||
2014 				     !rt_task(task) ||
2015 				     is_migration_disabled(task) ||
2016 				     !task_on_rq_queued(task))) {
2017 
2018 				double_unlock_balance(rq, lowest_rq);
2019 				lowest_rq = NULL;
2020 				break;
2021 			}
2022 		}
2023 
2024 		/* If this rq is still suitable use it. */
2025 		if (lowest_rq->rt.highest_prio.curr > task->prio)
2026 			break;
2027 
2028 		/* try again */
2029 		double_unlock_balance(rq, lowest_rq);
2030 		lowest_rq = NULL;
2031 	}
2032 
2033 	return lowest_rq;
2034 }
2035 
pick_next_pushable_task(struct rq * rq)2036 static struct task_struct *pick_next_pushable_task(struct rq *rq)
2037 {
2038 	struct task_struct *p;
2039 
2040 	if (!has_pushable_tasks(rq))
2041 		return NULL;
2042 
2043 	p = plist_first_entry(&rq->rt.pushable_tasks,
2044 			      struct task_struct, pushable_tasks);
2045 
2046 	BUG_ON(rq->cpu != task_cpu(p));
2047 	BUG_ON(task_current(rq, p));
2048 	BUG_ON(p->nr_cpus_allowed <= 1);
2049 
2050 	BUG_ON(!task_on_rq_queued(p));
2051 	BUG_ON(!rt_task(p));
2052 
2053 	return p;
2054 }
2055 
2056 /*
2057  * If the current CPU has more than one RT task, see if the non
2058  * running task can migrate over to a CPU that is running a task
2059  * of lesser priority.
2060  */
push_rt_task(struct rq * rq,bool pull)2061 static int push_rt_task(struct rq *rq, bool pull)
2062 {
2063 	struct task_struct *next_task;
2064 	struct rq *lowest_rq;
2065 	int ret = 0;
2066 
2067 	if (!rq->rt.overloaded)
2068 		return 0;
2069 
2070 	next_task = pick_next_pushable_task(rq);
2071 	if (!next_task)
2072 		return 0;
2073 
2074 retry:
2075 	/*
2076 	 * It's possible that the next_task slipped in of
2077 	 * higher priority than current. If that's the case
2078 	 * just reschedule current.
2079 	 */
2080 	if (unlikely(next_task->prio < rq->curr->prio)) {
2081 		resched_curr(rq);
2082 		return 0;
2083 	}
2084 
2085 	if (is_migration_disabled(next_task)) {
2086 		struct task_struct *push_task = NULL;
2087 		int cpu;
2088 
2089 		if (!pull || rq->push_busy)
2090 			return 0;
2091 
2092 		/*
2093 		 * Invoking find_lowest_rq() on anything but an RT task doesn't
2094 		 * make sense. Per the above priority check, curr has to
2095 		 * be of higher priority than next_task, so no need to
2096 		 * reschedule when bailing out.
2097 		 *
2098 		 * Note that the stoppers are masqueraded as SCHED_FIFO
2099 		 * (cf. sched_set_stop_task()), so we can't rely on rt_task().
2100 		 */
2101 		if (rq->curr->sched_class != &rt_sched_class)
2102 			return 0;
2103 
2104 		cpu = find_lowest_rq(rq->curr);
2105 		if (cpu == -1 || cpu == rq->cpu)
2106 			return 0;
2107 
2108 		/*
2109 		 * Given we found a CPU with lower priority than @next_task,
2110 		 * therefore it should be running. However we cannot migrate it
2111 		 * to this other CPU, instead attempt to push the current
2112 		 * running task on this CPU away.
2113 		 */
2114 		push_task = get_push_task(rq);
2115 		if (push_task) {
2116 			preempt_disable();
2117 			raw_spin_rq_unlock(rq);
2118 			stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2119 					    push_task, &rq->push_work);
2120 			preempt_enable();
2121 			raw_spin_rq_lock(rq);
2122 		}
2123 
2124 		return 0;
2125 	}
2126 
2127 	if (WARN_ON(next_task == rq->curr))
2128 		return 0;
2129 
2130 	/* We might release rq lock */
2131 	get_task_struct(next_task);
2132 
2133 	/* find_lock_lowest_rq locks the rq if found */
2134 	lowest_rq = find_lock_lowest_rq(next_task, rq);
2135 	if (!lowest_rq) {
2136 		struct task_struct *task;
2137 		/*
2138 		 * find_lock_lowest_rq releases rq->lock
2139 		 * so it is possible that next_task has migrated.
2140 		 *
2141 		 * We need to make sure that the task is still on the same
2142 		 * run-queue and is also still the next task eligible for
2143 		 * pushing.
2144 		 */
2145 		task = pick_next_pushable_task(rq);
2146 		if (task == next_task) {
2147 			/*
2148 			 * The task hasn't migrated, and is still the next
2149 			 * eligible task, but we failed to find a run-queue
2150 			 * to push it to.  Do not retry in this case, since
2151 			 * other CPUs will pull from us when ready.
2152 			 */
2153 			goto out;
2154 		}
2155 
2156 		if (!task)
2157 			/* No more tasks, just exit */
2158 			goto out;
2159 
2160 		/*
2161 		 * Something has shifted, try again.
2162 		 */
2163 		put_task_struct(next_task);
2164 		next_task = task;
2165 		goto retry;
2166 	}
2167 
2168 	deactivate_task(rq, next_task, 0);
2169 	set_task_cpu(next_task, lowest_rq->cpu);
2170 	activate_task(lowest_rq, next_task, 0);
2171 	resched_curr(lowest_rq);
2172 	ret = 1;
2173 
2174 	double_unlock_balance(rq, lowest_rq);
2175 out:
2176 	put_task_struct(next_task);
2177 
2178 	return ret;
2179 }
2180 
push_rt_tasks(struct rq * rq)2181 static void push_rt_tasks(struct rq *rq)
2182 {
2183 	/* push_rt_task will return true if it moved an RT */
2184 	while (push_rt_task(rq, false))
2185 		;
2186 }
2187 
2188 #ifdef HAVE_RT_PUSH_IPI
2189 
2190 /*
2191  * When a high priority task schedules out from a CPU and a lower priority
2192  * task is scheduled in, a check is made to see if there's any RT tasks
2193  * on other CPUs that are waiting to run because a higher priority RT task
2194  * is currently running on its CPU. In this case, the CPU with multiple RT
2195  * tasks queued on it (overloaded) needs to be notified that a CPU has opened
2196  * up that may be able to run one of its non-running queued RT tasks.
2197  *
2198  * All CPUs with overloaded RT tasks need to be notified as there is currently
2199  * no way to know which of these CPUs have the highest priority task waiting
2200  * to run. Instead of trying to take a spinlock on each of these CPUs,
2201  * which has shown to cause large latency when done on machines with many
2202  * CPUs, sending an IPI to the CPUs to have them push off the overloaded
2203  * RT tasks waiting to run.
2204  *
2205  * Just sending an IPI to each of the CPUs is also an issue, as on large
2206  * count CPU machines, this can cause an IPI storm on a CPU, especially
2207  * if its the only CPU with multiple RT tasks queued, and a large number
2208  * of CPUs scheduling a lower priority task at the same time.
2209  *
2210  * Each root domain has its own irq work function that can iterate over
2211  * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT
2212  * task must be checked if there's one or many CPUs that are lowering
2213  * their priority, there's a single irq work iterator that will try to
2214  * push off RT tasks that are waiting to run.
2215  *
2216  * When a CPU schedules a lower priority task, it will kick off the
2217  * irq work iterator that will jump to each CPU with overloaded RT tasks.
2218  * As it only takes the first CPU that schedules a lower priority task
2219  * to start the process, the rto_start variable is incremented and if
2220  * the atomic result is one, then that CPU will try to take the rto_lock.
2221  * This prevents high contention on the lock as the process handles all
2222  * CPUs scheduling lower priority tasks.
2223  *
2224  * All CPUs that are scheduling a lower priority task will increment the
2225  * rt_loop_next variable. This will make sure that the irq work iterator
2226  * checks all RT overloaded CPUs whenever a CPU schedules a new lower
2227  * priority task, even if the iterator is in the middle of a scan. Incrementing
2228  * the rt_loop_next will cause the iterator to perform another scan.
2229  *
2230  */
rto_next_cpu(struct root_domain * rd)2231 static int rto_next_cpu(struct root_domain *rd)
2232 {
2233 	int next;
2234 	int cpu;
2235 
2236 	/*
2237 	 * When starting the IPI RT pushing, the rto_cpu is set to -1,
2238 	 * rt_next_cpu() will simply return the first CPU found in
2239 	 * the rto_mask.
2240 	 *
2241 	 * If rto_next_cpu() is called with rto_cpu is a valid CPU, it
2242 	 * will return the next CPU found in the rto_mask.
2243 	 *
2244 	 * If there are no more CPUs left in the rto_mask, then a check is made
2245 	 * against rto_loop and rto_loop_next. rto_loop is only updated with
2246 	 * the rto_lock held, but any CPU may increment the rto_loop_next
2247 	 * without any locking.
2248 	 */
2249 	for (;;) {
2250 
2251 		/* When rto_cpu is -1 this acts like cpumask_first() */
2252 		cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);
2253 
2254 		rd->rto_cpu = cpu;
2255 
2256 		if (cpu < nr_cpu_ids)
2257 			return cpu;
2258 
2259 		rd->rto_cpu = -1;
2260 
2261 		/*
2262 		 * ACQUIRE ensures we see the @rto_mask changes
2263 		 * made prior to the @next value observed.
2264 		 *
2265 		 * Matches WMB in rt_set_overload().
2266 		 */
2267 		next = atomic_read_acquire(&rd->rto_loop_next);
2268 
2269 		if (rd->rto_loop == next)
2270 			break;
2271 
2272 		rd->rto_loop = next;
2273 	}
2274 
2275 	return -1;
2276 }
2277 
rto_start_trylock(atomic_t * v)2278 static inline bool rto_start_trylock(atomic_t *v)
2279 {
2280 	return !atomic_cmpxchg_acquire(v, 0, 1);
2281 }
2282 
rto_start_unlock(atomic_t * v)2283 static inline void rto_start_unlock(atomic_t *v)
2284 {
2285 	atomic_set_release(v, 0);
2286 }
2287 
tell_cpu_to_push(struct rq * rq)2288 static void tell_cpu_to_push(struct rq *rq)
2289 {
2290 	int cpu = -1;
2291 
2292 	/* Keep the loop going if the IPI is currently active */
2293 	atomic_inc(&rq->rd->rto_loop_next);
2294 
2295 	/* Only one CPU can initiate a loop at a time */
2296 	if (!rto_start_trylock(&rq->rd->rto_loop_start))
2297 		return;
2298 
2299 	raw_spin_lock(&rq->rd->rto_lock);
2300 
2301 	/*
2302 	 * The rto_cpu is updated under the lock, if it has a valid CPU
2303 	 * then the IPI is still running and will continue due to the
2304 	 * update to loop_next, and nothing needs to be done here.
2305 	 * Otherwise it is finishing up and an ipi needs to be sent.
2306 	 */
2307 	if (rq->rd->rto_cpu < 0)
2308 		cpu = rto_next_cpu(rq->rd);
2309 
2310 	raw_spin_unlock(&rq->rd->rto_lock);
2311 
2312 	rto_start_unlock(&rq->rd->rto_loop_start);
2313 
2314 	if (cpu >= 0) {
2315 		/* Make sure the rd does not get freed while pushing */
2316 		sched_get_rd(rq->rd);
2317 		irq_work_queue_on(&rq->rd->rto_push_work, cpu);
2318 	}
2319 }
2320 
2321 /* Called from hardirq context */
rto_push_irq_work_func(struct irq_work * work)2322 void rto_push_irq_work_func(struct irq_work *work)
2323 {
2324 	struct root_domain *rd =
2325 		container_of(work, struct root_domain, rto_push_work);
2326 	struct rq *rq;
2327 	int cpu;
2328 
2329 	rq = this_rq();
2330 
2331 	/*
2332 	 * We do not need to grab the lock to check for has_pushable_tasks.
2333 	 * When it gets updated, a check is made if a push is possible.
2334 	 */
2335 	if (has_pushable_tasks(rq)) {
2336 		raw_spin_rq_lock(rq);
2337 		while (push_rt_task(rq, true))
2338 			;
2339 		raw_spin_rq_unlock(rq);
2340 	}
2341 
2342 	raw_spin_lock(&rd->rto_lock);
2343 
2344 	/* Pass the IPI to the next rt overloaded queue */
2345 	cpu = rto_next_cpu(rd);
2346 
2347 	raw_spin_unlock(&rd->rto_lock);
2348 
2349 	if (cpu < 0) {
2350 		sched_put_rd(rd);
2351 		return;
2352 	}
2353 
2354 	/* Try the next RT overloaded CPU */
2355 	irq_work_queue_on(&rd->rto_push_work, cpu);
2356 }
2357 #endif /* HAVE_RT_PUSH_IPI */
2358 
pull_rt_task(struct rq * this_rq)2359 static void pull_rt_task(struct rq *this_rq)
2360 {
2361 	int this_cpu = this_rq->cpu, cpu;
2362 	bool resched = false;
2363 	struct task_struct *p, *push_task;
2364 	struct rq *src_rq;
2365 	int rt_overload_count = rt_overloaded(this_rq);
2366 
2367 	if (likely(!rt_overload_count))
2368 		return;
2369 
2370 	/*
2371 	 * Match the barrier from rt_set_overloaded; this guarantees that if we
2372 	 * see overloaded we must also see the rto_mask bit.
2373 	 */
2374 	smp_rmb();
2375 
2376 	/* If we are the only overloaded CPU do nothing */
2377 	if (rt_overload_count == 1 &&
2378 	    cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))
2379 		return;
2380 
2381 #ifdef HAVE_RT_PUSH_IPI
2382 	if (sched_feat(RT_PUSH_IPI)) {
2383 		tell_cpu_to_push(this_rq);
2384 		return;
2385 	}
2386 #endif
2387 
2388 	for_each_cpu(cpu, this_rq->rd->rto_mask) {
2389 		if (this_cpu == cpu)
2390 			continue;
2391 
2392 		src_rq = cpu_rq(cpu);
2393 
2394 		/*
2395 		 * Don't bother taking the src_rq->lock if the next highest
2396 		 * task is known to be lower-priority than our current task.
2397 		 * This may look racy, but if this value is about to go
2398 		 * logically higher, the src_rq will push this task away.
2399 		 * And if its going logically lower, we do not care
2400 		 */
2401 		if (src_rq->rt.highest_prio.next >=
2402 		    this_rq->rt.highest_prio.curr)
2403 			continue;
2404 
2405 		/*
2406 		 * We can potentially drop this_rq's lock in
2407 		 * double_lock_balance, and another CPU could
2408 		 * alter this_rq
2409 		 */
2410 		push_task = NULL;
2411 		double_lock_balance(this_rq, src_rq);
2412 
2413 		/*
2414 		 * We can pull only a task, which is pushable
2415 		 * on its rq, and no others.
2416 		 */
2417 		p = pick_highest_pushable_task(src_rq, this_cpu);
2418 
2419 		/*
2420 		 * Do we have an RT task that preempts
2421 		 * the to-be-scheduled task?
2422 		 */
2423 		if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
2424 			WARN_ON(p == src_rq->curr);
2425 			WARN_ON(!task_on_rq_queued(p));
2426 
2427 			/*
2428 			 * There's a chance that p is higher in priority
2429 			 * than what's currently running on its CPU.
2430 			 * This is just that p is waking up and hasn't
2431 			 * had a chance to schedule. We only pull
2432 			 * p if it is lower in priority than the
2433 			 * current task on the run queue
2434 			 */
2435 			if (p->prio < src_rq->curr->prio)
2436 				goto skip;
2437 
2438 			if (is_migration_disabled(p)) {
2439 				push_task = get_push_task(src_rq);
2440 			} else {
2441 				deactivate_task(src_rq, p, 0);
2442 				set_task_cpu(p, this_cpu);
2443 				activate_task(this_rq, p, 0);
2444 				resched = true;
2445 			}
2446 			/*
2447 			 * We continue with the search, just in
2448 			 * case there's an even higher prio task
2449 			 * in another runqueue. (low likelihood
2450 			 * but possible)
2451 			 */
2452 		}
2453 skip:
2454 		double_unlock_balance(this_rq, src_rq);
2455 
2456 		if (push_task) {
2457 			preempt_disable();
2458 			raw_spin_rq_unlock(this_rq);
2459 			stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2460 					    push_task, &src_rq->push_work);
2461 			preempt_enable();
2462 			raw_spin_rq_lock(this_rq);
2463 		}
2464 	}
2465 
2466 	if (resched)
2467 		resched_curr(this_rq);
2468 }
2469 
2470 /*
2471  * If we are not running and we are not going to reschedule soon, we should
2472  * try to push tasks away now
2473  */
task_woken_rt(struct rq * rq,struct task_struct * p)2474 static void task_woken_rt(struct rq *rq, struct task_struct *p)
2475 {
2476 	bool need_to_push = !task_on_cpu(rq, p) &&
2477 			    !test_tsk_need_resched(rq->curr) &&
2478 			    p->nr_cpus_allowed > 1 &&
2479 			    (dl_task(rq->curr) || rt_task(rq->curr)) &&
2480 			    (rq->curr->nr_cpus_allowed < 2 ||
2481 			     rq->curr->prio <= p->prio);
2482 
2483 	if (need_to_push)
2484 		push_rt_tasks(rq);
2485 }
2486 
2487 /* Assumes rq->lock is held */
rq_online_rt(struct rq * rq)2488 static void rq_online_rt(struct rq *rq)
2489 {
2490 	if (rq->rt.overloaded)
2491 		rt_set_overload(rq);
2492 
2493 	__enable_runtime(rq);
2494 
2495 	cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
2496 }
2497 
2498 /* Assumes rq->lock is held */
rq_offline_rt(struct rq * rq)2499 static void rq_offline_rt(struct rq *rq)
2500 {
2501 	if (rq->rt.overloaded)
2502 		rt_clear_overload(rq);
2503 
2504 	__disable_runtime(rq);
2505 
2506 	cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
2507 }
2508 
2509 /*
2510  * When switch from the rt queue, we bring ourselves to a position
2511  * that we might want to pull RT tasks from other runqueues.
2512  */
switched_from_rt(struct rq * rq,struct task_struct * p)2513 static void switched_from_rt(struct rq *rq, struct task_struct *p)
2514 {
2515 	/*
2516 	 * If there are other RT tasks then we will reschedule
2517 	 * and the scheduling of the other RT tasks will handle
2518 	 * the balancing. But if we are the last RT task
2519 	 * we may need to handle the pulling of RT tasks
2520 	 * now.
2521 	 */
2522 	if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
2523 		return;
2524 
2525 	rt_queue_pull_task(rq);
2526 }
2527 
init_sched_rt_class(void)2528 void __init init_sched_rt_class(void)
2529 {
2530 	unsigned int i;
2531 
2532 	for_each_possible_cpu(i) {
2533 		zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
2534 					GFP_KERNEL, cpu_to_node(i));
2535 	}
2536 }
2537 #endif /* CONFIG_SMP */
2538 
2539 /*
2540  * When switching a task to RT, we may overload the runqueue
2541  * with RT tasks. In this case we try to push them off to
2542  * other runqueues.
2543  */
switched_to_rt(struct rq * rq,struct task_struct * p)2544 static void switched_to_rt(struct rq *rq, struct task_struct *p)
2545 {
2546 	/*
2547 	 * If we are running, update the avg_rt tracking, as the running time
2548 	 * will now on be accounted into the latter.
2549 	 */
2550 	if (task_current(rq, p)) {
2551 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2552 		return;
2553 	}
2554 
2555 	/*
2556 	 * If we are not running we may need to preempt the current
2557 	 * running task. If that current running task is also an RT task
2558 	 * then see if we can move to another run queue.
2559 	 */
2560 	if (task_on_rq_queued(p)) {
2561 #ifdef CONFIG_SMP
2562 		if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
2563 			rt_queue_push_tasks(rq);
2564 #endif /* CONFIG_SMP */
2565 		if (p->prio < rq->curr->prio && cpu_online(cpu_of(rq)))
2566 			resched_curr(rq);
2567 	}
2568 }
2569 
2570 /*
2571  * Priority of the task has changed. This may cause
2572  * us to initiate a push or pull.
2573  */
2574 static void
prio_changed_rt(struct rq * rq,struct task_struct * p,int oldprio)2575 prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
2576 {
2577 	if (!task_on_rq_queued(p))
2578 		return;
2579 
2580 	if (task_current(rq, p)) {
2581 #ifdef CONFIG_SMP
2582 		/*
2583 		 * If our priority decreases while running, we
2584 		 * may need to pull tasks to this runqueue.
2585 		 */
2586 		if (oldprio < p->prio)
2587 			rt_queue_pull_task(rq);
2588 
2589 		/*
2590 		 * If there's a higher priority task waiting to run
2591 		 * then reschedule.
2592 		 */
2593 		if (p->prio > rq->rt.highest_prio.curr)
2594 			resched_curr(rq);
2595 #else
2596 		/* For UP simply resched on drop of prio */
2597 		if (oldprio < p->prio)
2598 			resched_curr(rq);
2599 #endif /* CONFIG_SMP */
2600 	} else {
2601 		/*
2602 		 * This task is not running, but if it is
2603 		 * greater than the current running task
2604 		 * then reschedule.
2605 		 */
2606 		if (p->prio < rq->curr->prio)
2607 			resched_curr(rq);
2608 	}
2609 }
2610 
2611 #ifdef CONFIG_POSIX_TIMERS
watchdog(struct rq * rq,struct task_struct * p)2612 static void watchdog(struct rq *rq, struct task_struct *p)
2613 {
2614 	unsigned long soft, hard;
2615 
2616 	/* max may change after cur was read, this will be fixed next tick */
2617 	soft = task_rlimit(p, RLIMIT_RTTIME);
2618 	hard = task_rlimit_max(p, RLIMIT_RTTIME);
2619 
2620 	if (soft != RLIM_INFINITY) {
2621 		unsigned long next;
2622 
2623 		if (p->rt.watchdog_stamp != jiffies) {
2624 			p->rt.timeout++;
2625 			p->rt.watchdog_stamp = jiffies;
2626 		}
2627 
2628 		next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
2629 		if (p->rt.timeout > next) {
2630 			posix_cputimers_rt_watchdog(&p->posix_cputimers,
2631 						    p->se.sum_exec_runtime);
2632 		}
2633 	}
2634 }
2635 #else
watchdog(struct rq * rq,struct task_struct * p)2636 static inline void watchdog(struct rq *rq, struct task_struct *p) { }
2637 #endif
2638 
2639 /*
2640  * scheduler tick hitting a task of our scheduling class.
2641  *
2642  * NOTE: This function can be called remotely by the tick offload that
2643  * goes along full dynticks. Therefore no local assumption can be made
2644  * and everything must be accessed through the @rq and @curr passed in
2645  * parameters.
2646  */
task_tick_rt(struct rq * rq,struct task_struct * p,int queued)2647 static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
2648 {
2649 	struct sched_rt_entity *rt_se = &p->rt;
2650 
2651 	update_curr_rt(rq);
2652 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
2653 
2654 	watchdog(rq, p);
2655 
2656 	/*
2657 	 * RR tasks need a special form of timeslice management.
2658 	 * FIFO tasks have no timeslices.
2659 	 */
2660 	if (p->policy != SCHED_RR)
2661 		return;
2662 
2663 	if (--p->rt.time_slice)
2664 		return;
2665 
2666 	p->rt.time_slice = sched_rr_timeslice;
2667 
2668 	/*
2669 	 * Requeue to the end of queue if we (and all of our ancestors) are not
2670 	 * the only element on the queue
2671 	 */
2672 	for_each_sched_rt_entity(rt_se) {
2673 		if (rt_se->run_list.prev != rt_se->run_list.next) {
2674 			requeue_task_rt(rq, p, 0);
2675 			resched_curr(rq);
2676 			return;
2677 		}
2678 	}
2679 }
2680 
get_rr_interval_rt(struct rq * rq,struct task_struct * task)2681 static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
2682 {
2683 	/*
2684 	 * Time slice is 0 for SCHED_FIFO tasks
2685 	 */
2686 	if (task->policy == SCHED_RR)
2687 		return sched_rr_timeslice;
2688 	else
2689 		return 0;
2690 }
2691 
2692 #ifdef CONFIG_SCHED_CORE
task_is_throttled_rt(struct task_struct * p,int cpu)2693 static int task_is_throttled_rt(struct task_struct *p, int cpu)
2694 {
2695 	struct rt_rq *rt_rq;
2696 
2697 #ifdef CONFIG_RT_GROUP_SCHED
2698 	rt_rq = task_group(p)->rt_rq[cpu];
2699 #else
2700 	rt_rq = &cpu_rq(cpu)->rt;
2701 #endif
2702 
2703 	return rt_rq_throttled(rt_rq);
2704 }
2705 #endif
2706 
2707 DEFINE_SCHED_CLASS(rt) = {
2708 
2709 	.enqueue_task		= enqueue_task_rt,
2710 	.dequeue_task		= dequeue_task_rt,
2711 	.yield_task		= yield_task_rt,
2712 
2713 	.check_preempt_curr	= check_preempt_curr_rt,
2714 
2715 	.pick_next_task		= pick_next_task_rt,
2716 	.put_prev_task		= put_prev_task_rt,
2717 	.set_next_task          = set_next_task_rt,
2718 
2719 #ifdef CONFIG_SMP
2720 	.balance		= balance_rt,
2721 	.pick_task		= pick_task_rt,
2722 	.select_task_rq		= select_task_rq_rt,
2723 	.set_cpus_allowed       = set_cpus_allowed_common,
2724 	.rq_online              = rq_online_rt,
2725 	.rq_offline             = rq_offline_rt,
2726 	.task_woken		= task_woken_rt,
2727 	.switched_from		= switched_from_rt,
2728 	.find_lock_rq		= find_lock_lowest_rq,
2729 #endif
2730 
2731 	.task_tick		= task_tick_rt,
2732 
2733 	.get_rr_interval	= get_rr_interval_rt,
2734 
2735 	.prio_changed		= prio_changed_rt,
2736 	.switched_to		= switched_to_rt,
2737 
2738 	.update_curr		= update_curr_rt,
2739 
2740 #ifdef CONFIG_SCHED_CORE
2741 	.task_is_throttled	= task_is_throttled_rt,
2742 #endif
2743 
2744 #ifdef CONFIG_UCLAMP_TASK
2745 	.uclamp_enabled		= 1,
2746 #endif
2747 };
2748 
2749 #ifdef CONFIG_RT_GROUP_SCHED
2750 /*
2751  * Ensure that the real time constraints are schedulable.
2752  */
2753 static DEFINE_MUTEX(rt_constraints_mutex);
2754 
tg_has_rt_tasks(struct task_group * tg)2755 static inline int tg_has_rt_tasks(struct task_group *tg)
2756 {
2757 	struct task_struct *task;
2758 	struct css_task_iter it;
2759 	int ret = 0;
2760 
2761 	/*
2762 	 * Autogroups do not have RT tasks; see autogroup_create().
2763 	 */
2764 	if (task_group_is_autogroup(tg))
2765 		return 0;
2766 
2767 	css_task_iter_start(&tg->css, 0, &it);
2768 	while (!ret && (task = css_task_iter_next(&it)))
2769 		ret |= rt_task(task);
2770 	css_task_iter_end(&it);
2771 
2772 	return ret;
2773 }
2774 
2775 struct rt_schedulable_data {
2776 	struct task_group *tg;
2777 	u64 rt_period;
2778 	u64 rt_runtime;
2779 };
2780 
tg_rt_schedulable(struct task_group * tg,void * data)2781 static int tg_rt_schedulable(struct task_group *tg, void *data)
2782 {
2783 	struct rt_schedulable_data *d = data;
2784 	struct task_group *child;
2785 	unsigned long total, sum = 0;
2786 	u64 period, runtime;
2787 
2788 	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2789 	runtime = tg->rt_bandwidth.rt_runtime;
2790 
2791 	if (tg == d->tg) {
2792 		period = d->rt_period;
2793 		runtime = d->rt_runtime;
2794 	}
2795 
2796 	/*
2797 	 * Cannot have more runtime than the period.
2798 	 */
2799 	if (runtime > period && runtime != RUNTIME_INF)
2800 		return -EINVAL;
2801 
2802 	/*
2803 	 * Ensure we don't starve existing RT tasks if runtime turns zero.
2804 	 */
2805 	if (rt_bandwidth_enabled() && !runtime &&
2806 	    tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))
2807 		return -EBUSY;
2808 
2809 	total = to_ratio(period, runtime);
2810 
2811 	/*
2812 	 * Nobody can have more than the global setting allows.
2813 	 */
2814 	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
2815 		return -EINVAL;
2816 
2817 	/*
2818 	 * The sum of our children's runtime should not exceed our own.
2819 	 */
2820 	list_for_each_entry_rcu(child, &tg->children, siblings) {
2821 		period = ktime_to_ns(child->rt_bandwidth.rt_period);
2822 		runtime = child->rt_bandwidth.rt_runtime;
2823 
2824 		if (child == d->tg) {
2825 			period = d->rt_period;
2826 			runtime = d->rt_runtime;
2827 		}
2828 
2829 		sum += to_ratio(period, runtime);
2830 	}
2831 
2832 	if (sum > total)
2833 		return -EINVAL;
2834 
2835 	return 0;
2836 }
2837 
__rt_schedulable(struct task_group * tg,u64 period,u64 runtime)2838 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
2839 {
2840 	int ret;
2841 
2842 	struct rt_schedulable_data data = {
2843 		.tg = tg,
2844 		.rt_period = period,
2845 		.rt_runtime = runtime,
2846 	};
2847 
2848 	rcu_read_lock();
2849 	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
2850 	rcu_read_unlock();
2851 
2852 	return ret;
2853 }
2854 
tg_set_rt_bandwidth(struct task_group * tg,u64 rt_period,u64 rt_runtime)2855 static int tg_set_rt_bandwidth(struct task_group *tg,
2856 		u64 rt_period, u64 rt_runtime)
2857 {
2858 	int i, err = 0;
2859 
2860 	/*
2861 	 * Disallowing the root group RT runtime is BAD, it would disallow the
2862 	 * kernel creating (and or operating) RT threads.
2863 	 */
2864 	if (tg == &root_task_group && rt_runtime == 0)
2865 		return -EINVAL;
2866 
2867 	/* No period doesn't make any sense. */
2868 	if (rt_period == 0)
2869 		return -EINVAL;
2870 
2871 	/*
2872 	 * Bound quota to defend quota against overflow during bandwidth shift.
2873 	 */
2874 	if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime)
2875 		return -EINVAL;
2876 
2877 	mutex_lock(&rt_constraints_mutex);
2878 	err = __rt_schedulable(tg, rt_period, rt_runtime);
2879 	if (err)
2880 		goto unlock;
2881 
2882 	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2883 	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
2884 	tg->rt_bandwidth.rt_runtime = rt_runtime;
2885 
2886 	for_each_possible_cpu(i) {
2887 		struct rt_rq *rt_rq = tg->rt_rq[i];
2888 
2889 		raw_spin_lock(&rt_rq->rt_runtime_lock);
2890 		rt_rq->rt_runtime = rt_runtime;
2891 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
2892 	}
2893 	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2894 unlock:
2895 	mutex_unlock(&rt_constraints_mutex);
2896 
2897 	return err;
2898 }
2899 
sched_group_set_rt_runtime(struct task_group * tg,long rt_runtime_us)2900 int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
2901 {
2902 	u64 rt_runtime, rt_period;
2903 
2904 	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2905 	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
2906 	if (rt_runtime_us < 0)
2907 		rt_runtime = RUNTIME_INF;
2908 	else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC)
2909 		return -EINVAL;
2910 
2911 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2912 }
2913 
sched_group_rt_runtime(struct task_group * tg)2914 long sched_group_rt_runtime(struct task_group *tg)
2915 {
2916 	u64 rt_runtime_us;
2917 
2918 	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
2919 		return -1;
2920 
2921 	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
2922 	do_div(rt_runtime_us, NSEC_PER_USEC);
2923 	return rt_runtime_us;
2924 }
2925 
sched_group_set_rt_period(struct task_group * tg,u64 rt_period_us)2926 int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
2927 {
2928 	u64 rt_runtime, rt_period;
2929 
2930 	if (rt_period_us > U64_MAX / NSEC_PER_USEC)
2931 		return -EINVAL;
2932 
2933 	rt_period = rt_period_us * NSEC_PER_USEC;
2934 	rt_runtime = tg->rt_bandwidth.rt_runtime;
2935 
2936 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2937 }
2938 
sched_group_rt_period(struct task_group * tg)2939 long sched_group_rt_period(struct task_group *tg)
2940 {
2941 	u64 rt_period_us;
2942 
2943 	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
2944 	do_div(rt_period_us, NSEC_PER_USEC);
2945 	return rt_period_us;
2946 }
2947 
2948 #ifdef CONFIG_SYSCTL
sched_rt_global_constraints(void)2949 static int sched_rt_global_constraints(void)
2950 {
2951 	int ret = 0;
2952 
2953 	mutex_lock(&rt_constraints_mutex);
2954 	ret = __rt_schedulable(NULL, 0, 0);
2955 	mutex_unlock(&rt_constraints_mutex);
2956 
2957 	return ret;
2958 }
2959 #endif /* CONFIG_SYSCTL */
2960 
sched_rt_can_attach(struct task_group * tg,struct task_struct * tsk)2961 int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
2962 {
2963 	/* Don't accept realtime tasks when there is no way for them to run */
2964 	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
2965 		return 0;
2966 
2967 	return 1;
2968 }
2969 
2970 #else /* !CONFIG_RT_GROUP_SCHED */
2971 
2972 #ifdef CONFIG_SYSCTL
sched_rt_global_constraints(void)2973 static int sched_rt_global_constraints(void)
2974 {
2975 	unsigned long flags;
2976 	int i;
2977 
2978 	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
2979 	for_each_possible_cpu(i) {
2980 		struct rt_rq *rt_rq = &cpu_rq(i)->rt;
2981 
2982 		raw_spin_lock(&rt_rq->rt_runtime_lock);
2983 		rt_rq->rt_runtime = global_rt_runtime();
2984 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
2985 	}
2986 	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
2987 
2988 	return 0;
2989 }
2990 #endif /* CONFIG_SYSCTL */
2991 #endif /* CONFIG_RT_GROUP_SCHED */
2992 
2993 #ifdef CONFIG_SYSCTL
sched_rt_global_validate(void)2994 static int sched_rt_global_validate(void)
2995 {
2996 	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
2997 		((sysctl_sched_rt_runtime > sysctl_sched_rt_period) ||
2998 		 ((u64)sysctl_sched_rt_runtime *
2999 			NSEC_PER_USEC > max_rt_runtime)))
3000 		return -EINVAL;
3001 
3002 	return 0;
3003 }
3004 
sched_rt_do_global(void)3005 static void sched_rt_do_global(void)
3006 {
3007 	unsigned long flags;
3008 
3009 	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
3010 	def_rt_bandwidth.rt_runtime = global_rt_runtime();
3011 	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
3012 	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
3013 }
3014 
sched_rt_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)3015 static int sched_rt_handler(struct ctl_table *table, int write, void *buffer,
3016 		size_t *lenp, loff_t *ppos)
3017 {
3018 	int old_period, old_runtime;
3019 	static DEFINE_MUTEX(mutex);
3020 	int ret;
3021 
3022 	mutex_lock(&mutex);
3023 	old_period = sysctl_sched_rt_period;
3024 	old_runtime = sysctl_sched_rt_runtime;
3025 
3026 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3027 
3028 	if (!ret && write) {
3029 		ret = sched_rt_global_validate();
3030 		if (ret)
3031 			goto undo;
3032 
3033 		ret = sched_dl_global_validate();
3034 		if (ret)
3035 			goto undo;
3036 
3037 		ret = sched_rt_global_constraints();
3038 		if (ret)
3039 			goto undo;
3040 
3041 		sched_rt_do_global();
3042 		sched_dl_do_global();
3043 	}
3044 	if (0) {
3045 undo:
3046 		sysctl_sched_rt_period = old_period;
3047 		sysctl_sched_rt_runtime = old_runtime;
3048 	}
3049 	mutex_unlock(&mutex);
3050 
3051 	return ret;
3052 }
3053 
sched_rr_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)3054 static int sched_rr_handler(struct ctl_table *table, int write, void *buffer,
3055 		size_t *lenp, loff_t *ppos)
3056 {
3057 	int ret;
3058 	static DEFINE_MUTEX(mutex);
3059 
3060 	mutex_lock(&mutex);
3061 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
3062 	/*
3063 	 * Make sure that internally we keep jiffies.
3064 	 * Also, writing zero resets the timeslice to default:
3065 	 */
3066 	if (!ret && write) {
3067 		sched_rr_timeslice =
3068 			sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
3069 			msecs_to_jiffies(sysctl_sched_rr_timeslice);
3070 
3071 		if (sysctl_sched_rr_timeslice <= 0)
3072 			sysctl_sched_rr_timeslice = jiffies_to_msecs(RR_TIMESLICE);
3073 	}
3074 	mutex_unlock(&mutex);
3075 
3076 	return ret;
3077 }
3078 #endif /* CONFIG_SYSCTL */
3079 
3080 #ifdef CONFIG_SCHED_DEBUG
print_rt_stats(struct seq_file * m,int cpu)3081 void print_rt_stats(struct seq_file *m, int cpu)
3082 {
3083 	rt_rq_iter_t iter;
3084 	struct rt_rq *rt_rq;
3085 
3086 	rcu_read_lock();
3087 	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
3088 		print_rt_rq(m, cpu, rt_rq);
3089 	rcu_read_unlock();
3090 }
3091 #endif /* CONFIG_SCHED_DEBUG */
3092