1 // SPDX-License-Identifier: GPL-2.0-only
2
3 /*
4 * A simple wrapper around refcount. An allocated sched_core_cookie's
5 * address is used to compute the cookie of the task.
6 */
7 struct sched_core_cookie {
8 refcount_t refcnt;
9 };
10
sched_core_alloc_cookie(void)11 static unsigned long sched_core_alloc_cookie(void)
12 {
13 struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL);
14 if (!ck)
15 return 0;
16
17 refcount_set(&ck->refcnt, 1);
18 sched_core_get();
19
20 return (unsigned long)ck;
21 }
22
sched_core_put_cookie(unsigned long cookie)23 static void sched_core_put_cookie(unsigned long cookie)
24 {
25 struct sched_core_cookie *ptr = (void *)cookie;
26
27 if (ptr && refcount_dec_and_test(&ptr->refcnt)) {
28 kfree(ptr);
29 sched_core_put();
30 }
31 }
32
sched_core_get_cookie(unsigned long cookie)33 static unsigned long sched_core_get_cookie(unsigned long cookie)
34 {
35 struct sched_core_cookie *ptr = (void *)cookie;
36
37 if (ptr)
38 refcount_inc(&ptr->refcnt);
39
40 return cookie;
41 }
42
43 /*
44 * sched_core_update_cookie - replace the cookie on a task
45 * @p: the task to update
46 * @cookie: the new cookie
47 *
48 * Effectively exchange the task cookie; caller is responsible for lifetimes on
49 * both ends.
50 *
51 * Returns: the old cookie
52 */
sched_core_update_cookie(struct task_struct * p,unsigned long cookie)53 static unsigned long sched_core_update_cookie(struct task_struct *p,
54 unsigned long cookie)
55 {
56 unsigned long old_cookie;
57 struct rq_flags rf;
58 struct rq *rq;
59 bool enqueued;
60
61 rq = task_rq_lock(p, &rf);
62
63 /*
64 * Since creating a cookie implies sched_core_get(), and we cannot set
65 * a cookie until after we've created it, similarly, we cannot destroy
66 * a cookie until after we've removed it, we must have core scheduling
67 * enabled here.
68 */
69 SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq));
70
71 enqueued = sched_core_enqueued(p);
72 if (enqueued)
73 sched_core_dequeue(rq, p, DEQUEUE_SAVE);
74
75 old_cookie = p->core_cookie;
76 p->core_cookie = cookie;
77
78 if (enqueued)
79 sched_core_enqueue(rq, p);
80
81 /*
82 * If task is currently running, it may not be compatible anymore after
83 * the cookie change, so enter the scheduler on its CPU to schedule it
84 * away.
85 *
86 * Note that it is possible that as a result of this cookie change, the
87 * core has now entered/left forced idle state. Defer accounting to the
88 * next scheduling edge, rather than always forcing a reschedule here.
89 */
90 if (task_running(rq, p))
91 resched_curr(rq);
92
93 task_rq_unlock(rq, p, &rf);
94
95 return old_cookie;
96 }
97
sched_core_clone_cookie(struct task_struct * p)98 static unsigned long sched_core_clone_cookie(struct task_struct *p)
99 {
100 unsigned long cookie, flags;
101
102 raw_spin_lock_irqsave(&p->pi_lock, flags);
103 cookie = sched_core_get_cookie(p->core_cookie);
104 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
105
106 return cookie;
107 }
108
sched_core_fork(struct task_struct * p)109 void sched_core_fork(struct task_struct *p)
110 {
111 RB_CLEAR_NODE(&p->core_node);
112 p->core_cookie = sched_core_clone_cookie(current);
113 }
114
sched_core_free(struct task_struct * p)115 void sched_core_free(struct task_struct *p)
116 {
117 sched_core_put_cookie(p->core_cookie);
118 }
119
__sched_core_set(struct task_struct * p,unsigned long cookie)120 static void __sched_core_set(struct task_struct *p, unsigned long cookie)
121 {
122 cookie = sched_core_get_cookie(cookie);
123 cookie = sched_core_update_cookie(p, cookie);
124 sched_core_put_cookie(cookie);
125 }
126
127 /* Called from prctl interface: PR_SCHED_CORE */
sched_core_share_pid(unsigned int cmd,pid_t pid,enum pid_type type,unsigned long uaddr)128 int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
129 unsigned long uaddr)
130 {
131 unsigned long cookie = 0, id = 0;
132 struct task_struct *task, *p;
133 struct pid *grp;
134 int err = 0;
135
136 if (!static_branch_likely(&sched_smt_present))
137 return -ENODEV;
138
139 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);
140 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD_GROUP != PIDTYPE_TGID);
141 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_PROCESS_GROUP != PIDTYPE_PGID);
142
143 if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 ||
144 (cmd != PR_SCHED_CORE_GET && uaddr))
145 return -EINVAL;
146
147 rcu_read_lock();
148 if (pid == 0) {
149 task = current;
150 } else {
151 task = find_task_by_vpid(pid);
152 if (!task) {
153 rcu_read_unlock();
154 return -ESRCH;
155 }
156 }
157 get_task_struct(task);
158 rcu_read_unlock();
159
160 /*
161 * Check if this process has the right to modify the specified
162 * process. Use the regular "ptrace_may_access()" checks.
163 */
164 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
165 err = -EPERM;
166 goto out;
167 }
168
169 switch (cmd) {
170 case PR_SCHED_CORE_GET:
171 if (type != PIDTYPE_PID || uaddr & 7) {
172 err = -EINVAL;
173 goto out;
174 }
175 cookie = sched_core_clone_cookie(task);
176 if (cookie) {
177 /* XXX improve ? */
178 ptr_to_hashval((void *)cookie, &id);
179 }
180 err = put_user(id, (u64 __user *)uaddr);
181 goto out;
182
183 case PR_SCHED_CORE_CREATE:
184 cookie = sched_core_alloc_cookie();
185 if (!cookie) {
186 err = -ENOMEM;
187 goto out;
188 }
189 break;
190
191 case PR_SCHED_CORE_SHARE_TO:
192 cookie = sched_core_clone_cookie(current);
193 break;
194
195 case PR_SCHED_CORE_SHARE_FROM:
196 if (type != PIDTYPE_PID) {
197 err = -EINVAL;
198 goto out;
199 }
200 cookie = sched_core_clone_cookie(task);
201 __sched_core_set(current, cookie);
202 goto out;
203
204 default:
205 err = -EINVAL;
206 goto out;
207 };
208
209 if (type == PIDTYPE_PID) {
210 __sched_core_set(task, cookie);
211 goto out;
212 }
213
214 read_lock(&tasklist_lock);
215 grp = task_pid_type(task, type);
216
217 do_each_pid_thread(grp, type, p) {
218 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) {
219 err = -EPERM;
220 goto out_tasklist;
221 }
222 } while_each_pid_thread(grp, type, p);
223
224 do_each_pid_thread(grp, type, p) {
225 __sched_core_set(p, cookie);
226 } while_each_pid_thread(grp, type, p);
227 out_tasklist:
228 read_unlock(&tasklist_lock);
229
230 out:
231 sched_core_put_cookie(cookie);
232 put_task_struct(task);
233 return err;
234 }
235
236 #ifdef CONFIG_SCHEDSTATS
237
238 /* REQUIRES: rq->core's clock recently updated. */
__sched_core_account_forceidle(struct rq * rq)239 void __sched_core_account_forceidle(struct rq *rq)
240 {
241 const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq));
242 u64 delta, now = rq_clock(rq->core);
243 struct rq *rq_i;
244 struct task_struct *p;
245 int i;
246
247 lockdep_assert_rq_held(rq);
248
249 WARN_ON_ONCE(!rq->core->core_forceidle_count);
250
251 if (rq->core->core_forceidle_start == 0)
252 return;
253
254 delta = now - rq->core->core_forceidle_start;
255 if (unlikely((s64)delta <= 0))
256 return;
257
258 rq->core->core_forceidle_start = now;
259
260 if (WARN_ON_ONCE(!rq->core->core_forceidle_occupation)) {
261 /* can't be forced idle without a running task */
262 } else if (rq->core->core_forceidle_count > 1 ||
263 rq->core->core_forceidle_occupation > 1) {
264 /*
265 * For larger SMT configurations, we need to scale the charged
266 * forced idle amount since there can be more than one forced
267 * idle sibling and more than one running cookied task.
268 */
269 delta *= rq->core->core_forceidle_count;
270 delta = div_u64(delta, rq->core->core_forceidle_occupation);
271 }
272
273 for_each_cpu(i, smt_mask) {
274 rq_i = cpu_rq(i);
275 p = rq_i->core_pick ?: rq_i->curr;
276
277 if (p == rq_i->idle)
278 continue;
279
280 __schedstat_add(p->stats.core_forceidle_sum, delta);
281 }
282 }
283
__sched_core_tick(struct rq * rq)284 void __sched_core_tick(struct rq *rq)
285 {
286 if (!rq->core->core_forceidle_count)
287 return;
288
289 if (rq != rq->core)
290 update_rq_clock(rq->core);
291
292 __sched_core_account_forceidle(rq);
293 }
294
295 #endif /* CONFIG_SCHEDSTATS */
296