1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_CPUMASK_H
3 #define __LINUX_CPUMASK_H
4 
5 /*
6  * Cpumasks provide a bitmap suitable for representing the
7  * set of CPU's in a system, one bit position per CPU number.  In general,
8  * only nr_cpu_ids (<= NR_CPUS) bits are valid.
9  */
10 #include <linux/kernel.h>
11 #include <linux/threads.h>
12 #include <linux/bitmap.h>
13 #include <linux/atomic.h>
14 #include <linux/bug.h>
15 
16 /* Don't assign or return these: may not be this big! */
17 typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
18 
19 /**
20  * cpumask_bits - get the bits in a cpumask
21  * @maskp: the struct cpumask *
22  *
23  * You should only assume nr_cpu_ids bits of this mask are valid.  This is
24  * a macro so it's const-correct.
25  */
26 #define cpumask_bits(maskp) ((maskp)->bits)
27 
28 /**
29  * cpumask_pr_args - printf args to output a cpumask
30  * @maskp: cpumask to be printed
31  *
32  * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
33  */
34 #define cpumask_pr_args(maskp)		nr_cpu_ids, cpumask_bits(maskp)
35 
36 #if NR_CPUS == 1
37 #define nr_cpu_ids		1U
38 #else
39 extern unsigned int nr_cpu_ids;
40 #endif
41 
42 #ifdef CONFIG_CPUMASK_OFFSTACK
43 /* Assuming NR_CPUS is huge, a runtime limit is more efficient.  Also,
44  * not all bits may be allocated. */
45 #define nr_cpumask_bits	nr_cpu_ids
46 #else
47 #define nr_cpumask_bits	((unsigned int)NR_CPUS)
48 #endif
49 
50 /*
51  * The following particular system cpumasks and operations manage
52  * possible, present, active and online cpus.
53  *
54  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
55  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
56  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
57  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
58  *
59  *  If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
60  *
61  *  The cpu_possible_mask is fixed at boot time, as the set of CPU id's
62  *  that it is possible might ever be plugged in at anytime during the
63  *  life of that system boot.  The cpu_present_mask is dynamic(*),
64  *  representing which CPUs are currently plugged in.  And
65  *  cpu_online_mask is the dynamic subset of cpu_present_mask,
66  *  indicating those CPUs available for scheduling.
67  *
68  *  If HOTPLUG is enabled, then cpu_possible_mask is forced to have
69  *  all NR_CPUS bits set, otherwise it is just the set of CPUs that
70  *  ACPI reports present at boot.
71  *
72  *  If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
73  *  depending on what ACPI reports as currently plugged in, otherwise
74  *  cpu_present_mask is just a copy of cpu_possible_mask.
75  *
76  *  (*) Well, cpu_present_mask is dynamic in the hotplug case.  If not
77  *      hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
78  *
79  * Subtleties:
80  * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
81  *    assumption that their single CPU is online.  The UP
82  *    cpu_{online,possible,present}_masks are placebos.  Changing them
83  *    will have no useful affect on the following num_*_cpus()
84  *    and cpu_*() macros in the UP case.  This ugliness is a UP
85  *    optimization - don't waste any instructions or memory references
86  *    asking if you're online or how many CPUs there are if there is
87  *    only one CPU.
88  */
89 
90 extern struct cpumask __cpu_possible_mask;
91 extern struct cpumask __cpu_online_mask;
92 extern struct cpumask __cpu_present_mask;
93 extern struct cpumask __cpu_active_mask;
94 extern struct cpumask __cpu_dying_mask;
95 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
96 #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
97 #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
98 #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
99 #define cpu_dying_mask    ((const struct cpumask *)&__cpu_dying_mask)
100 
101 extern atomic_t __num_online_cpus;
102 
103 extern cpumask_t cpus_booted_once_mask;
104 
cpu_max_bits_warn(unsigned int cpu,unsigned int bits)105 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
106 {
107 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
108 	WARN_ON_ONCE(cpu >= bits);
109 #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
110 }
111 
112 /* verify cpu argument to cpumask_* operators */
cpumask_check(unsigned int cpu)113 static __always_inline unsigned int cpumask_check(unsigned int cpu)
114 {
115 	cpu_max_bits_warn(cpu, nr_cpumask_bits);
116 	return cpu;
117 }
118 
119 #if NR_CPUS == 1
120 /* Uniprocessor.  Assume all masks are "1". */
cpumask_first(const struct cpumask * srcp)121 static inline unsigned int cpumask_first(const struct cpumask *srcp)
122 {
123 	return 0;
124 }
125 
cpumask_first_zero(const struct cpumask * srcp)126 static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
127 {
128 	return 0;
129 }
130 
cpumask_first_and(const struct cpumask * srcp1,const struct cpumask * srcp2)131 static inline unsigned int cpumask_first_and(const struct cpumask *srcp1,
132 					     const struct cpumask *srcp2)
133 {
134 	return 0;
135 }
136 
cpumask_last(const struct cpumask * srcp)137 static inline unsigned int cpumask_last(const struct cpumask *srcp)
138 {
139 	return 0;
140 }
141 
142 /* Valid inputs for n are -1 and 0. */
cpumask_next(int n,const struct cpumask * srcp)143 static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
144 {
145 	return n+1;
146 }
147 
cpumask_next_zero(int n,const struct cpumask * srcp)148 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
149 {
150 	return n+1;
151 }
152 
cpumask_next_and(int n,const struct cpumask * srcp,const struct cpumask * andp)153 static inline unsigned int cpumask_next_and(int n,
154 					    const struct cpumask *srcp,
155 					    const struct cpumask *andp)
156 {
157 	return n+1;
158 }
159 
cpumask_next_wrap(int n,const struct cpumask * mask,int start,bool wrap)160 static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask,
161 					     int start, bool wrap)
162 {
163 	/* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */
164 	return (wrap && n == 0);
165 }
166 
167 /* cpu must be a valid cpu, ie 0, so there's no other choice. */
cpumask_any_but(const struct cpumask * mask,unsigned int cpu)168 static inline unsigned int cpumask_any_but(const struct cpumask *mask,
169 					   unsigned int cpu)
170 {
171 	return 1;
172 }
173 
cpumask_local_spread(unsigned int i,int node)174 static inline unsigned int cpumask_local_spread(unsigned int i, int node)
175 {
176 	return 0;
177 }
178 
cpumask_any_and_distribute(const struct cpumask * src1p,const struct cpumask * src2p)179 static inline int cpumask_any_and_distribute(const struct cpumask *src1p,
180 					     const struct cpumask *src2p) {
181 	return cpumask_first_and(src1p, src2p);
182 }
183 
cpumask_any_distribute(const struct cpumask * srcp)184 static inline int cpumask_any_distribute(const struct cpumask *srcp)
185 {
186 	return cpumask_first(srcp);
187 }
188 
189 #define for_each_cpu(cpu, mask)			\
190 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
191 #define for_each_cpu_not(cpu, mask)		\
192 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
193 #define for_each_cpu_wrap(cpu, mask, start)	\
194 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
195 #define for_each_cpu_and(cpu, mask1, mask2)	\
196 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2)
197 #else
198 /**
199  * cpumask_first - get the first cpu in a cpumask
200  * @srcp: the cpumask pointer
201  *
202  * Returns >= nr_cpu_ids if no cpus set.
203  */
cpumask_first(const struct cpumask * srcp)204 static inline unsigned int cpumask_first(const struct cpumask *srcp)
205 {
206 	return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
207 }
208 
209 /**
210  * cpumask_first_zero - get the first unset cpu in a cpumask
211  * @srcp: the cpumask pointer
212  *
213  * Returns >= nr_cpu_ids if all cpus are set.
214  */
cpumask_first_zero(const struct cpumask * srcp)215 static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
216 {
217 	return find_first_zero_bit(cpumask_bits(srcp), nr_cpumask_bits);
218 }
219 
220 /**
221  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
222  * @src1p: the first input
223  * @src2p: the second input
224  *
225  * Returns >= nr_cpu_ids if no cpus set in both.  See also cpumask_next_and().
226  */
227 static inline
cpumask_first_and(const struct cpumask * srcp1,const struct cpumask * srcp2)228 unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
229 {
230 	return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits);
231 }
232 
233 /**
234  * cpumask_last - get the last CPU in a cpumask
235  * @srcp:	- the cpumask pointer
236  *
237  * Returns	>= nr_cpumask_bits if no CPUs set.
238  */
cpumask_last(const struct cpumask * srcp)239 static inline unsigned int cpumask_last(const struct cpumask *srcp)
240 {
241 	return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
242 }
243 
244 unsigned int __pure cpumask_next(int n, const struct cpumask *srcp);
245 
246 /**
247  * cpumask_next_zero - get the next unset cpu in a cpumask
248  * @n: the cpu prior to the place to search (ie. return will be > @n)
249  * @srcp: the cpumask pointer
250  *
251  * Returns >= nr_cpu_ids if no further cpus unset.
252  */
cpumask_next_zero(int n,const struct cpumask * srcp)253 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
254 {
255 	/* -1 is a legal arg here. */
256 	if (n != -1)
257 		cpumask_check(n);
258 	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
259 }
260 
261 int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
262 int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
263 unsigned int cpumask_local_spread(unsigned int i, int node);
264 int cpumask_any_and_distribute(const struct cpumask *src1p,
265 			       const struct cpumask *src2p);
266 int cpumask_any_distribute(const struct cpumask *srcp);
267 
268 /**
269  * for_each_cpu - iterate over every cpu in a mask
270  * @cpu: the (optionally unsigned) integer iterator
271  * @mask: the cpumask pointer
272  *
273  * After the loop, cpu is >= nr_cpu_ids.
274  */
275 #define for_each_cpu(cpu, mask)				\
276 	for ((cpu) = -1;				\
277 		(cpu) = cpumask_next((cpu), (mask)),	\
278 		(cpu) < nr_cpu_ids;)
279 
280 /**
281  * for_each_cpu_not - iterate over every cpu in a complemented mask
282  * @cpu: the (optionally unsigned) integer iterator
283  * @mask: the cpumask pointer
284  *
285  * After the loop, cpu is >= nr_cpu_ids.
286  */
287 #define for_each_cpu_not(cpu, mask)				\
288 	for ((cpu) = -1;					\
289 		(cpu) = cpumask_next_zero((cpu), (mask)),	\
290 		(cpu) < nr_cpu_ids;)
291 
292 extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
293 
294 /**
295  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
296  * @cpu: the (optionally unsigned) integer iterator
297  * @mask: the cpumask pointer
298  * @start: the start location
299  *
300  * The implementation does not assume any bit in @mask is set (including @start).
301  *
302  * After the loop, cpu is >= nr_cpu_ids.
303  */
304 #define for_each_cpu_wrap(cpu, mask, start)					\
305 	for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false);	\
306 	     (cpu) < nr_cpumask_bits;						\
307 	     (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
308 
309 /**
310  * for_each_cpu_and - iterate over every cpu in both masks
311  * @cpu: the (optionally unsigned) integer iterator
312  * @mask1: the first cpumask pointer
313  * @mask2: the second cpumask pointer
314  *
315  * This saves a temporary CPU mask in many places.  It is equivalent to:
316  *	struct cpumask tmp;
317  *	cpumask_and(&tmp, &mask1, &mask2);
318  *	for_each_cpu(cpu, &tmp)
319  *		...
320  *
321  * After the loop, cpu is >= nr_cpu_ids.
322  */
323 #define for_each_cpu_and(cpu, mask1, mask2)				\
324 	for ((cpu) = -1;						\
325 		(cpu) = cpumask_next_and((cpu), (mask1), (mask2)),	\
326 		(cpu) < nr_cpu_ids;)
327 #endif /* SMP */
328 
329 #define CPU_BITS_NONE						\
330 {								\
331 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
332 }
333 
334 #define CPU_BITS_CPU0						\
335 {								\
336 	[0] =  1UL						\
337 }
338 
339 /**
340  * cpumask_set_cpu - set a cpu in a cpumask
341  * @cpu: cpu number (< nr_cpu_ids)
342  * @dstp: the cpumask pointer
343  */
cpumask_set_cpu(unsigned int cpu,struct cpumask * dstp)344 static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
345 {
346 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));
347 }
348 
__cpumask_set_cpu(unsigned int cpu,struct cpumask * dstp)349 static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
350 {
351 	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
352 }
353 
354 
355 /**
356  * cpumask_clear_cpu - clear a cpu in a cpumask
357  * @cpu: cpu number (< nr_cpu_ids)
358  * @dstp: the cpumask pointer
359  */
cpumask_clear_cpu(int cpu,struct cpumask * dstp)360 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
361 {
362 	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
363 }
364 
__cpumask_clear_cpu(int cpu,struct cpumask * dstp)365 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
366 {
367 	__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
368 }
369 
370 /**
371  * cpumask_test_cpu - test for a cpu in a cpumask
372  * @cpu: cpu number (< nr_cpu_ids)
373  * @cpumask: the cpumask pointer
374  *
375  * Returns 1 if @cpu is set in @cpumask, else returns 0
376  */
cpumask_test_cpu(int cpu,const struct cpumask * cpumask)377 static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
378 {
379 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
380 }
381 
382 /**
383  * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
384  * @cpu: cpu number (< nr_cpu_ids)
385  * @cpumask: the cpumask pointer
386  *
387  * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
388  *
389  * test_and_set_bit wrapper for cpumasks.
390  */
cpumask_test_and_set_cpu(int cpu,struct cpumask * cpumask)391 static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
392 {
393 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
394 }
395 
396 /**
397  * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
398  * @cpu: cpu number (< nr_cpu_ids)
399  * @cpumask: the cpumask pointer
400  *
401  * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
402  *
403  * test_and_clear_bit wrapper for cpumasks.
404  */
cpumask_test_and_clear_cpu(int cpu,struct cpumask * cpumask)405 static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
406 {
407 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
408 }
409 
410 /**
411  * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
412  * @dstp: the cpumask pointer
413  */
cpumask_setall(struct cpumask * dstp)414 static inline void cpumask_setall(struct cpumask *dstp)
415 {
416 	bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
417 }
418 
419 /**
420  * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
421  * @dstp: the cpumask pointer
422  */
cpumask_clear(struct cpumask * dstp)423 static inline void cpumask_clear(struct cpumask *dstp)
424 {
425 	bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
426 }
427 
428 /**
429  * cpumask_and - *dstp = *src1p & *src2p
430  * @dstp: the cpumask result
431  * @src1p: the first input
432  * @src2p: the second input
433  *
434  * If *@dstp is empty, returns 0, else returns 1
435  */
cpumask_and(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)436 static inline int cpumask_and(struct cpumask *dstp,
437 			       const struct cpumask *src1p,
438 			       const struct cpumask *src2p)
439 {
440 	return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
441 				       cpumask_bits(src2p), nr_cpumask_bits);
442 }
443 
444 /**
445  * cpumask_or - *dstp = *src1p | *src2p
446  * @dstp: the cpumask result
447  * @src1p: the first input
448  * @src2p: the second input
449  */
cpumask_or(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)450 static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
451 			      const struct cpumask *src2p)
452 {
453 	bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
454 				      cpumask_bits(src2p), nr_cpumask_bits);
455 }
456 
457 /**
458  * cpumask_xor - *dstp = *src1p ^ *src2p
459  * @dstp: the cpumask result
460  * @src1p: the first input
461  * @src2p: the second input
462  */
cpumask_xor(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)463 static inline void cpumask_xor(struct cpumask *dstp,
464 			       const struct cpumask *src1p,
465 			       const struct cpumask *src2p)
466 {
467 	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
468 				       cpumask_bits(src2p), nr_cpumask_bits);
469 }
470 
471 /**
472  * cpumask_andnot - *dstp = *src1p & ~*src2p
473  * @dstp: the cpumask result
474  * @src1p: the first input
475  * @src2p: the second input
476  *
477  * If *@dstp is empty, returns 0, else returns 1
478  */
cpumask_andnot(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)479 static inline int cpumask_andnot(struct cpumask *dstp,
480 				  const struct cpumask *src1p,
481 				  const struct cpumask *src2p)
482 {
483 	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
484 					  cpumask_bits(src2p), nr_cpumask_bits);
485 }
486 
487 /**
488  * cpumask_complement - *dstp = ~*srcp
489  * @dstp: the cpumask result
490  * @srcp: the input to invert
491  */
cpumask_complement(struct cpumask * dstp,const struct cpumask * srcp)492 static inline void cpumask_complement(struct cpumask *dstp,
493 				      const struct cpumask *srcp)
494 {
495 	bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
496 					      nr_cpumask_bits);
497 }
498 
499 /**
500  * cpumask_equal - *src1p == *src2p
501  * @src1p: the first input
502  * @src2p: the second input
503  */
cpumask_equal(const struct cpumask * src1p,const struct cpumask * src2p)504 static inline bool cpumask_equal(const struct cpumask *src1p,
505 				const struct cpumask *src2p)
506 {
507 	return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
508 						 nr_cpumask_bits);
509 }
510 
511 /**
512  * cpumask_or_equal - *src1p | *src2p == *src3p
513  * @src1p: the first input
514  * @src2p: the second input
515  * @src3p: the third input
516  */
cpumask_or_equal(const struct cpumask * src1p,const struct cpumask * src2p,const struct cpumask * src3p)517 static inline bool cpumask_or_equal(const struct cpumask *src1p,
518 				    const struct cpumask *src2p,
519 				    const struct cpumask *src3p)
520 {
521 	return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
522 			       cpumask_bits(src3p), nr_cpumask_bits);
523 }
524 
525 /**
526  * cpumask_intersects - (*src1p & *src2p) != 0
527  * @src1p: the first input
528  * @src2p: the second input
529  */
cpumask_intersects(const struct cpumask * src1p,const struct cpumask * src2p)530 static inline bool cpumask_intersects(const struct cpumask *src1p,
531 				     const struct cpumask *src2p)
532 {
533 	return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
534 						      nr_cpumask_bits);
535 }
536 
537 /**
538  * cpumask_subset - (*src1p & ~*src2p) == 0
539  * @src1p: the first input
540  * @src2p: the second input
541  *
542  * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
543  */
cpumask_subset(const struct cpumask * src1p,const struct cpumask * src2p)544 static inline int cpumask_subset(const struct cpumask *src1p,
545 				 const struct cpumask *src2p)
546 {
547 	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
548 						  nr_cpumask_bits);
549 }
550 
551 /**
552  * cpumask_empty - *srcp == 0
553  * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
554  */
cpumask_empty(const struct cpumask * srcp)555 static inline bool cpumask_empty(const struct cpumask *srcp)
556 {
557 	return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
558 }
559 
560 /**
561  * cpumask_full - *srcp == 0xFFFFFFFF...
562  * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
563  */
cpumask_full(const struct cpumask * srcp)564 static inline bool cpumask_full(const struct cpumask *srcp)
565 {
566 	return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
567 }
568 
569 /**
570  * cpumask_weight - Count of bits in *srcp
571  * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
572  */
cpumask_weight(const struct cpumask * srcp)573 static inline unsigned int cpumask_weight(const struct cpumask *srcp)
574 {
575 	return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
576 }
577 
578 /**
579  * cpumask_shift_right - *dstp = *srcp >> n
580  * @dstp: the cpumask result
581  * @srcp: the input to shift
582  * @n: the number of bits to shift by
583  */
cpumask_shift_right(struct cpumask * dstp,const struct cpumask * srcp,int n)584 static inline void cpumask_shift_right(struct cpumask *dstp,
585 				       const struct cpumask *srcp, int n)
586 {
587 	bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
588 					       nr_cpumask_bits);
589 }
590 
591 /**
592  * cpumask_shift_left - *dstp = *srcp << n
593  * @dstp: the cpumask result
594  * @srcp: the input to shift
595  * @n: the number of bits to shift by
596  */
cpumask_shift_left(struct cpumask * dstp,const struct cpumask * srcp,int n)597 static inline void cpumask_shift_left(struct cpumask *dstp,
598 				      const struct cpumask *srcp, int n)
599 {
600 	bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
601 					      nr_cpumask_bits);
602 }
603 
604 /**
605  * cpumask_copy - *dstp = *srcp
606  * @dstp: the result
607  * @srcp: the input cpumask
608  */
cpumask_copy(struct cpumask * dstp,const struct cpumask * srcp)609 static inline void cpumask_copy(struct cpumask *dstp,
610 				const struct cpumask *srcp)
611 {
612 	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
613 }
614 
615 /**
616  * cpumask_any - pick a "random" cpu from *srcp
617  * @srcp: the input cpumask
618  *
619  * Returns >= nr_cpu_ids if no cpus set.
620  */
621 #define cpumask_any(srcp) cpumask_first(srcp)
622 
623 /**
624  * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
625  * @mask1: the first input cpumask
626  * @mask2: the second input cpumask
627  *
628  * Returns >= nr_cpu_ids if no cpus set.
629  */
630 #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
631 
632 /**
633  * cpumask_of - the cpumask containing just a given cpu
634  * @cpu: the cpu (<= nr_cpu_ids)
635  */
636 #define cpumask_of(cpu) (get_cpu_mask(cpu))
637 
638 /**
639  * cpumask_parse_user - extract a cpumask from a user string
640  * @buf: the buffer to extract from
641  * @len: the length of the buffer
642  * @dstp: the cpumask to set.
643  *
644  * Returns -errno, or 0 for success.
645  */
cpumask_parse_user(const char __user * buf,int len,struct cpumask * dstp)646 static inline int cpumask_parse_user(const char __user *buf, int len,
647 				     struct cpumask *dstp)
648 {
649 	return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
650 }
651 
652 /**
653  * cpumask_parselist_user - extract a cpumask from a user string
654  * @buf: the buffer to extract from
655  * @len: the length of the buffer
656  * @dstp: the cpumask to set.
657  *
658  * Returns -errno, or 0 for success.
659  */
cpumask_parselist_user(const char __user * buf,int len,struct cpumask * dstp)660 static inline int cpumask_parselist_user(const char __user *buf, int len,
661 				     struct cpumask *dstp)
662 {
663 	return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
664 				     nr_cpumask_bits);
665 }
666 
667 /**
668  * cpumask_parse - extract a cpumask from a string
669  * @buf: the buffer to extract from
670  * @dstp: the cpumask to set.
671  *
672  * Returns -errno, or 0 for success.
673  */
cpumask_parse(const char * buf,struct cpumask * dstp)674 static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
675 {
676 	return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
677 }
678 
679 /**
680  * cpulist_parse - extract a cpumask from a user string of ranges
681  * @buf: the buffer to extract from
682  * @dstp: the cpumask to set.
683  *
684  * Returns -errno, or 0 for success.
685  */
cpulist_parse(const char * buf,struct cpumask * dstp)686 static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
687 {
688 	return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
689 }
690 
691 /**
692  * cpumask_size - size to allocate for a 'struct cpumask' in bytes
693  */
cpumask_size(void)694 static inline unsigned int cpumask_size(void)
695 {
696 	return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
697 }
698 
699 /*
700  * cpumask_var_t: struct cpumask for stack usage.
701  *
702  * Oh, the wicked games we play!  In order to make kernel coding a
703  * little more difficult, we typedef cpumask_var_t to an array or a
704  * pointer: doing &mask on an array is a noop, so it still works.
705  *
706  * ie.
707  *	cpumask_var_t tmpmask;
708  *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
709  *		return -ENOMEM;
710  *
711  *	  ... use 'tmpmask' like a normal struct cpumask * ...
712  *
713  *	free_cpumask_var(tmpmask);
714  *
715  *
716  * However, one notable exception is there. alloc_cpumask_var() allocates
717  * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
718  * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
719  *
720  *	cpumask_var_t tmpmask;
721  *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
722  *		return -ENOMEM;
723  *
724  *	var = *tmpmask;
725  *
726  * This code makes NR_CPUS length memcopy and brings to a memory corruption.
727  * cpumask_copy() provide safe copy functionality.
728  *
729  * Note that there is another evil here: If you define a cpumask_var_t
730  * as a percpu variable then the way to obtain the address of the cpumask
731  * structure differently influences what this_cpu_* operation needs to be
732  * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
733  * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
734  * other type of cpumask_var_t implementation is configured.
735  *
736  * Please also note that __cpumask_var_read_mostly can be used to declare
737  * a cpumask_var_t variable itself (not its content) as read mostly.
738  */
739 #ifdef CONFIG_CPUMASK_OFFSTACK
740 typedef struct cpumask *cpumask_var_t;
741 
742 #define this_cpu_cpumask_var_ptr(x)	this_cpu_read(x)
743 #define __cpumask_var_read_mostly	__read_mostly
744 
745 bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
746 bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
747 bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
748 bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
749 void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
750 void free_cpumask_var(cpumask_var_t mask);
751 void free_bootmem_cpumask_var(cpumask_var_t mask);
752 
cpumask_available(cpumask_var_t mask)753 static inline bool cpumask_available(cpumask_var_t mask)
754 {
755 	return mask != NULL;
756 }
757 
758 #else
759 typedef struct cpumask cpumask_var_t[1];
760 
761 #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
762 #define __cpumask_var_read_mostly
763 
alloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)764 static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
765 {
766 	return true;
767 }
768 
alloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)769 static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
770 					  int node)
771 {
772 	return true;
773 }
774 
zalloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)775 static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
776 {
777 	cpumask_clear(*mask);
778 	return true;
779 }
780 
zalloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)781 static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
782 					  int node)
783 {
784 	cpumask_clear(*mask);
785 	return true;
786 }
787 
alloc_bootmem_cpumask_var(cpumask_var_t * mask)788 static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
789 {
790 }
791 
free_cpumask_var(cpumask_var_t mask)792 static inline void free_cpumask_var(cpumask_var_t mask)
793 {
794 }
795 
free_bootmem_cpumask_var(cpumask_var_t mask)796 static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
797 {
798 }
799 
cpumask_available(cpumask_var_t mask)800 static inline bool cpumask_available(cpumask_var_t mask)
801 {
802 	return true;
803 }
804 #endif /* CONFIG_CPUMASK_OFFSTACK */
805 
806 /* It's common to want to use cpu_all_mask in struct member initializers,
807  * so it has to refer to an address rather than a pointer. */
808 extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
809 #define cpu_all_mask to_cpumask(cpu_all_bits)
810 
811 /* First bits of cpu_bit_bitmap are in fact unset. */
812 #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
813 
814 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
815 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
816 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
817 
818 /* Wrappers for arch boot code to manipulate normally-constant masks */
819 void init_cpu_present(const struct cpumask *src);
820 void init_cpu_possible(const struct cpumask *src);
821 void init_cpu_online(const struct cpumask *src);
822 
reset_cpu_possible_mask(void)823 static inline void reset_cpu_possible_mask(void)
824 {
825 	bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
826 }
827 
828 static inline void
set_cpu_possible(unsigned int cpu,bool possible)829 set_cpu_possible(unsigned int cpu, bool possible)
830 {
831 	if (possible)
832 		cpumask_set_cpu(cpu, &__cpu_possible_mask);
833 	else
834 		cpumask_clear_cpu(cpu, &__cpu_possible_mask);
835 }
836 
837 static inline void
set_cpu_present(unsigned int cpu,bool present)838 set_cpu_present(unsigned int cpu, bool present)
839 {
840 	if (present)
841 		cpumask_set_cpu(cpu, &__cpu_present_mask);
842 	else
843 		cpumask_clear_cpu(cpu, &__cpu_present_mask);
844 }
845 
846 void set_cpu_online(unsigned int cpu, bool online);
847 
848 static inline void
set_cpu_active(unsigned int cpu,bool active)849 set_cpu_active(unsigned int cpu, bool active)
850 {
851 	if (active)
852 		cpumask_set_cpu(cpu, &__cpu_active_mask);
853 	else
854 		cpumask_clear_cpu(cpu, &__cpu_active_mask);
855 }
856 
857 static inline void
set_cpu_dying(unsigned int cpu,bool dying)858 set_cpu_dying(unsigned int cpu, bool dying)
859 {
860 	if (dying)
861 		cpumask_set_cpu(cpu, &__cpu_dying_mask);
862 	else
863 		cpumask_clear_cpu(cpu, &__cpu_dying_mask);
864 }
865 
866 /**
867  * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
868  * @bitmap: the bitmap
869  *
870  * There are a few places where cpumask_var_t isn't appropriate and
871  * static cpumasks must be used (eg. very early boot), yet we don't
872  * expose the definition of 'struct cpumask'.
873  *
874  * This does the conversion, and can be used as a constant initializer.
875  */
876 #define to_cpumask(bitmap)						\
877 	((struct cpumask *)(1 ? (bitmap)				\
878 			    : (void *)sizeof(__check_is_bitmap(bitmap))))
879 
__check_is_bitmap(const unsigned long * bitmap)880 static inline int __check_is_bitmap(const unsigned long *bitmap)
881 {
882 	return 1;
883 }
884 
885 /*
886  * Special-case data structure for "single bit set only" constant CPU masks.
887  *
888  * We pre-generate all the 64 (or 32) possible bit positions, with enough
889  * padding to the left and the right, and return the constant pointer
890  * appropriately offset.
891  */
892 extern const unsigned long
893 	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
894 
get_cpu_mask(unsigned int cpu)895 static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
896 {
897 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
898 	p -= cpu / BITS_PER_LONG;
899 	return to_cpumask(p);
900 }
901 
902 #if NR_CPUS > 1
903 /**
904  * num_online_cpus() - Read the number of online CPUs
905  *
906  * Despite the fact that __num_online_cpus is of type atomic_t, this
907  * interface gives only a momentary snapshot and is not protected against
908  * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
909  * region.
910  */
num_online_cpus(void)911 static inline unsigned int num_online_cpus(void)
912 {
913 	return atomic_read(&__num_online_cpus);
914 }
915 #define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
916 #define num_present_cpus()	cpumask_weight(cpu_present_mask)
917 #define num_active_cpus()	cpumask_weight(cpu_active_mask)
918 
cpu_online(unsigned int cpu)919 static inline bool cpu_online(unsigned int cpu)
920 {
921 	return cpumask_test_cpu(cpu, cpu_online_mask);
922 }
923 
cpu_possible(unsigned int cpu)924 static inline bool cpu_possible(unsigned int cpu)
925 {
926 	return cpumask_test_cpu(cpu, cpu_possible_mask);
927 }
928 
cpu_present(unsigned int cpu)929 static inline bool cpu_present(unsigned int cpu)
930 {
931 	return cpumask_test_cpu(cpu, cpu_present_mask);
932 }
933 
cpu_active(unsigned int cpu)934 static inline bool cpu_active(unsigned int cpu)
935 {
936 	return cpumask_test_cpu(cpu, cpu_active_mask);
937 }
938 
cpu_dying(unsigned int cpu)939 static inline bool cpu_dying(unsigned int cpu)
940 {
941 	return cpumask_test_cpu(cpu, cpu_dying_mask);
942 }
943 
944 #else
945 
946 #define num_online_cpus()	1U
947 #define num_possible_cpus()	1U
948 #define num_present_cpus()	1U
949 #define num_active_cpus()	1U
950 
cpu_online(unsigned int cpu)951 static inline bool cpu_online(unsigned int cpu)
952 {
953 	return cpu == 0;
954 }
955 
cpu_possible(unsigned int cpu)956 static inline bool cpu_possible(unsigned int cpu)
957 {
958 	return cpu == 0;
959 }
960 
cpu_present(unsigned int cpu)961 static inline bool cpu_present(unsigned int cpu)
962 {
963 	return cpu == 0;
964 }
965 
cpu_active(unsigned int cpu)966 static inline bool cpu_active(unsigned int cpu)
967 {
968 	return cpu == 0;
969 }
970 
cpu_dying(unsigned int cpu)971 static inline bool cpu_dying(unsigned int cpu)
972 {
973 	return false;
974 }
975 
976 #endif /* NR_CPUS > 1 */
977 
978 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))
979 
980 #if NR_CPUS <= BITS_PER_LONG
981 #define CPU_BITS_ALL						\
982 {								\
983 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
984 }
985 
986 #else /* NR_CPUS > BITS_PER_LONG */
987 
988 #define CPU_BITS_ALL						\
989 {								\
990 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,		\
991 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
992 }
993 #endif /* NR_CPUS > BITS_PER_LONG */
994 
995 /**
996  * cpumap_print_to_pagebuf  - copies the cpumask into the buffer either
997  *	as comma-separated list of cpus or hex values of cpumask
998  * @list: indicates whether the cpumap must be list
999  * @mask: the cpumask to copy
1000  * @buf: the buffer to copy into
1001  *
1002  * Returns the length of the (null-terminated) @buf string, zero if
1003  * nothing is copied.
1004  */
1005 static inline ssize_t
cpumap_print_to_pagebuf(bool list,char * buf,const struct cpumask * mask)1006 cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
1007 {
1008 	return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
1009 				      nr_cpu_ids);
1010 }
1011 
1012 /**
1013  * cpumap_print_bitmask_to_buf  - copies the cpumask into the buffer as
1014  *	hex values of cpumask
1015  *
1016  * @buf: the buffer to copy into
1017  * @mask: the cpumask to copy
1018  * @off: in the string from which we are copying, we copy to @buf
1019  * @count: the maximum number of bytes to print
1020  *
1021  * The function prints the cpumask into the buffer as hex values of
1022  * cpumask; Typically used by bin_attribute to export cpumask bitmask
1023  * ABI.
1024  *
1025  * Returns the length of how many bytes have been copied, excluding
1026  * terminating '\0'.
1027  */
1028 static inline ssize_t
cpumap_print_bitmask_to_buf(char * buf,const struct cpumask * mask,loff_t off,size_t count)1029 cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
1030 		loff_t off, size_t count)
1031 {
1032 	return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1033 				   nr_cpu_ids, off, count) - 1;
1034 }
1035 
1036 /**
1037  * cpumap_print_list_to_buf  - copies the cpumask into the buffer as
1038  *	comma-separated list of cpus
1039  *
1040  * Everything is same with the above cpumap_print_bitmask_to_buf()
1041  * except the print format.
1042  */
1043 static inline ssize_t
cpumap_print_list_to_buf(char * buf,const struct cpumask * mask,loff_t off,size_t count)1044 cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
1045 		loff_t off, size_t count)
1046 {
1047 	return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1048 				   nr_cpu_ids, off, count) - 1;
1049 }
1050 
1051 #if NR_CPUS <= BITS_PER_LONG
1052 #define CPU_MASK_ALL							\
1053 (cpumask_t) { {								\
1054 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1055 } }
1056 #else
1057 #define CPU_MASK_ALL							\
1058 (cpumask_t) { {								\
1059 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,			\
1060 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1061 } }
1062 #endif /* NR_CPUS > BITS_PER_LONG */
1063 
1064 #define CPU_MASK_NONE							\
1065 (cpumask_t) { {								\
1066 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] =  0UL				\
1067 } }
1068 
1069 #define CPU_MASK_CPU0							\
1070 (cpumask_t) { {								\
1071 	[0] =  1UL							\
1072 } }
1073 
1074 /*
1075  * Provide a valid theoretical max size for cpumap and cpulist sysfs files
1076  * to avoid breaking userspace which may allocate a buffer based on the size
1077  * reported by e.g. fstat.
1078  *
1079  * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1080  *
1081  * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1082  * to 2 orders of magnitude larger than 8192. And then we divide by 2 to
1083  * cover a worst-case of every other cpu being on one of two nodes for a
1084  * very large NR_CPUS.
1085  *
1086  *  Use PAGE_SIZE as a minimum for smaller configurations.
1087  */
1088 #define CPUMAP_FILE_MAX_BYTES  ((((NR_CPUS * 9)/32 - 1) > PAGE_SIZE) \
1089 					? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
1090 #define CPULIST_FILE_MAX_BYTES  (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
1091 
1092 #endif /* __LINUX_CPUMASK_H */
1093