1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_CPUMASK_H
3 #define __LINUX_CPUMASK_H
4 
5 /*
6  * Cpumasks provide a bitmap suitable for representing the
7  * set of CPU's in a system, one bit position per CPU number.  In general,
8  * only nr_cpu_ids (<= NR_CPUS) bits are valid.
9  */
10 #include <linux/kernel.h>
11 #include <linux/threads.h>
12 #include <linux/bitmap.h>
13 #include <linux/atomic.h>
14 #include <linux/bug.h>
15 #include <linux/gfp_types.h>
16 #include <linux/numa.h>
17 
18 /* Don't assign or return these: may not be this big! */
19 typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
20 
21 /**
22  * cpumask_bits - get the bits in a cpumask
23  * @maskp: the struct cpumask *
24  *
25  * You should only assume nr_cpu_ids bits of this mask are valid.  This is
26  * a macro so it's const-correct.
27  */
28 #define cpumask_bits(maskp) ((maskp)->bits)
29 
30 /**
31  * cpumask_pr_args - printf args to output a cpumask
32  * @maskp: cpumask to be printed
33  *
34  * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
35  */
36 #define cpumask_pr_args(maskp)		nr_cpu_ids, cpumask_bits(maskp)
37 
38 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
39 #define nr_cpu_ids ((unsigned int)NR_CPUS)
40 #else
41 extern unsigned int nr_cpu_ids;
42 #endif
43 
set_nr_cpu_ids(unsigned int nr)44 static inline void set_nr_cpu_ids(unsigned int nr)
45 {
46 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
47 	WARN_ON(nr != nr_cpu_ids);
48 #else
49 	nr_cpu_ids = nr;
50 #endif
51 }
52 
53 /*
54  * We have several different "preferred sizes" for the cpumask
55  * operations, depending on operation.
56  *
57  * For example, the bitmap scanning and operating operations have
58  * optimized routines that work for the single-word case, but only when
59  * the size is constant. So if NR_CPUS fits in one single word, we are
60  * better off using that small constant, in order to trigger the
61  * optimized bit finding. That is 'small_cpumask_size'.
62  *
63  * The clearing and copying operations will similarly perform better
64  * with a constant size, but we limit that size arbitrarily to four
65  * words. We call this 'large_cpumask_size'.
66  *
67  * Finally, some operations just want the exact limit, either because
68  * they set bits or just don't have any faster fixed-sized versions. We
69  * call this just 'nr_cpumask_bits'.
70  *
71  * Note that these optional constants are always guaranteed to be at
72  * least as big as 'nr_cpu_ids' itself is, and all our cpumask
73  * allocations are at least that size (see cpumask_size()). The
74  * optimization comes from being able to potentially use a compile-time
75  * constant instead of a run-time generated exact number of CPUs.
76  */
77 #if NR_CPUS <= BITS_PER_LONG
78   #define small_cpumask_bits ((unsigned int)NR_CPUS)
79   #define large_cpumask_bits ((unsigned int)NR_CPUS)
80 #elif NR_CPUS <= 4*BITS_PER_LONG
81   #define small_cpumask_bits nr_cpu_ids
82   #define large_cpumask_bits ((unsigned int)NR_CPUS)
83 #else
84   #define small_cpumask_bits nr_cpu_ids
85   #define large_cpumask_bits nr_cpu_ids
86 #endif
87 #define nr_cpumask_bits nr_cpu_ids
88 
89 /*
90  * The following particular system cpumasks and operations manage
91  * possible, present, active and online cpus.
92  *
93  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
94  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
95  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
96  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
97  *
98  *  If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
99  *
100  *  The cpu_possible_mask is fixed at boot time, as the set of CPU id's
101  *  that it is possible might ever be plugged in at anytime during the
102  *  life of that system boot.  The cpu_present_mask is dynamic(*),
103  *  representing which CPUs are currently plugged in.  And
104  *  cpu_online_mask is the dynamic subset of cpu_present_mask,
105  *  indicating those CPUs available for scheduling.
106  *
107  *  If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
108  *  depending on what ACPI reports as currently plugged in, otherwise
109  *  cpu_present_mask is just a copy of cpu_possible_mask.
110  *
111  *  (*) Well, cpu_present_mask is dynamic in the hotplug case.  If not
112  *      hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
113  *
114  * Subtleties:
115  * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
116  *    assumption that their single CPU is online.  The UP
117  *    cpu_{online,possible,present}_masks are placebos.  Changing them
118  *    will have no useful affect on the following num_*_cpus()
119  *    and cpu_*() macros in the UP case.  This ugliness is a UP
120  *    optimization - don't waste any instructions or memory references
121  *    asking if you're online or how many CPUs there are if there is
122  *    only one CPU.
123  */
124 
125 extern struct cpumask __cpu_possible_mask;
126 extern struct cpumask __cpu_online_mask;
127 extern struct cpumask __cpu_present_mask;
128 extern struct cpumask __cpu_active_mask;
129 extern struct cpumask __cpu_dying_mask;
130 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
131 #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
132 #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
133 #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
134 #define cpu_dying_mask    ((const struct cpumask *)&__cpu_dying_mask)
135 
136 extern atomic_t __num_online_cpus;
137 
138 extern cpumask_t cpus_booted_once_mask;
139 
cpu_max_bits_warn(unsigned int cpu,unsigned int bits)140 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
141 {
142 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
143 	WARN_ON_ONCE(cpu >= bits);
144 #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
145 }
146 
147 /* verify cpu argument to cpumask_* operators */
cpumask_check(unsigned int cpu)148 static __always_inline unsigned int cpumask_check(unsigned int cpu)
149 {
150 	cpu_max_bits_warn(cpu, small_cpumask_bits);
151 	return cpu;
152 }
153 
154 /**
155  * cpumask_first - get the first cpu in a cpumask
156  * @srcp: the cpumask pointer
157  *
158  * Returns >= nr_cpu_ids if no cpus set.
159  */
cpumask_first(const struct cpumask * srcp)160 static inline unsigned int cpumask_first(const struct cpumask *srcp)
161 {
162 	return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
163 }
164 
165 /**
166  * cpumask_first_zero - get the first unset cpu in a cpumask
167  * @srcp: the cpumask pointer
168  *
169  * Returns >= nr_cpu_ids if all cpus are set.
170  */
cpumask_first_zero(const struct cpumask * srcp)171 static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
172 {
173 	return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
174 }
175 
176 /**
177  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
178  * @srcp1: the first input
179  * @srcp2: the second input
180  *
181  * Returns >= nr_cpu_ids if no cpus set in both.  See also cpumask_next_and().
182  */
183 static inline
cpumask_first_and(const struct cpumask * srcp1,const struct cpumask * srcp2)184 unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
185 {
186 	return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
187 }
188 
189 /**
190  * cpumask_last - get the last CPU in a cpumask
191  * @srcp:	- the cpumask pointer
192  *
193  * Returns	>= nr_cpumask_bits if no CPUs set.
194  */
cpumask_last(const struct cpumask * srcp)195 static inline unsigned int cpumask_last(const struct cpumask *srcp)
196 {
197 	return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
198 }
199 
200 /**
201  * cpumask_next - get the next cpu in a cpumask
202  * @n: the cpu prior to the place to search (ie. return will be > @n)
203  * @srcp: the cpumask pointer
204  *
205  * Returns >= nr_cpu_ids if no further cpus set.
206  */
207 static inline
cpumask_next(int n,const struct cpumask * srcp)208 unsigned int cpumask_next(int n, const struct cpumask *srcp)
209 {
210 	/* -1 is a legal arg here. */
211 	if (n != -1)
212 		cpumask_check(n);
213 	return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);
214 }
215 
216 /**
217  * cpumask_next_zero - get the next unset cpu in a cpumask
218  * @n: the cpu prior to the place to search (ie. return will be > @n)
219  * @srcp: the cpumask pointer
220  *
221  * Returns >= nr_cpu_ids if no further cpus unset.
222  */
cpumask_next_zero(int n,const struct cpumask * srcp)223 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
224 {
225 	/* -1 is a legal arg here. */
226 	if (n != -1)
227 		cpumask_check(n);
228 	return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);
229 }
230 
231 #if NR_CPUS == 1
232 /* Uniprocessor: there is only one valid CPU */
cpumask_local_spread(unsigned int i,int node)233 static inline unsigned int cpumask_local_spread(unsigned int i, int node)
234 {
235 	return 0;
236 }
237 
cpumask_any_and_distribute(const struct cpumask * src1p,const struct cpumask * src2p)238 static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
239 						      const struct cpumask *src2p)
240 {
241 	return cpumask_first_and(src1p, src2p);
242 }
243 
cpumask_any_distribute(const struct cpumask * srcp)244 static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
245 {
246 	return cpumask_first(srcp);
247 }
248 #else
249 unsigned int cpumask_local_spread(unsigned int i, int node);
250 unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
251 			       const struct cpumask *src2p);
252 unsigned int cpumask_any_distribute(const struct cpumask *srcp);
253 #endif /* NR_CPUS */
254 
255 /**
256  * cpumask_next_and - get the next cpu in *src1p & *src2p
257  * @n: the cpu prior to the place to search (ie. return will be > @n)
258  * @src1p: the first cpumask pointer
259  * @src2p: the second cpumask pointer
260  *
261  * Returns >= nr_cpu_ids if no further cpus set in both.
262  */
263 static inline
cpumask_next_and(int n,const struct cpumask * src1p,const struct cpumask * src2p)264 unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
265 		     const struct cpumask *src2p)
266 {
267 	/* -1 is a legal arg here. */
268 	if (n != -1)
269 		cpumask_check(n);
270 	return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
271 		small_cpumask_bits, n + 1);
272 }
273 
274 /**
275  * for_each_cpu - iterate over every cpu in a mask
276  * @cpu: the (optionally unsigned) integer iterator
277  * @mask: the cpumask pointer
278  *
279  * After the loop, cpu is >= nr_cpu_ids.
280  */
281 #define for_each_cpu(cpu, mask)				\
282 	for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
283 
284 #if NR_CPUS == 1
285 static inline
cpumask_next_wrap(int n,const struct cpumask * mask,int start,bool wrap)286 unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
287 {
288 	cpumask_check(start);
289 	if (n != -1)
290 		cpumask_check(n);
291 
292 	/*
293 	 * Return the first available CPU when wrapping, or when starting before cpu0,
294 	 * since there is only one valid option.
295 	 */
296 	if (wrap && n >= 0)
297 		return nr_cpumask_bits;
298 
299 	return cpumask_first(mask);
300 }
301 #else
302 unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
303 #endif
304 
305 /**
306  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
307  * @cpu: the (optionally unsigned) integer iterator
308  * @mask: the cpumask pointer
309  * @start: the start location
310  *
311  * The implementation does not assume any bit in @mask is set (including @start).
312  *
313  * After the loop, cpu is >= nr_cpu_ids.
314  */
315 #define for_each_cpu_wrap(cpu, mask, start)				\
316 	for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
317 
318 /**
319  * for_each_cpu_and - iterate over every cpu in both masks
320  * @cpu: the (optionally unsigned) integer iterator
321  * @mask1: the first cpumask pointer
322  * @mask2: the second cpumask pointer
323  *
324  * This saves a temporary CPU mask in many places.  It is equivalent to:
325  *	struct cpumask tmp;
326  *	cpumask_and(&tmp, &mask1, &mask2);
327  *	for_each_cpu(cpu, &tmp)
328  *		...
329  *
330  * After the loop, cpu is >= nr_cpu_ids.
331  */
332 #define for_each_cpu_and(cpu, mask1, mask2)				\
333 	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
334 
335 /**
336  * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
337  *			 those present in another.
338  * @cpu: the (optionally unsigned) integer iterator
339  * @mask1: the first cpumask pointer
340  * @mask2: the second cpumask pointer
341  *
342  * This saves a temporary CPU mask in many places.  It is equivalent to:
343  *	struct cpumask tmp;
344  *	cpumask_andnot(&tmp, &mask1, &mask2);
345  *	for_each_cpu(cpu, &tmp)
346  *		...
347  *
348  * After the loop, cpu is >= nr_cpu_ids.
349  */
350 #define for_each_cpu_andnot(cpu, mask1, mask2)				\
351 	for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
352 
353 /**
354  * for_each_cpu_or - iterate over every cpu present in either mask
355  * @cpu: the (optionally unsigned) integer iterator
356  * @mask1: the first cpumask pointer
357  * @mask2: the second cpumask pointer
358  *
359  * This saves a temporary CPU mask in many places.  It is equivalent to:
360  *	struct cpumask tmp;
361  *	cpumask_or(&tmp, &mask1, &mask2);
362  *	for_each_cpu(cpu, &tmp)
363  *		...
364  *
365  * After the loop, cpu is >= nr_cpu_ids.
366  */
367 #define for_each_cpu_or(cpu, mask1, mask2)				\
368 	for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
369 
370 /**
371  * cpumask_any_but - return a "random" in a cpumask, but not this one.
372  * @mask: the cpumask to search
373  * @cpu: the cpu to ignore.
374  *
375  * Often used to find any cpu but smp_processor_id() in a mask.
376  * Returns >= nr_cpu_ids if no cpus set.
377  */
378 static inline
cpumask_any_but(const struct cpumask * mask,unsigned int cpu)379 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
380 {
381 	unsigned int i;
382 
383 	cpumask_check(cpu);
384 	for_each_cpu(i, mask)
385 		if (i != cpu)
386 			break;
387 	return i;
388 }
389 
390 /**
391  * cpumask_nth - get the first cpu in a cpumask
392  * @srcp: the cpumask pointer
393  * @cpu: the N'th cpu to find, starting from 0
394  *
395  * Returns >= nr_cpu_ids if such cpu doesn't exist.
396  */
cpumask_nth(unsigned int cpu,const struct cpumask * srcp)397 static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
398 {
399 	return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
400 }
401 
402 /**
403  * cpumask_nth_and - get the first cpu in 2 cpumasks
404  * @srcp1: the cpumask pointer
405  * @srcp2: the cpumask pointer
406  * @cpu: the N'th cpu to find, starting from 0
407  *
408  * Returns >= nr_cpu_ids if such cpu doesn't exist.
409  */
410 static inline
cpumask_nth_and(unsigned int cpu,const struct cpumask * srcp1,const struct cpumask * srcp2)411 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
412 							const struct cpumask *srcp2)
413 {
414 	return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
415 				small_cpumask_bits, cpumask_check(cpu));
416 }
417 
418 /**
419  * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd.
420  * @srcp1: the cpumask pointer
421  * @srcp2: the cpumask pointer
422  * @cpu: the N'th cpu to find, starting from 0
423  *
424  * Returns >= nr_cpu_ids if such cpu doesn't exist.
425  */
426 static inline
cpumask_nth_andnot(unsigned int cpu,const struct cpumask * srcp1,const struct cpumask * srcp2)427 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
428 							const struct cpumask *srcp2)
429 {
430 	return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
431 				small_cpumask_bits, cpumask_check(cpu));
432 }
433 
434 /**
435  * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
436  * @srcp1: the cpumask pointer
437  * @srcp2: the cpumask pointer
438  * @srcp3: the cpumask pointer
439  * @cpu: the N'th cpu to find, starting from 0
440  *
441  * Returns >= nr_cpu_ids if such cpu doesn't exist.
442  */
443 static __always_inline
cpumask_nth_and_andnot(unsigned int cpu,const struct cpumask * srcp1,const struct cpumask * srcp2,const struct cpumask * srcp3)444 unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,
445 							const struct cpumask *srcp2,
446 							const struct cpumask *srcp3)
447 {
448 	return find_nth_and_andnot_bit(cpumask_bits(srcp1),
449 					cpumask_bits(srcp2),
450 					cpumask_bits(srcp3),
451 					small_cpumask_bits, cpumask_check(cpu));
452 }
453 
454 #define CPU_BITS_NONE						\
455 {								\
456 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
457 }
458 
459 #define CPU_BITS_CPU0						\
460 {								\
461 	[0] =  1UL						\
462 }
463 
464 /**
465  * cpumask_set_cpu - set a cpu in a cpumask
466  * @cpu: cpu number (< nr_cpu_ids)
467  * @dstp: the cpumask pointer
468  */
cpumask_set_cpu(unsigned int cpu,struct cpumask * dstp)469 static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
470 {
471 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));
472 }
473 
__cpumask_set_cpu(unsigned int cpu,struct cpumask * dstp)474 static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
475 {
476 	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
477 }
478 
479 
480 /**
481  * cpumask_clear_cpu - clear a cpu in a cpumask
482  * @cpu: cpu number (< nr_cpu_ids)
483  * @dstp: the cpumask pointer
484  */
cpumask_clear_cpu(int cpu,struct cpumask * dstp)485 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
486 {
487 	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
488 }
489 
__cpumask_clear_cpu(int cpu,struct cpumask * dstp)490 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
491 {
492 	__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
493 }
494 
495 /**
496  * cpumask_test_cpu - test for a cpu in a cpumask
497  * @cpu: cpu number (< nr_cpu_ids)
498  * @cpumask: the cpumask pointer
499  *
500  * Returns true if @cpu is set in @cpumask, else returns false
501  */
cpumask_test_cpu(int cpu,const struct cpumask * cpumask)502 static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
503 {
504 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
505 }
506 
507 /**
508  * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
509  * @cpu: cpu number (< nr_cpu_ids)
510  * @cpumask: the cpumask pointer
511  *
512  * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
513  *
514  * test_and_set_bit wrapper for cpumasks.
515  */
cpumask_test_and_set_cpu(int cpu,struct cpumask * cpumask)516 static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
517 {
518 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
519 }
520 
521 /**
522  * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
523  * @cpu: cpu number (< nr_cpu_ids)
524  * @cpumask: the cpumask pointer
525  *
526  * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
527  *
528  * test_and_clear_bit wrapper for cpumasks.
529  */
cpumask_test_and_clear_cpu(int cpu,struct cpumask * cpumask)530 static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
531 {
532 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
533 }
534 
535 /**
536  * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
537  * @dstp: the cpumask pointer
538  */
cpumask_setall(struct cpumask * dstp)539 static inline void cpumask_setall(struct cpumask *dstp)
540 {
541 	if (small_const_nbits(small_cpumask_bits)) {
542 		cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
543 		return;
544 	}
545 	bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
546 }
547 
548 /**
549  * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
550  * @dstp: the cpumask pointer
551  */
cpumask_clear(struct cpumask * dstp)552 static inline void cpumask_clear(struct cpumask *dstp)
553 {
554 	bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
555 }
556 
557 /**
558  * cpumask_and - *dstp = *src1p & *src2p
559  * @dstp: the cpumask result
560  * @src1p: the first input
561  * @src2p: the second input
562  *
563  * If *@dstp is empty, returns false, else returns true
564  */
cpumask_and(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)565 static inline bool cpumask_and(struct cpumask *dstp,
566 			       const struct cpumask *src1p,
567 			       const struct cpumask *src2p)
568 {
569 	return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
570 				       cpumask_bits(src2p), small_cpumask_bits);
571 }
572 
573 /**
574  * cpumask_or - *dstp = *src1p | *src2p
575  * @dstp: the cpumask result
576  * @src1p: the first input
577  * @src2p: the second input
578  */
cpumask_or(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)579 static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
580 			      const struct cpumask *src2p)
581 {
582 	bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
583 				      cpumask_bits(src2p), small_cpumask_bits);
584 }
585 
586 /**
587  * cpumask_xor - *dstp = *src1p ^ *src2p
588  * @dstp: the cpumask result
589  * @src1p: the first input
590  * @src2p: the second input
591  */
cpumask_xor(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)592 static inline void cpumask_xor(struct cpumask *dstp,
593 			       const struct cpumask *src1p,
594 			       const struct cpumask *src2p)
595 {
596 	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
597 				       cpumask_bits(src2p), small_cpumask_bits);
598 }
599 
600 /**
601  * cpumask_andnot - *dstp = *src1p & ~*src2p
602  * @dstp: the cpumask result
603  * @src1p: the first input
604  * @src2p: the second input
605  *
606  * If *@dstp is empty, returns false, else returns true
607  */
cpumask_andnot(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)608 static inline bool cpumask_andnot(struct cpumask *dstp,
609 				  const struct cpumask *src1p,
610 				  const struct cpumask *src2p)
611 {
612 	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
613 					  cpumask_bits(src2p), small_cpumask_bits);
614 }
615 
616 /**
617  * cpumask_equal - *src1p == *src2p
618  * @src1p: the first input
619  * @src2p: the second input
620  */
cpumask_equal(const struct cpumask * src1p,const struct cpumask * src2p)621 static inline bool cpumask_equal(const struct cpumask *src1p,
622 				const struct cpumask *src2p)
623 {
624 	return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
625 						 small_cpumask_bits);
626 }
627 
628 /**
629  * cpumask_or_equal - *src1p | *src2p == *src3p
630  * @src1p: the first input
631  * @src2p: the second input
632  * @src3p: the third input
633  */
cpumask_or_equal(const struct cpumask * src1p,const struct cpumask * src2p,const struct cpumask * src3p)634 static inline bool cpumask_or_equal(const struct cpumask *src1p,
635 				    const struct cpumask *src2p,
636 				    const struct cpumask *src3p)
637 {
638 	return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
639 			       cpumask_bits(src3p), small_cpumask_bits);
640 }
641 
642 /**
643  * cpumask_intersects - (*src1p & *src2p) != 0
644  * @src1p: the first input
645  * @src2p: the second input
646  */
cpumask_intersects(const struct cpumask * src1p,const struct cpumask * src2p)647 static inline bool cpumask_intersects(const struct cpumask *src1p,
648 				     const struct cpumask *src2p)
649 {
650 	return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
651 						      small_cpumask_bits);
652 }
653 
654 /**
655  * cpumask_subset - (*src1p & ~*src2p) == 0
656  * @src1p: the first input
657  * @src2p: the second input
658  *
659  * Returns true if *@src1p is a subset of *@src2p, else returns false
660  */
cpumask_subset(const struct cpumask * src1p,const struct cpumask * src2p)661 static inline bool cpumask_subset(const struct cpumask *src1p,
662 				 const struct cpumask *src2p)
663 {
664 	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
665 						  small_cpumask_bits);
666 }
667 
668 /**
669  * cpumask_empty - *srcp == 0
670  * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
671  */
cpumask_empty(const struct cpumask * srcp)672 static inline bool cpumask_empty(const struct cpumask *srcp)
673 {
674 	return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
675 }
676 
677 /**
678  * cpumask_full - *srcp == 0xFFFFFFFF...
679  * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
680  */
cpumask_full(const struct cpumask * srcp)681 static inline bool cpumask_full(const struct cpumask *srcp)
682 {
683 	return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
684 }
685 
686 /**
687  * cpumask_weight - Count of bits in *srcp
688  * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
689  */
cpumask_weight(const struct cpumask * srcp)690 static inline unsigned int cpumask_weight(const struct cpumask *srcp)
691 {
692 	return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
693 }
694 
695 /**
696  * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
697  * @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
698  * @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
699  */
cpumask_weight_and(const struct cpumask * srcp1,const struct cpumask * srcp2)700 static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
701 						const struct cpumask *srcp2)
702 {
703 	return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
704 }
705 
706 /**
707  * cpumask_shift_right - *dstp = *srcp >> n
708  * @dstp: the cpumask result
709  * @srcp: the input to shift
710  * @n: the number of bits to shift by
711  */
cpumask_shift_right(struct cpumask * dstp,const struct cpumask * srcp,int n)712 static inline void cpumask_shift_right(struct cpumask *dstp,
713 				       const struct cpumask *srcp, int n)
714 {
715 	bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
716 					       small_cpumask_bits);
717 }
718 
719 /**
720  * cpumask_shift_left - *dstp = *srcp << n
721  * @dstp: the cpumask result
722  * @srcp: the input to shift
723  * @n: the number of bits to shift by
724  */
cpumask_shift_left(struct cpumask * dstp,const struct cpumask * srcp,int n)725 static inline void cpumask_shift_left(struct cpumask *dstp,
726 				      const struct cpumask *srcp, int n)
727 {
728 	bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
729 					      nr_cpumask_bits);
730 }
731 
732 /**
733  * cpumask_copy - *dstp = *srcp
734  * @dstp: the result
735  * @srcp: the input cpumask
736  */
cpumask_copy(struct cpumask * dstp,const struct cpumask * srcp)737 static inline void cpumask_copy(struct cpumask *dstp,
738 				const struct cpumask *srcp)
739 {
740 	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
741 }
742 
743 /**
744  * cpumask_any - pick a "random" cpu from *srcp
745  * @srcp: the input cpumask
746  *
747  * Returns >= nr_cpu_ids if no cpus set.
748  */
749 #define cpumask_any(srcp) cpumask_first(srcp)
750 
751 /**
752  * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
753  * @mask1: the first input cpumask
754  * @mask2: the second input cpumask
755  *
756  * Returns >= nr_cpu_ids if no cpus set.
757  */
758 #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
759 
760 /**
761  * cpumask_of - the cpumask containing just a given cpu
762  * @cpu: the cpu (<= nr_cpu_ids)
763  */
764 #define cpumask_of(cpu) (get_cpu_mask(cpu))
765 
766 /**
767  * cpumask_parse_user - extract a cpumask from a user string
768  * @buf: the buffer to extract from
769  * @len: the length of the buffer
770  * @dstp: the cpumask to set.
771  *
772  * Returns -errno, or 0 for success.
773  */
cpumask_parse_user(const char __user * buf,int len,struct cpumask * dstp)774 static inline int cpumask_parse_user(const char __user *buf, int len,
775 				     struct cpumask *dstp)
776 {
777 	return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
778 }
779 
780 /**
781  * cpumask_parselist_user - extract a cpumask from a user string
782  * @buf: the buffer to extract from
783  * @len: the length of the buffer
784  * @dstp: the cpumask to set.
785  *
786  * Returns -errno, or 0 for success.
787  */
cpumask_parselist_user(const char __user * buf,int len,struct cpumask * dstp)788 static inline int cpumask_parselist_user(const char __user *buf, int len,
789 				     struct cpumask *dstp)
790 {
791 	return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
792 				     nr_cpumask_bits);
793 }
794 
795 /**
796  * cpumask_parse - extract a cpumask from a string
797  * @buf: the buffer to extract from
798  * @dstp: the cpumask to set.
799  *
800  * Returns -errno, or 0 for success.
801  */
cpumask_parse(const char * buf,struct cpumask * dstp)802 static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
803 {
804 	return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
805 }
806 
807 /**
808  * cpulist_parse - extract a cpumask from a user string of ranges
809  * @buf: the buffer to extract from
810  * @dstp: the cpumask to set.
811  *
812  * Returns -errno, or 0 for success.
813  */
cpulist_parse(const char * buf,struct cpumask * dstp)814 static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
815 {
816 	return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
817 }
818 
819 /**
820  * cpumask_size - size to allocate for a 'struct cpumask' in bytes
821  */
cpumask_size(void)822 static inline unsigned int cpumask_size(void)
823 {
824 	return BITS_TO_LONGS(large_cpumask_bits) * sizeof(long);
825 }
826 
827 /*
828  * cpumask_var_t: struct cpumask for stack usage.
829  *
830  * Oh, the wicked games we play!  In order to make kernel coding a
831  * little more difficult, we typedef cpumask_var_t to an array or a
832  * pointer: doing &mask on an array is a noop, so it still works.
833  *
834  * ie.
835  *	cpumask_var_t tmpmask;
836  *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
837  *		return -ENOMEM;
838  *
839  *	  ... use 'tmpmask' like a normal struct cpumask * ...
840  *
841  *	free_cpumask_var(tmpmask);
842  *
843  *
844  * However, one notable exception is there. alloc_cpumask_var() allocates
845  * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
846  * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
847  *
848  *	cpumask_var_t tmpmask;
849  *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
850  *		return -ENOMEM;
851  *
852  *	var = *tmpmask;
853  *
854  * This code makes NR_CPUS length memcopy and brings to a memory corruption.
855  * cpumask_copy() provide safe copy functionality.
856  *
857  * Note that there is another evil here: If you define a cpumask_var_t
858  * as a percpu variable then the way to obtain the address of the cpumask
859  * structure differently influences what this_cpu_* operation needs to be
860  * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
861  * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
862  * other type of cpumask_var_t implementation is configured.
863  *
864  * Please also note that __cpumask_var_read_mostly can be used to declare
865  * a cpumask_var_t variable itself (not its content) as read mostly.
866  */
867 #ifdef CONFIG_CPUMASK_OFFSTACK
868 typedef struct cpumask *cpumask_var_t;
869 
870 #define this_cpu_cpumask_var_ptr(x)	this_cpu_read(x)
871 #define __cpumask_var_read_mostly	__read_mostly
872 
873 bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
874 
875 static inline
zalloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)876 bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
877 {
878 	return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
879 }
880 
881 /**
882  * alloc_cpumask_var - allocate a struct cpumask
883  * @mask: pointer to cpumask_var_t where the cpumask is returned
884  * @flags: GFP_ flags
885  *
886  * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
887  * a nop returning a constant 1 (in <linux/cpumask.h>).
888  *
889  * See alloc_cpumask_var_node.
890  */
891 static inline
alloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)892 bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
893 {
894 	return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
895 }
896 
897 static inline
zalloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)898 bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
899 {
900 	return alloc_cpumask_var(mask, flags | __GFP_ZERO);
901 }
902 
903 void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
904 void free_cpumask_var(cpumask_var_t mask);
905 void free_bootmem_cpumask_var(cpumask_var_t mask);
906 
cpumask_available(cpumask_var_t mask)907 static inline bool cpumask_available(cpumask_var_t mask)
908 {
909 	return mask != NULL;
910 }
911 
912 #else
913 typedef struct cpumask cpumask_var_t[1];
914 
915 #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
916 #define __cpumask_var_read_mostly
917 
alloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)918 static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
919 {
920 	return true;
921 }
922 
alloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)923 static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
924 					  int node)
925 {
926 	return true;
927 }
928 
zalloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)929 static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
930 {
931 	cpumask_clear(*mask);
932 	return true;
933 }
934 
zalloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)935 static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
936 					  int node)
937 {
938 	cpumask_clear(*mask);
939 	return true;
940 }
941 
alloc_bootmem_cpumask_var(cpumask_var_t * mask)942 static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
943 {
944 }
945 
free_cpumask_var(cpumask_var_t mask)946 static inline void free_cpumask_var(cpumask_var_t mask)
947 {
948 }
949 
free_bootmem_cpumask_var(cpumask_var_t mask)950 static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
951 {
952 }
953 
cpumask_available(cpumask_var_t mask)954 static inline bool cpumask_available(cpumask_var_t mask)
955 {
956 	return true;
957 }
958 #endif /* CONFIG_CPUMASK_OFFSTACK */
959 
960 /* It's common to want to use cpu_all_mask in struct member initializers,
961  * so it has to refer to an address rather than a pointer. */
962 extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
963 #define cpu_all_mask to_cpumask(cpu_all_bits)
964 
965 /* First bits of cpu_bit_bitmap are in fact unset. */
966 #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
967 
968 #if NR_CPUS == 1
969 /* Uniprocessor: the possible/online/present masks are always "1" */
970 #define for_each_possible_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)
971 #define for_each_online_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)
972 #define for_each_present_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)
973 #else
974 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
975 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
976 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
977 #endif
978 
979 /* Wrappers for arch boot code to manipulate normally-constant masks */
980 void init_cpu_present(const struct cpumask *src);
981 void init_cpu_possible(const struct cpumask *src);
982 void init_cpu_online(const struct cpumask *src);
983 
reset_cpu_possible_mask(void)984 static inline void reset_cpu_possible_mask(void)
985 {
986 	bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
987 }
988 
989 static inline void
set_cpu_possible(unsigned int cpu,bool possible)990 set_cpu_possible(unsigned int cpu, bool possible)
991 {
992 	if (possible)
993 		cpumask_set_cpu(cpu, &__cpu_possible_mask);
994 	else
995 		cpumask_clear_cpu(cpu, &__cpu_possible_mask);
996 }
997 
998 static inline void
set_cpu_present(unsigned int cpu,bool present)999 set_cpu_present(unsigned int cpu, bool present)
1000 {
1001 	if (present)
1002 		cpumask_set_cpu(cpu, &__cpu_present_mask);
1003 	else
1004 		cpumask_clear_cpu(cpu, &__cpu_present_mask);
1005 }
1006 
1007 void set_cpu_online(unsigned int cpu, bool online);
1008 
1009 static inline void
set_cpu_active(unsigned int cpu,bool active)1010 set_cpu_active(unsigned int cpu, bool active)
1011 {
1012 	if (active)
1013 		cpumask_set_cpu(cpu, &__cpu_active_mask);
1014 	else
1015 		cpumask_clear_cpu(cpu, &__cpu_active_mask);
1016 }
1017 
1018 static inline void
set_cpu_dying(unsigned int cpu,bool dying)1019 set_cpu_dying(unsigned int cpu, bool dying)
1020 {
1021 	if (dying)
1022 		cpumask_set_cpu(cpu, &__cpu_dying_mask);
1023 	else
1024 		cpumask_clear_cpu(cpu, &__cpu_dying_mask);
1025 }
1026 
1027 /**
1028  * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
1029  * @bitmap: the bitmap
1030  *
1031  * There are a few places where cpumask_var_t isn't appropriate and
1032  * static cpumasks must be used (eg. very early boot), yet we don't
1033  * expose the definition of 'struct cpumask'.
1034  *
1035  * This does the conversion, and can be used as a constant initializer.
1036  */
1037 #define to_cpumask(bitmap)						\
1038 	((struct cpumask *)(1 ? (bitmap)				\
1039 			    : (void *)sizeof(__check_is_bitmap(bitmap))))
1040 
__check_is_bitmap(const unsigned long * bitmap)1041 static inline int __check_is_bitmap(const unsigned long *bitmap)
1042 {
1043 	return 1;
1044 }
1045 
1046 /*
1047  * Special-case data structure for "single bit set only" constant CPU masks.
1048  *
1049  * We pre-generate all the 64 (or 32) possible bit positions, with enough
1050  * padding to the left and the right, and return the constant pointer
1051  * appropriately offset.
1052  */
1053 extern const unsigned long
1054 	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1055 
get_cpu_mask(unsigned int cpu)1056 static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
1057 {
1058 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
1059 	p -= cpu / BITS_PER_LONG;
1060 	return to_cpumask(p);
1061 }
1062 
1063 #if NR_CPUS > 1
1064 /**
1065  * num_online_cpus() - Read the number of online CPUs
1066  *
1067  * Despite the fact that __num_online_cpus is of type atomic_t, this
1068  * interface gives only a momentary snapshot and is not protected against
1069  * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1070  * region.
1071  */
num_online_cpus(void)1072 static __always_inline unsigned int num_online_cpus(void)
1073 {
1074 	return raw_atomic_read(&__num_online_cpus);
1075 }
1076 #define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
1077 #define num_present_cpus()	cpumask_weight(cpu_present_mask)
1078 #define num_active_cpus()	cpumask_weight(cpu_active_mask)
1079 
cpu_online(unsigned int cpu)1080 static inline bool cpu_online(unsigned int cpu)
1081 {
1082 	return cpumask_test_cpu(cpu, cpu_online_mask);
1083 }
1084 
cpu_possible(unsigned int cpu)1085 static inline bool cpu_possible(unsigned int cpu)
1086 {
1087 	return cpumask_test_cpu(cpu, cpu_possible_mask);
1088 }
1089 
cpu_present(unsigned int cpu)1090 static inline bool cpu_present(unsigned int cpu)
1091 {
1092 	return cpumask_test_cpu(cpu, cpu_present_mask);
1093 }
1094 
cpu_active(unsigned int cpu)1095 static inline bool cpu_active(unsigned int cpu)
1096 {
1097 	return cpumask_test_cpu(cpu, cpu_active_mask);
1098 }
1099 
cpu_dying(unsigned int cpu)1100 static inline bool cpu_dying(unsigned int cpu)
1101 {
1102 	return cpumask_test_cpu(cpu, cpu_dying_mask);
1103 }
1104 
1105 #else
1106 
1107 #define num_online_cpus()	1U
1108 #define num_possible_cpus()	1U
1109 #define num_present_cpus()	1U
1110 #define num_active_cpus()	1U
1111 
cpu_online(unsigned int cpu)1112 static inline bool cpu_online(unsigned int cpu)
1113 {
1114 	return cpu == 0;
1115 }
1116 
cpu_possible(unsigned int cpu)1117 static inline bool cpu_possible(unsigned int cpu)
1118 {
1119 	return cpu == 0;
1120 }
1121 
cpu_present(unsigned int cpu)1122 static inline bool cpu_present(unsigned int cpu)
1123 {
1124 	return cpu == 0;
1125 }
1126 
cpu_active(unsigned int cpu)1127 static inline bool cpu_active(unsigned int cpu)
1128 {
1129 	return cpu == 0;
1130 }
1131 
cpu_dying(unsigned int cpu)1132 static inline bool cpu_dying(unsigned int cpu)
1133 {
1134 	return false;
1135 }
1136 
1137 #endif /* NR_CPUS > 1 */
1138 
1139 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))
1140 
1141 #if NR_CPUS <= BITS_PER_LONG
1142 #define CPU_BITS_ALL						\
1143 {								\
1144 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1145 }
1146 
1147 #else /* NR_CPUS > BITS_PER_LONG */
1148 
1149 #define CPU_BITS_ALL						\
1150 {								\
1151 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,		\
1152 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1153 }
1154 #endif /* NR_CPUS > BITS_PER_LONG */
1155 
1156 /**
1157  * cpumap_print_to_pagebuf  - copies the cpumask into the buffer either
1158  *	as comma-separated list of cpus or hex values of cpumask
1159  * @list: indicates whether the cpumap must be list
1160  * @mask: the cpumask to copy
1161  * @buf: the buffer to copy into
1162  *
1163  * Returns the length of the (null-terminated) @buf string, zero if
1164  * nothing is copied.
1165  */
1166 static inline ssize_t
cpumap_print_to_pagebuf(bool list,char * buf,const struct cpumask * mask)1167 cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
1168 {
1169 	return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
1170 				      nr_cpu_ids);
1171 }
1172 
1173 /**
1174  * cpumap_print_bitmask_to_buf  - copies the cpumask into the buffer as
1175  *	hex values of cpumask
1176  *
1177  * @buf: the buffer to copy into
1178  * @mask: the cpumask to copy
1179  * @off: in the string from which we are copying, we copy to @buf
1180  * @count: the maximum number of bytes to print
1181  *
1182  * The function prints the cpumask into the buffer as hex values of
1183  * cpumask; Typically used by bin_attribute to export cpumask bitmask
1184  * ABI.
1185  *
1186  * Returns the length of how many bytes have been copied, excluding
1187  * terminating '\0'.
1188  */
1189 static inline ssize_t
cpumap_print_bitmask_to_buf(char * buf,const struct cpumask * mask,loff_t off,size_t count)1190 cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
1191 		loff_t off, size_t count)
1192 {
1193 	return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1194 				   nr_cpu_ids, off, count) - 1;
1195 }
1196 
1197 /**
1198  * cpumap_print_list_to_buf  - copies the cpumask into the buffer as
1199  *	comma-separated list of cpus
1200  * @buf: the buffer to copy into
1201  * @mask: the cpumask to copy
1202  * @off: in the string from which we are copying, we copy to @buf
1203  * @count: the maximum number of bytes to print
1204  *
1205  * Everything is same with the above cpumap_print_bitmask_to_buf()
1206  * except the print format.
1207  */
1208 static inline ssize_t
cpumap_print_list_to_buf(char * buf,const struct cpumask * mask,loff_t off,size_t count)1209 cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
1210 		loff_t off, size_t count)
1211 {
1212 	return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1213 				   nr_cpu_ids, off, count) - 1;
1214 }
1215 
1216 #if NR_CPUS <= BITS_PER_LONG
1217 #define CPU_MASK_ALL							\
1218 (cpumask_t) { {								\
1219 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1220 } }
1221 #else
1222 #define CPU_MASK_ALL							\
1223 (cpumask_t) { {								\
1224 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,			\
1225 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1226 } }
1227 #endif /* NR_CPUS > BITS_PER_LONG */
1228 
1229 #define CPU_MASK_NONE							\
1230 (cpumask_t) { {								\
1231 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] =  0UL				\
1232 } }
1233 
1234 #define CPU_MASK_CPU0							\
1235 (cpumask_t) { {								\
1236 	[0] =  1UL							\
1237 } }
1238 
1239 /*
1240  * Provide a valid theoretical max size for cpumap and cpulist sysfs files
1241  * to avoid breaking userspace which may allocate a buffer based on the size
1242  * reported by e.g. fstat.
1243  *
1244  * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1245  *
1246  * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1247  * to 2 orders of magnitude larger than 8192. And then we divide by 2 to
1248  * cover a worst-case of every other cpu being on one of two nodes for a
1249  * very large NR_CPUS.
1250  *
1251  *  Use PAGE_SIZE as a minimum for smaller configurations while avoiding
1252  *  unsigned comparison to -1.
1253  */
1254 #define CPUMAP_FILE_MAX_BYTES  (((NR_CPUS * 9)/32 > PAGE_SIZE) \
1255 					? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
1256 #define CPULIST_FILE_MAX_BYTES  (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
1257 
1258 #endif /* __LINUX_CPUMASK_H */
1259