1 /* SPDX-License-Identifier: GPL-2.0 */
7  * set of CPU's in a system, one bit position per CPU number.  In general,
22  * cpumask_bits - get the bits in a cpumask
26  * a macro so it's const-correct.
28 #define cpumask_bits(maskp) ((maskp)->bits)
31  * cpumask_pr_args - printf args to output a cpumask
38 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
46 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)  in set_nr_cpu_ids()
60  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
61  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
62  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
63  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
67  *  The cpu_possible_mask is fixed at boot time, as the set of CPU id's
82  * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
83  *    assumption that their single CPU is online.  The UP
87  *    optimization - don't waste any instructions or memory references
89  *    only one CPU.
107 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)  in cpu_max_bits_warn()  argument
110 	WARN_ON_ONCE(cpu >= bits);  in cpu_max_bits_warn()
114 /* verify cpu argument to cpumask_* operators */
115 static __always_inline unsigned int cpumask_check(unsigned int cpu)  in cpumask_check()  argument
117 	cpu_max_bits_warn(cpu, nr_cpumask_bits);  in cpumask_check()
118 	return cpu;  in cpumask_check()
122  * cpumask_first - get the first cpu in a cpumask
133  * cpumask_first_zero - get the first unset cpu in a cpumask
144  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
157  * cpumask_last - get the last CPU in a cpumask
158  * @srcp:	- the cpumask pointer
168  * cpumask_next - get the next cpu in a cpumask
169  * @n: the cpu prior to the place to search (ie. return will be > @n)
177 	/* -1 is a legal arg here. */  in cpumask_next()
178 	if (n != -1)  in cpumask_next()
180 	return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);  in cpumask_next()
184  * cpumask_next_zero - get the next unset cpu in a cpumask
185  * @n: the cpu prior to the place to search (ie. return will be > @n)
192 	/* -1 is a legal arg here. */  in cpumask_next_zero()
193 	if (n != -1)  in cpumask_next_zero()
195 	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);  in cpumask_next_zero()
198 #if NR_CPUS == 1
199 /* Uniprocessor: there is only one valid CPU */
223  * cpumask_next_and - get the next cpu in *src1p & *src2p
224  * @n: the cpu prior to the place to search (ie. return will be > @n)
234 	/* -1 is a legal arg here. */  in cpumask_next_and()
235 	if (n != -1)  in cpumask_next_and()
238 		nr_cpumask_bits, n + 1);  in cpumask_next_and()
242  * for_each_cpu - iterate over every cpu in a mask
243  * @cpu: the (optionally unsigned) integer iterator
246  * After the loop, cpu is >= nr_cpu_ids.
248 #define for_each_cpu(cpu, mask)				\  argument
249 	for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits)
252  * for_each_cpu_not - iterate over every cpu in a complemented mask
253  * @cpu: the (optionally unsigned) integer iterator
256  * After the loop, cpu is >= nr_cpu_ids.
258 #define for_each_cpu_not(cpu, mask)				\  argument
259 	for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits)
261 #if NR_CPUS == 1
266 	if (n != -1)  in cpumask_next_wrap()
270 	 * Return the first available CPU when wrapping, or when starting before cpu0,  in cpumask_next_wrap()
283  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
284  * @cpu: the (optionally unsigned) integer iterator
290  * After the loop, cpu is >= nr_cpu_ids.
292 #define for_each_cpu_wrap(cpu, mask, start)				\  argument
293 	for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start)
296  * for_each_cpu_and - iterate over every cpu in both masks
297  * @cpu: the (optionally unsigned) integer iterator
301  * This saves a temporary CPU mask in many places.  It is equivalent to:
304  *	for_each_cpu(cpu, &tmp)
307  * After the loop, cpu is >= nr_cpu_ids.
309 #define for_each_cpu_and(cpu, mask1, mask2)				\  argument
310 	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits)
313  * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
315  * @cpu: the (optionally unsigned) integer iterator
319  * This saves a temporary CPU mask in many places.  It is equivalent to:
322  *	for_each_cpu(cpu, &tmp)
325  * After the loop, cpu is >= nr_cpu_ids.
327 #define for_each_cpu_andnot(cpu, mask1, mask2)				\  argument
328 	for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits)
331  * cpumask_any_but - return a "random" in a cpumask, but not this one.
333  * @cpu: the cpu to ignore.
335  * Often used to find any cpu but smp_processor_id() in a mask.
339 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)  in cpumask_any_but()  argument
343 	cpumask_check(cpu);  in cpumask_any_but()
345 		if (i != cpu)  in cpumask_any_but()
351  * cpumask_nth - get the first cpu in a cpumask
353  * @cpu: the N'th cpu to find, starting from 0
355  * Returns >= nr_cpu_ids if such cpu doesn't exist.
357 static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)  in cpumask_nth()  argument
359 	return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu));  in cpumask_nth()
363  * cpumask_nth_and - get the first cpu in 2 cpumasks
366  * @cpu: the N'th cpu to find, starting from 0
368  * Returns >= nr_cpu_ids if such cpu doesn't exist.
371 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_and()  argument
375 				nr_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_and()
379  * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd.
382  * @cpu: the N'th cpu to find, starting from 0
384  * Returns >= nr_cpu_ids if such cpu doesn't exist.
387 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_andnot()  argument
391 				nr_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_andnot()
396 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
401 	[0] =  1UL						\
405  * cpumask_set_cpu - set a cpu in a cpumask
406  * @cpu: cpu number (< nr_cpu_ids)
409 static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)  in cpumask_set_cpu()  argument
411 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));  in cpumask_set_cpu()
414 static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)  in __cpumask_set_cpu()  argument
416 	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));  in __cpumask_set_cpu()
421  * cpumask_clear_cpu - clear a cpu in a cpumask
422  * @cpu: cpu number (< nr_cpu_ids)
425 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)  in cpumask_clear_cpu()  argument
427 	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));  in cpumask_clear_cpu()
430 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)  in __cpumask_clear_cpu()  argument
432 	__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));  in __cpumask_clear_cpu()
436  * cpumask_test_cpu - test for a cpu in a cpumask
437  * @cpu: cpu number (< nr_cpu_ids)
440  * Returns true if @cpu is set in @cpumask, else returns false
442 static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)  in cpumask_test_cpu()  argument
444 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));  in cpumask_test_cpu()
448  * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
449  * @cpu: cpu number (< nr_cpu_ids)
452  * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
456 static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)  in cpumask_test_and_set_cpu()  argument
458 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));  in cpumask_test_and_set_cpu()
462  * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
463  * @cpu: cpu number (< nr_cpu_ids)
466  * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
470 static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)  in cpumask_test_and_clear_cpu()  argument
472 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));  in cpumask_test_and_clear_cpu()
476  * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
485  * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
494  * cpumask_and - *dstp = *src1p & *src2p
510  * cpumask_or - *dstp = *src1p | *src2p
523  * cpumask_xor - *dstp = *src1p ^ *src2p
537  * cpumask_andnot - *dstp = *src1p & ~*src2p
553  * cpumask_complement - *dstp = ~*srcp
565  * cpumask_equal - *src1p == *src2p
577  * cpumask_or_equal - *src1p | *src2p == *src3p
591  * cpumask_intersects - (*src1p & *src2p) != 0
603  * cpumask_subset - (*src1p & ~*src2p) == 0
617  * cpumask_empty - *srcp == 0
626  * cpumask_full - *srcp == 0xFFFFFFFF...
635  * cpumask_weight - Count of bits in *srcp
644  * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
655  * cpumask_shift_right - *dstp = *srcp >> n
668  * cpumask_shift_left - *dstp = *srcp << n
681  * cpumask_copy - *dstp = *srcp
692  * cpumask_any - pick a "random" cpu from *srcp
700  * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
709  * cpumask_of - the cpumask containing just a given cpu
710  * @cpu: the cpu (<= nr_cpu_ids)
712 #define cpumask_of(cpu) (get_cpu_mask(cpu))  argument
715  * cpumask_parse_user - extract a cpumask from a user string
720  * Returns -errno, or 0 for success.
729  * cpumask_parselist_user - extract a cpumask from a user string
734  * Returns -errno, or 0 for success.
744  * cpumask_parse - extract a cpumask from a string
748  * Returns -errno, or 0 for success.
756  * cpulist_parse - extract a cpumask from a user string of ranges
760  * Returns -errno, or 0 for success.
768  * cpumask_size - size to allocate for a 'struct cpumask' in bytes
785  *		return -ENOMEM;
798  *		return -ENOMEM;
830  * alloc_cpumask_var - allocate a struct cpumask
835  * a nop returning a constant 1 (in <linux/cpumask.h>).
861 typedef struct cpumask cpumask_var_t[1];
916 #if NR_CPUS == 1
917 /* Uniprocessor: the possible/online/present masks are always "1" */
918 #define for_each_possible_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
919 #define for_each_online_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
920 #define for_each_present_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
922 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)  argument
923 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)  argument
924 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)  argument
927 /* Wrappers for arch boot code to manipulate normally-constant masks */
938 set_cpu_possible(unsigned int cpu, bool possible)  in set_cpu_possible()  argument
941 		cpumask_set_cpu(cpu, &__cpu_possible_mask);  in set_cpu_possible()
943 		cpumask_clear_cpu(cpu, &__cpu_possible_mask);  in set_cpu_possible()
947 set_cpu_present(unsigned int cpu, bool present)  in set_cpu_present()  argument
950 		cpumask_set_cpu(cpu, &__cpu_present_mask);  in set_cpu_present()
952 		cpumask_clear_cpu(cpu, &__cpu_present_mask);  in set_cpu_present()
955 void set_cpu_online(unsigned int cpu, bool online);
958 set_cpu_active(unsigned int cpu, bool active)  in set_cpu_active()  argument
961 		cpumask_set_cpu(cpu, &__cpu_active_mask);  in set_cpu_active()
963 		cpumask_clear_cpu(cpu, &__cpu_active_mask);  in set_cpu_active()
967 set_cpu_dying(unsigned int cpu, bool dying)  in set_cpu_dying()  argument
970 		cpumask_set_cpu(cpu, &__cpu_dying_mask);  in set_cpu_dying()
972 		cpumask_clear_cpu(cpu, &__cpu_dying_mask);  in set_cpu_dying()
976  * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
986 	((struct cpumask *)(1 ? (bitmap)				\
991 	return 1;  in __check_is_bitmap()
995  * Special-case data structure for "single bit set only" constant CPU masks.
997  * We pre-generate all the 64 (or 32) possible bit positions, with enough
1002 	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1004 static inline const struct cpumask *get_cpu_mask(unsigned int cpu)  in get_cpu_mask()  argument
1006 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];  in get_cpu_mask()
1007 	p -= cpu / BITS_PER_LONG;  in get_cpu_mask()
1011 #if NR_CPUS > 1
1013  * num_online_cpus() - Read the number of online CPUs
1017  * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1028 static inline bool cpu_online(unsigned int cpu)  in cpu_online()  argument
1030 	return cpumask_test_cpu(cpu, cpu_online_mask);  in cpu_online()
1033 static inline bool cpu_possible(unsigned int cpu)  in cpu_possible()  argument
1035 	return cpumask_test_cpu(cpu, cpu_possible_mask);  in cpu_possible()
1038 static inline bool cpu_present(unsigned int cpu)  in cpu_present()  argument
1040 	return cpumask_test_cpu(cpu, cpu_present_mask);  in cpu_present()
1043 static inline bool cpu_active(unsigned int cpu)  in cpu_active()  argument
1045 	return cpumask_test_cpu(cpu, cpu_active_mask);  in cpu_active()
1048 static inline bool cpu_dying(unsigned int cpu)  in cpu_dying()  argument
1050 	return cpumask_test_cpu(cpu, cpu_dying_mask);  in cpu_dying()
1055 #define num_online_cpus()	1U
1056 #define num_possible_cpus()	1U
1057 #define num_present_cpus()	1U
1058 #define num_active_cpus()	1U
1060 static inline bool cpu_online(unsigned int cpu)  in cpu_online()  argument
1062 	return cpu == 0;  in cpu_online()
1065 static inline bool cpu_possible(unsigned int cpu)  in cpu_possible()  argument
1067 	return cpu == 0;  in cpu_possible()
1070 static inline bool cpu_present(unsigned int cpu)  in cpu_present()  argument
1072 	return cpu == 0;  in cpu_present()
1075 static inline bool cpu_active(unsigned int cpu)  in cpu_active()  argument
1077 	return cpu == 0;  in cpu_active()
1080 static inline bool cpu_dying(unsigned int cpu)  in cpu_dying()  argument
1085 #endif /* NR_CPUS > 1 */
1087 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))  argument
1092 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1099 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,		\
1100 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1105  * cpumap_print_to_pagebuf  - copies the cpumask into the buffer either
1106  *	as comma-separated list of cpus or hex values of cpumask
1111  * Returns the length of the (null-terminated) @buf string, zero if
1122  * cpumap_print_bitmask_to_buf  - copies the cpumask into the buffer as
1142 				   nr_cpu_ids, off, count) - 1;  in cpumap_print_bitmask_to_buf()
1146  * cpumap_print_list_to_buf  - copies the cpumask into the buffer as
1147  *	comma-separated list of cpus
1157 				   nr_cpu_ids, off, count) - 1;  in cpumap_print_list_to_buf()
1163 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1168 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,			\
1169 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1175 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] =  0UL				\
1180 	[0] =  1UL							\
1188  * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1190  * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1192  * cover a worst-case of every other cpu being on one of two nodes for a
1196  *  unsigned comparison to -1.
1199 					? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)