44 static inline unsigned long *
45 __bitops_word(unsigned long nr, const volatile unsigned long *ptr)  in __bitops_word()
47 	unsigned long addr;  in __bitops_word()
49 	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);  in __bitops_word()
50 	return (unsigned long *)addr;  in __bitops_word()
53 static inline unsigned long __bitops_mask(unsigned long nr)  in __bitops_mask()
58 static __always_inline void arch_set_bit(unsigned long nr, volatile unsigned long *ptr)  in arch_set_bit()
60 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_set_bit()
61 	unsigned long mask = __bitops_mask(nr);  in arch_set_bit()
63 	__atomic64_or(mask, (long *)addr);  in arch_set_bit()
66 static __always_inline void arch_clear_bit(unsigned long nr, volatile unsigned long *ptr)  in arch_clear_bit()
68 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_clear_bit()
69 	unsigned long mask = __bitops_mask(nr);  in arch_clear_bit()
71 	__atomic64_and(~mask, (long *)addr);  in arch_clear_bit()
74 static __always_inline void arch_change_bit(unsigned long nr,  in arch_change_bit()
75 					    volatile unsigned long *ptr)  in arch_change_bit()
77 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_change_bit()
78 	unsigned long mask = __bitops_mask(nr);  in arch_change_bit()
80 	__atomic64_xor(mask, (long *)addr);  in arch_change_bit()
83 static inline bool arch_test_and_set_bit(unsigned long nr,  in arch_test_and_set_bit()
84 					 volatile unsigned long *ptr)  in arch_test_and_set_bit()
86 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_and_set_bit()
87 	unsigned long mask = __bitops_mask(nr);  in arch_test_and_set_bit()
88 	unsigned long old;  in arch_test_and_set_bit()
90 	old = __atomic64_or_barrier(mask, (long *)addr);  in arch_test_and_set_bit()
94 static inline bool arch_test_and_clear_bit(unsigned long nr,  in arch_test_and_clear_bit()
95 					   volatile unsigned long *ptr)  in arch_test_and_clear_bit()
97 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_and_clear_bit()
98 	unsigned long mask = __bitops_mask(nr);  in arch_test_and_clear_bit()
99 	unsigned long old;  in arch_test_and_clear_bit()
101 	old = __atomic64_and_barrier(~mask, (long *)addr);  in arch_test_and_clear_bit()
105 static inline bool arch_test_and_change_bit(unsigned long nr,  in arch_test_and_change_bit()
106 					    volatile unsigned long *ptr)  in arch_test_and_change_bit()
108 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_and_change_bit()
109 	unsigned long mask = __bitops_mask(nr);  in arch_test_and_change_bit()
110 	unsigned long old;  in arch_test_and_change_bit()
112 	old = __atomic64_xor_barrier(mask, (long *)addr);  in arch_test_and_change_bit()
116 static inline void arch___set_bit(unsigned long nr, volatile unsigned long *ptr)  in arch___set_bit()
118 	unsigned long *addr = __bitops_word(nr, ptr);  in arch___set_bit()
119 	unsigned long mask = __bitops_mask(nr);  in arch___set_bit()
124 static inline void arch___clear_bit(unsigned long nr,  in arch___clear_bit()
125 				    volatile unsigned long *ptr)  in arch___clear_bit()
127 	unsigned long *addr = __bitops_word(nr, ptr);  in arch___clear_bit()
128 	unsigned long mask = __bitops_mask(nr);  in arch___clear_bit()
133 static inline void arch___change_bit(unsigned long nr,  in arch___change_bit()
134 				     volatile unsigned long *ptr)  in arch___change_bit()
136 	unsigned long *addr = __bitops_word(nr, ptr);  in arch___change_bit()
137 	unsigned long mask = __bitops_mask(nr);  in arch___change_bit()
142 static inline bool arch___test_and_set_bit(unsigned long nr,  in arch___test_and_set_bit()
143 					   volatile unsigned long *ptr)  in arch___test_and_set_bit()
145 	unsigned long *addr = __bitops_word(nr, ptr);  in arch___test_and_set_bit()
146 	unsigned long mask = __bitops_mask(nr);  in arch___test_and_set_bit()
147 	unsigned long old;  in arch___test_and_set_bit()
154 static inline bool arch___test_and_clear_bit(unsigned long nr,  in arch___test_and_clear_bit()
155 					     volatile unsigned long *ptr)  in arch___test_and_clear_bit()
157 	unsigned long *addr = __bitops_word(nr, ptr);  in arch___test_and_clear_bit()
158 	unsigned long mask = __bitops_mask(nr);  in arch___test_and_clear_bit()
159 	unsigned long old;  in arch___test_and_clear_bit()
166 static inline bool arch___test_and_change_bit(unsigned long nr,  in arch___test_and_change_bit()
167 					      volatile unsigned long *ptr)  in arch___test_and_change_bit()
169 	unsigned long *addr = __bitops_word(nr, ptr);  in arch___test_and_change_bit()
170 	unsigned long mask = __bitops_mask(nr);  in arch___test_and_change_bit()
171 	unsigned long old;  in arch___test_and_change_bit()
178 static inline bool arch_test_bit(unsigned long nr,  in arch_test_bit()
179 				 const volatile unsigned long *ptr)  in arch_test_bit()
181 	const volatile unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_bit()
182 	unsigned long mask = __bitops_mask(nr);  in arch_test_bit()
187 static inline bool arch_test_and_set_bit_lock(unsigned long nr,  in arch_test_and_set_bit_lock()
188 					      volatile unsigned long *ptr)  in arch_test_and_set_bit_lock()
195 static inline void arch_clear_bit_unlock(unsigned long nr,  in arch_clear_bit_unlock()
196 					 volatile unsigned long *ptr)  in arch_clear_bit_unlock()
202 static inline void arch___clear_bit_unlock(unsigned long nr,  in arch___clear_bit_unlock()
203 					   volatile unsigned long *ptr)  in arch___clear_bit_unlock()
218 unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
219 unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
220 				unsigned long offset);
227 static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in set_bit_inv()
232 static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in clear_bit_inv()
237 static inline bool test_and_clear_bit_inv(unsigned long nr,  in test_and_clear_bit_inv()
238 					  volatile unsigned long *ptr)  in test_and_clear_bit_inv()
243 static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in __set_bit_inv()
248 static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in __clear_bit_inv()
253 static inline bool test_bit_inv(unsigned long nr,  in test_bit_inv()
254 				const volatile unsigned long *ptr)  in test_bit_inv()
269 static inline unsigned char __flogr(unsigned long word)  in __flogr()
272 		unsigned long bit = 0;  in __flogr()
318 static inline unsigned long __ffs(unsigned long word)  in __ffs()
332 	unsigned long mask = 2 * BITS_PER_LONG - 1;  in ffs()
339  * __fls - find last (most-significant) set bit in a long word
344 static inline unsigned long __fls(unsigned long word)  in __fls()
360 static inline int fls64(unsigned long word)  in fls64()
362 	unsigned long mask = 2 * BITS_PER_LONG - 1;  in fls64()