72 static inline void __seqcount_init(seqcount_t *s, const char *name,  in __seqcount_init()  argument
78 	lockdep_init_map(&s->dep_map, name, key, 0);  in __seqcount_init()
79 	s->sequence = 0;  in __seqcount_init()
91 # define seqcount_init(s)						\  argument
94 		__seqcount_init((s), #s, &__key);			\
97 static inline void seqcount_lockdep_reader_access(const seqcount_t *s)  in seqcount_lockdep_reader_access()  argument
99 	seqcount_t *l = (seqcount_t *)s;  in seqcount_lockdep_reader_access()
110 # define seqcount_init(s) __seqcount_init(s, NULL, NULL)  argument
176 #define seqcount_LOCKNAME_init(s, _lock, lockname)			\  argument
178 		seqcount_##lockname##_t *____s = (s);			\
183 #define seqcount_raw_spinlock_init(s, lock)	seqcount_LOCKNAME_init(s, lock, raw_spinlock)  argument
184 #define seqcount_spinlock_init(s, lock)		seqcount_LOCKNAME_init(s, lock, spinlock)  argument
185 #define seqcount_rwlock_init(s, lock)		seqcount_LOCKNAME_init(s, lock, rwlock);  argument
186 #define seqcount_mutex_init(s, lock)		seqcount_LOCKNAME_init(s, lock, mutex);  argument
187 #define seqcount_ww_mutex_init(s, lock)		seqcount_LOCKNAME_init(s, lock, ww_mutex);  argument
207 __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s)			\
209 	return &s->seqcount;						\
213 __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s)	\
215 	unsigned seq = READ_ONCE(s->seqcount.sequence);			\
222 		__SEQ_LOCK(lockbase##_unlock(s->lock));			\
228 		seq = READ_ONCE(s->seqcount.sequence);			\
235 __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s)	\
245 __seqprop_##lockname##_assert(const seqcount_##lockname##_t *s)		\
254 static inline seqcount_t *__seqprop_ptr(seqcount_t *s)  in __seqprop_ptr()  argument
256 	return s;  in __seqprop_ptr()
259 static inline unsigned __seqprop_sequence(const seqcount_t *s)  in __seqprop_sequence()  argument
261 	return READ_ONCE(s->sequence);  in __seqprop_sequence()
264 static inline bool __seqprop_preemptible(const seqcount_t *s)  in __seqprop_preemptible()  argument
269 static inline void __seqprop_assert(const seqcount_t *s)  in __seqprop_assert()  argument
276 …COUNT_LOCKNAME(raw_spinlock, raw_spinlock_t,  false,    s->lock,        raw_spin, raw_spin_lock(s-…
277 SEQCOUNT_LOCKNAME(spinlock,     spinlock_t,      __SEQ_RT, s->lock,        spin,     spin_lock(s->l…
278 SEQCOUNT_LOCKNAME(rwlock,       rwlock_t,        __SEQ_RT, s->lock,        read,     read_lock(s->l…
279 SEQCOUNT_LOCKNAME(mutex,        struct mutex,    true,     s->lock,        mutex,    mutex_lock(s->…
280 …OUNT_LOCKNAME(ww_mutex,     struct ww_mutex, true,     &s->lock->base, ww_mutex, ww_mutex_lock(s->…
299 #define __seqprop_case(s, lockname, prop)				\  argument
300 	seqcount_##lockname##_t: __seqprop_##lockname##_##prop((void *)(s))
302 #define __seqprop(s, prop) _Generic(*(s),				\  argument
303 	seqcount_t:		__seqprop_##prop((void *)(s)),		\
304 	__seqprop_case((s),	raw_spinlock,	prop),			\
305 	__seqprop_case((s),	spinlock,	prop),			\
306 	__seqprop_case((s),	rwlock,		prop),			\
307 	__seqprop_case((s),	mutex,		prop),			\
308 	__seqprop_case((s),	ww_mutex,	prop))
310 #define __seqcount_ptr(s)		__seqprop(s, ptr)  argument
311 #define __seqcount_sequence(s)		__seqprop(s, sequence)  argument
312 #define __seqcount_lock_preemptible(s)	__seqprop(s, preemptible)  argument
313 #define __seqcount_assert_lock_held(s)	__seqprop(s, assert)  argument
329 #define __read_seqcount_begin(s)					\  argument
333 	while ((seq = __seqcount_sequence(s)) & 1)			\
346 #define raw_read_seqcount_begin(s)					\  argument
348 	unsigned seq = __read_seqcount_begin(s);			\
360 #define read_seqcount_begin(s)						\  argument
362 	seqcount_lockdep_reader_access(__seqcount_ptr(s));		\
363 	raw_read_seqcount_begin(s);					\
377 #define raw_read_seqcount(s)						\  argument
379 	unsigned seq = __seqcount_sequence(s);				\
403 #define raw_seqcount_begin(s)						\  argument
409 	raw_read_seqcount(s) & ~1;					\
427 #define __read_seqcount_retry(s, start)					\  argument
428 	__read_seqcount_t_retry(__seqcount_ptr(s), start)
430 static inline int __read_seqcount_t_retry(const seqcount_t *s, unsigned start)  in __read_seqcount_t_retry()  argument
433 	return unlikely(READ_ONCE(s->sequence) != start);  in __read_seqcount_t_retry()
447 #define read_seqcount_retry(s, start)					\  argument
448 	read_seqcount_t_retry(__seqcount_ptr(s), start)
450 static inline int read_seqcount_t_retry(const seqcount_t *s, unsigned start)  in read_seqcount_t_retry()  argument
453 	return __read_seqcount_t_retry(s, start);  in read_seqcount_t_retry()
460 #define raw_write_seqcount_begin(s)					\  argument
462 	if (__seqcount_lock_preemptible(s))				\
465 	raw_write_seqcount_t_begin(__seqcount_ptr(s));			\
468 static inline void raw_write_seqcount_t_begin(seqcount_t *s)  in raw_write_seqcount_t_begin()  argument
471 	s->sequence++;  in raw_write_seqcount_t_begin()
479 #define raw_write_seqcount_end(s)					\  argument
481 	raw_write_seqcount_t_end(__seqcount_ptr(s));			\
483 	if (__seqcount_lock_preemptible(s))				\
487 static inline void raw_write_seqcount_t_end(seqcount_t *s)  in raw_write_seqcount_t_end()  argument
490 	s->sequence++;  in raw_write_seqcount_t_end()
502 #define write_seqcount_begin_nested(s, subclass)			\  argument
504 	__seqcount_assert_lock_held(s);					\
506 	if (__seqcount_lock_preemptible(s))				\
509 	write_seqcount_t_begin_nested(__seqcount_ptr(s), subclass);	\
512 static inline void write_seqcount_t_begin_nested(seqcount_t *s, int subclass)  in write_seqcount_t_begin_nested()  argument
514 	raw_write_seqcount_t_begin(s);  in write_seqcount_t_begin_nested()
515 	seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_);  in write_seqcount_t_begin_nested()
529 #define write_seqcount_begin(s)						\  argument
531 	__seqcount_assert_lock_held(s);					\
533 	if (__seqcount_lock_preemptible(s))				\
536 	write_seqcount_t_begin(__seqcount_ptr(s));			\
539 static inline void write_seqcount_t_begin(seqcount_t *s)  in write_seqcount_t_begin()  argument
541 	write_seqcount_t_begin_nested(s, 0);  in write_seqcount_t_begin()
550 #define write_seqcount_end(s)						\  argument
552 	write_seqcount_t_end(__seqcount_ptr(s));			\
554 	if (__seqcount_lock_preemptible(s))				\
558 static inline void write_seqcount_t_end(seqcount_t *s)  in write_seqcount_t_end()  argument
560 	seqcount_release(&s->dep_map, _RET_IP_);  in write_seqcount_t_end()
561 	raw_write_seqcount_t_end(s);  in write_seqcount_t_end()
605 #define raw_write_seqcount_barrier(s)					\  argument
606 	raw_write_seqcount_t_barrier(__seqcount_ptr(s))
608 static inline void raw_write_seqcount_t_barrier(seqcount_t *s)  in raw_write_seqcount_t_barrier()  argument
611 	s->sequence++;  in raw_write_seqcount_t_barrier()
613 	s->sequence++;  in raw_write_seqcount_t_barrier()
625 #define write_seqcount_invalidate(s)					\  argument
626 	write_seqcount_t_invalidate(__seqcount_ptr(s))
628 static inline void write_seqcount_t_invalidate(seqcount_t *s)  in write_seqcount_t_invalidate()  argument
632 	s->sequence+=2;  in write_seqcount_t_invalidate()
662 static inline void seqcount_latch_init(seqcount_latch_t *s)  in seqcount_latch_init()  argument
664 	seqcount_init(&s->seqcount);  in seqcount_latch_init()
678 static inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s)  in raw_read_seqcount_latch()  argument
684 	return READ_ONCE(s->seqcount.sequence);  in raw_read_seqcount_latch()
695 read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)  in read_seqcount_latch_retry()  argument
697 	return read_seqcount_retry(&s->seqcount, start);  in read_seqcount_latch_retry()
781 static inline void raw_write_seqcount_latch(seqcount_latch_t *s)  in raw_write_seqcount_latch()  argument
784 	s->seqcount.sequence++;  in raw_write_seqcount_latch()