72 static inline void __seqcount_init(seqcount_t *s, const char *name,  in __seqcount_init()  argument
78 	lockdep_init_map(&s->dep_map, name, key, 0);  in __seqcount_init()
79 	s->sequence = 0;  in __seqcount_init()
91 # define seqcount_init(s)						\  argument
94 		__seqcount_init((s), #s, &__key);			\
97 static inline void seqcount_lockdep_reader_access(const seqcount_t *s)  in seqcount_lockdep_reader_access()  argument
99 	seqcount_t *l = (seqcount_t *)s;  in seqcount_lockdep_reader_access()
110 # define seqcount_init(s) __seqcount_init(s, NULL, NULL)  argument
176 #define seqcount_LOCKNAME_init(s, _lock, lockname)			\  argument
178 		seqcount_##lockname##_t *____s = (s);			\
183 #define seqcount_raw_spinlock_init(s, lock)	seqcount_LOCKNAME_init(s, lock, raw_spinlock)  argument
184 #define seqcount_spinlock_init(s, lock)		seqcount_LOCKNAME_init(s, lock, spinlock)  argument
185 #define seqcount_rwlock_init(s, lock)		seqcount_LOCKNAME_init(s, lock, rwlock)  argument
186 #define seqcount_mutex_init(s, lock)		seqcount_LOCKNAME_init(s, lock, mutex)  argument
187 #define seqcount_ww_mutex_init(s, lock)		seqcount_LOCKNAME_init(s, lock, ww_mutex)  argument
207 __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s)			\
209 	return &s->seqcount;						\
213 __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s)	\
215 	unsigned seq = READ_ONCE(s->seqcount.sequence);			\
222 		__SEQ_LOCK(lockbase##_unlock(s->lock));			\
228 		seq = READ_ONCE(s->seqcount.sequence);			\
235 __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s)	\
245 __seqprop_##lockname##_assert(const seqcount_##lockname##_t *s)		\
254 static inline seqcount_t *__seqprop_ptr(seqcount_t *s)  in __seqprop_ptr()  argument
256 	return s;  in __seqprop_ptr()
259 static inline unsigned __seqprop_sequence(const seqcount_t *s)  in __seqprop_sequence()  argument
261 	return READ_ONCE(s->sequence);  in __seqprop_sequence()
264 static inline bool __seqprop_preemptible(const seqcount_t *s)  in __seqprop_preemptible()  argument
269 static inline void __seqprop_assert(const seqcount_t *s)  in __seqprop_assert()  argument
276 …COUNT_LOCKNAME(raw_spinlock, raw_spinlock_t,  false,    s->lock,        raw_spin, raw_spin_lock(s-…
277 SEQCOUNT_LOCKNAME(spinlock,     spinlock_t,      __SEQ_RT, s->lock,        spin,     spin_lock(s->l…
278 SEQCOUNT_LOCKNAME(rwlock,       rwlock_t,        __SEQ_RT, s->lock,        read,     read_lock(s->l…
279 SEQCOUNT_LOCKNAME(mutex,        struct mutex,    true,     s->lock,        mutex,    mutex_lock(s->…
280 …OUNT_LOCKNAME(ww_mutex,     struct ww_mutex, true,     &s->lock->base, ww_mutex, ww_mutex_lock(s->…
299 #define __seqprop_case(s, lockname, prop)				\  argument
300 	seqcount_##lockname##_t: __seqprop_##lockname##_##prop((void *)(s))
302 #define __seqprop(s, prop) _Generic(*(s),				\  argument
303 	seqcount_t:		__seqprop_##prop((void *)(s)),		\
304 	__seqprop_case((s),	raw_spinlock,	prop),			\
305 	__seqprop_case((s),	spinlock,	prop),			\
306 	__seqprop_case((s),	rwlock,		prop),			\
307 	__seqprop_case((s),	mutex,		prop),			\
308 	__seqprop_case((s),	ww_mutex,	prop))
310 #define seqprop_ptr(s)			__seqprop(s, ptr)  argument
311 #define seqprop_sequence(s)		__seqprop(s, sequence)  argument
312 #define seqprop_preemptible(s)		__seqprop(s, preemptible)  argument
313 #define seqprop_assert(s)		__seqprop(s, assert)  argument
329 #define __read_seqcount_begin(s)					\  argument
333 	while ((__seq = seqprop_sequence(s)) & 1)			\
346 #define raw_read_seqcount_begin(s)					\  argument
348 	unsigned _seq = __read_seqcount_begin(s);			\
360 #define read_seqcount_begin(s)						\  argument
362 	seqcount_lockdep_reader_access(seqprop_ptr(s));			\
363 	raw_read_seqcount_begin(s);					\
377 #define raw_read_seqcount(s)						\  argument
379 	unsigned __seq = seqprop_sequence(s);				\
403 #define raw_seqcount_begin(s)						\  argument
409 	raw_read_seqcount(s) & ~1;					\
427 #define __read_seqcount_retry(s, start)					\  argument
428 	do___read_seqcount_retry(seqprop_ptr(s), start)
430 static inline int do___read_seqcount_retry(const seqcount_t *s, unsigned start)  in do___read_seqcount_retry()  argument
433 	return unlikely(READ_ONCE(s->sequence) != start);  in do___read_seqcount_retry()
447 #define read_seqcount_retry(s, start)					\  argument
448 	do_read_seqcount_retry(seqprop_ptr(s), start)
450 static inline int do_read_seqcount_retry(const seqcount_t *s, unsigned start)  in do_read_seqcount_retry()  argument
453 	return do___read_seqcount_retry(s, start);  in do_read_seqcount_retry()
462 #define raw_write_seqcount_begin(s)					\  argument
464 	if (seqprop_preemptible(s))					\
467 	do_raw_write_seqcount_begin(seqprop_ptr(s));			\
470 static inline void do_raw_write_seqcount_begin(seqcount_t *s)  in do_raw_write_seqcount_begin()  argument
473 	s->sequence++;  in do_raw_write_seqcount_begin()
483 #define raw_write_seqcount_end(s)					\  argument
485 	do_raw_write_seqcount_end(seqprop_ptr(s));			\
487 	if (seqprop_preemptible(s))					\
491 static inline void do_raw_write_seqcount_end(seqcount_t *s)  in do_raw_write_seqcount_end()  argument
494 	s->sequence++;  in do_raw_write_seqcount_end()
507 #define write_seqcount_begin_nested(s, subclass)			\  argument
509 	seqprop_assert(s);						\
511 	if (seqprop_preemptible(s))					\
514 	do_write_seqcount_begin_nested(seqprop_ptr(s), subclass);	\
517 static inline void do_write_seqcount_begin_nested(seqcount_t *s, int subclass)  in do_write_seqcount_begin_nested()  argument
519 	do_raw_write_seqcount_begin(s);  in do_write_seqcount_begin_nested()
520 	seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_);  in do_write_seqcount_begin_nested()
533 #define write_seqcount_begin(s)						\  argument
535 	seqprop_assert(s);						\
537 	if (seqprop_preemptible(s))					\
540 	do_write_seqcount_begin(seqprop_ptr(s));			\
543 static inline void do_write_seqcount_begin(seqcount_t *s)  in do_write_seqcount_begin()  argument
545 	do_write_seqcount_begin_nested(s, 0);  in do_write_seqcount_begin()
555 #define write_seqcount_end(s)						\  argument
557 	do_write_seqcount_end(seqprop_ptr(s));				\
559 	if (seqprop_preemptible(s))					\
563 static inline void do_write_seqcount_end(seqcount_t *s)  in do_write_seqcount_end()  argument
565 	seqcount_release(&s->dep_map, _RET_IP_);  in do_write_seqcount_end()
566 	do_raw_write_seqcount_end(s);  in do_write_seqcount_end()
610 #define raw_write_seqcount_barrier(s)					\  argument
611 	do_raw_write_seqcount_barrier(seqprop_ptr(s))
613 static inline void do_raw_write_seqcount_barrier(seqcount_t *s)  in do_raw_write_seqcount_barrier()  argument
616 	s->sequence++;  in do_raw_write_seqcount_barrier()
618 	s->sequence++;  in do_raw_write_seqcount_barrier()
630 #define write_seqcount_invalidate(s)					\  argument
631 	do_write_seqcount_invalidate(seqprop_ptr(s))
633 static inline void do_write_seqcount_invalidate(seqcount_t *s)  in do_write_seqcount_invalidate()  argument
637 	s->sequence+=2;  in do_write_seqcount_invalidate()
667 #define seqcount_latch_init(s) seqcount_init(&(s)->seqcount)  argument
680 static inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s)  in raw_read_seqcount_latch()  argument
686 	return READ_ONCE(s->seqcount.sequence);  in raw_read_seqcount_latch()
697 read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)  in read_seqcount_latch_retry()  argument
699 	return read_seqcount_retry(&s->seqcount, start);  in read_seqcount_latch_retry()
783 static inline void raw_write_seqcount_latch(seqcount_latch_t *s)  in raw_write_seqcount_latch()  argument
786 	s->seqcount.sequence++;  in raw_write_seqcount_latch()