32 static inline void percpu_down_read_preempt_disable(struct percpu_rw_semaphore *sem)  in percpu_down_read_preempt_disable()  argument
36 	rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_);  in percpu_down_read_preempt_disable()
47 	__this_cpu_inc(*sem->read_count);  in percpu_down_read_preempt_disable()
48 	if (unlikely(!rcu_sync_is_idle(&sem->rss)))  in percpu_down_read_preempt_disable()
49 		__percpu_down_read(sem, false); /* Unconditional memory barrier */  in percpu_down_read_preempt_disable()
57 static inline void percpu_down_read(struct percpu_rw_semaphore *sem)  in percpu_down_read()  argument
59 	percpu_down_read_preempt_disable(sem);  in percpu_down_read()
63 static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem)  in percpu_down_read_trylock()  argument
71 	__this_cpu_inc(*sem->read_count);  in percpu_down_read_trylock()
72 	if (unlikely(!rcu_sync_is_idle(&sem->rss)))  in percpu_down_read_trylock()
73 		ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */  in percpu_down_read_trylock()
81 		rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_);  in percpu_down_read_trylock()
86 static inline void percpu_up_read_preempt_enable(struct percpu_rw_semaphore *sem)  in percpu_up_read_preempt_enable()  argument
96 	if (likely(rcu_sync_is_idle(&sem->rss)))  in percpu_up_read_preempt_enable()
97 		__this_cpu_dec(*sem->read_count);  in percpu_up_read_preempt_enable()
99 		__percpu_up_read(sem); /* Unconditional memory barrier */  in percpu_up_read_preempt_enable()
102 	rwsem_release(&sem->rw_sem.dep_map, 1, _RET_IP_);  in percpu_up_read_preempt_enable()
105 static inline void percpu_up_read(struct percpu_rw_semaphore *sem)  in percpu_up_read()  argument
108 	percpu_up_read_preempt_enable(sem);  in percpu_up_read()
119 #define percpu_init_rwsem(sem)					\  argument
122 	__percpu_init_rwsem(sem, #sem, &rwsem_key);		\
125 #define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem)  argument
127 #define percpu_rwsem_assert_held(sem)				\  argument
128 	lockdep_assert_held(&(sem)->rw_sem)
130 static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,  in percpu_rwsem_release()  argument
133 	lock_release(&sem->rw_sem.dep_map, 1, ip);  in percpu_rwsem_release()
136 		sem->rw_sem.owner = RWSEM_OWNER_UNKNOWN;  in percpu_rwsem_release()
140 static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,  in percpu_rwsem_acquire()  argument
143 	lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);  in percpu_rwsem_acquire()
146 		sem->rw_sem.owner = current;  in percpu_rwsem_acquire()