10 int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,  in __percpu_init_rwsem()  argument
13 	sem->read_count = alloc_percpu(int);  in __percpu_init_rwsem()
14 	if (unlikely(!sem->read_count))  in __percpu_init_rwsem()
18 	rcu_sync_init(&sem->rss, RCU_SCHED_SYNC);  in __percpu_init_rwsem()
19 	__init_rwsem(&sem->rw_sem, name, rwsem_key);  in __percpu_init_rwsem()
20 	rcuwait_init(&sem->writer);  in __percpu_init_rwsem()
21 	sem->readers_block = 0;  in __percpu_init_rwsem()
26 void percpu_free_rwsem(struct percpu_rw_semaphore *sem)  in percpu_free_rwsem()  argument
32 	if (!sem->read_count)  in percpu_free_rwsem()
35 	rcu_sync_dtor(&sem->rss);  in percpu_free_rwsem()
36 	free_percpu(sem->read_count);  in percpu_free_rwsem()
37 	sem->read_count = NULL; /* catch use after free bugs */  in percpu_free_rwsem()
41 int __percpu_down_read(struct percpu_rw_semaphore *sem, int try)  in __percpu_down_read()  argument
64 	if (likely(!smp_load_acquire(&sem->readers_block)))  in __percpu_down_read()
71 	__percpu_up_read(sem);  in __percpu_down_read()
85 	__down_read(&sem->rw_sem);  in __percpu_down_read()
86 	this_cpu_inc(*sem->read_count);  in __percpu_down_read()
87 	__up_read(&sem->rw_sem);  in __percpu_down_read()
94 void __percpu_up_read(struct percpu_rw_semaphore *sem)  in __percpu_up_read()  argument
102 	__this_cpu_dec(*sem->read_count);  in __percpu_up_read()
105 	rcuwait_wake_up(&sem->writer);  in __percpu_up_read()
125 static bool readers_active_check(struct percpu_rw_semaphore *sem)  in readers_active_check()  argument
127 	if (per_cpu_sum(*sem->read_count) != 0)  in readers_active_check()
140 void percpu_down_write(struct percpu_rw_semaphore *sem)  in percpu_down_write()  argument
143 	rcu_sync_enter(&sem->rss);  in percpu_down_write()
145 	down_write(&sem->rw_sem);  in percpu_down_write()
151 	WRITE_ONCE(sem->readers_block, 1);  in percpu_down_write()
162 	rcuwait_wait_event(&sem->writer, readers_active_check(sem));  in percpu_down_write()
166 void percpu_up_write(struct percpu_rw_semaphore *sem)  in percpu_up_write()  argument
178 	smp_store_release(&sem->readers_block, 0);  in percpu_up_write()
183 	up_write(&sem->rw_sem);  in percpu_up_write()
190 	rcu_sync_exit(&sem->rss);  in percpu_up_write()