80 void __init_ldsem(struct ld_semaphore *sem, const char *name,  in __init_ldsem()  argument
87 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));  in __init_ldsem()
88 	lockdep_init_map(&sem->dep_map, name, key, 0);  in __init_ldsem()
90 	atomic_long_set(&sem->count, LDSEM_UNLOCKED);  in __init_ldsem()
91 	sem->wait_readers = 0;  in __init_ldsem()
92 	raw_spin_lock_init(&sem->wait_lock);  in __init_ldsem()
93 	INIT_LIST_HEAD(&sem->read_wait);  in __init_ldsem()
94 	INIT_LIST_HEAD(&sem->write_wait);  in __init_ldsem()
97 static void __ldsem_wake_readers(struct ld_semaphore *sem)  in __ldsem_wake_readers()  argument
108 	adjust = sem->wait_readers * (LDSEM_ACTIVE_BIAS - LDSEM_WAIT_BIAS);  in __ldsem_wake_readers()
109 	count = atomic_long_add_return(adjust, &sem->count);  in __ldsem_wake_readers()
113 		if (atomic_long_try_cmpxchg(&sem->count, &count, count - adjust))  in __ldsem_wake_readers()
117 	list_for_each_entry_safe(waiter, next, &sem->read_wait, list) {  in __ldsem_wake_readers()
124 	INIT_LIST_HEAD(&sem->read_wait);  in __ldsem_wake_readers()
125 	sem->wait_readers = 0;  in __ldsem_wake_readers()
128 static inline int writer_trylock(struct ld_semaphore *sem)  in writer_trylock()  argument
134 	long count = atomic_long_add_return(LDSEM_ACTIVE_BIAS, &sem->count);  in writer_trylock()
138 		if (atomic_long_try_cmpxchg(&sem->count, &count, count - LDSEM_ACTIVE_BIAS))  in writer_trylock()
143 static void __ldsem_wake_writer(struct ld_semaphore *sem)  in __ldsem_wake_writer()  argument
147 	waiter = list_entry(sem->write_wait.next, struct ldsem_waiter, list);  in __ldsem_wake_writer()
159 static void __ldsem_wake(struct ld_semaphore *sem)  in __ldsem_wake()  argument
161 	if (!list_empty(&sem->write_wait))  in __ldsem_wake()
162 		__ldsem_wake_writer(sem);  in __ldsem_wake()
163 	else if (!list_empty(&sem->read_wait))  in __ldsem_wake()
164 		__ldsem_wake_readers(sem);  in __ldsem_wake()
167 static void ldsem_wake(struct ld_semaphore *sem)  in ldsem_wake()  argument
171 	raw_spin_lock_irqsave(&sem->wait_lock, flags);  in ldsem_wake()
172 	__ldsem_wake(sem);  in ldsem_wake()
173 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);  in ldsem_wake()
180 down_read_failed(struct ld_semaphore *sem, long count, long timeout)  in down_read_failed()  argument
186 	raw_spin_lock_irq(&sem->wait_lock);  in down_read_failed()
194 		if (atomic_long_try_cmpxchg(&sem->count, &count, count + adjust)) {  in down_read_failed()
199 			raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
200 			return sem;  in down_read_failed()
204 	list_add_tail(&waiter.list, &sem->read_wait);  in down_read_failed()
205 	sem->wait_readers++;  in down_read_failed()
212 		__ldsem_wake(sem);  in down_read_failed()
214 	raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
235 		raw_spin_lock_irq(&sem->wait_lock);  in down_read_failed()
237 			atomic_long_add_return(-LDSEM_WAIT_BIAS, &sem->count);  in down_read_failed()
239 			raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
243 		raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
246 	return sem;  in down_read_failed()
253 down_write_failed(struct ld_semaphore *sem, long count, long timeout)  in down_write_failed()  argument
260 	raw_spin_lock_irq(&sem->wait_lock);  in down_write_failed()
268 		if (atomic_long_try_cmpxchg(&sem->count, &count, count + adjust))  in down_write_failed()
271 			raw_spin_unlock_irq(&sem->wait_lock);  in down_write_failed()
272 			return sem;  in down_write_failed()
276 	list_add_tail(&waiter.list, &sem->write_wait);  in down_write_failed()
284 		raw_spin_unlock_irq(&sem->wait_lock);  in down_write_failed()
286 		raw_spin_lock_irq(&sem->wait_lock);  in down_write_failed()
288 		locked = writer_trylock(sem);  in down_write_failed()
294 		atomic_long_add_return(-LDSEM_WAIT_BIAS, &sem->count);  in down_write_failed()
296 	raw_spin_unlock_irq(&sem->wait_lock);  in down_write_failed()
303 	return sem;  in down_write_failed()
308 static int __ldsem_down_read_nested(struct ld_semaphore *sem,  in __ldsem_down_read_nested()  argument
313 	lockdep_acquire_read(sem, subclass, 0, _RET_IP_);  in __ldsem_down_read_nested()
315 	count = atomic_long_add_return(LDSEM_READ_BIAS, &sem->count);  in __ldsem_down_read_nested()
317 		lock_stat(sem, contended);  in __ldsem_down_read_nested()
318 		if (!down_read_failed(sem, count, timeout)) {  in __ldsem_down_read_nested()
319 			lockdep_release(sem, 1, _RET_IP_);  in __ldsem_down_read_nested()
323 	lock_stat(sem, acquired);  in __ldsem_down_read_nested()
327 static int __ldsem_down_write_nested(struct ld_semaphore *sem,  in __ldsem_down_write_nested()  argument
332 	lockdep_acquire(sem, subclass, 0, _RET_IP_);  in __ldsem_down_write_nested()
334 	count = atomic_long_add_return(LDSEM_WRITE_BIAS, &sem->count);  in __ldsem_down_write_nested()
336 		lock_stat(sem, contended);  in __ldsem_down_write_nested()
337 		if (!down_write_failed(sem, count, timeout)) {  in __ldsem_down_write_nested()
338 			lockdep_release(sem, 1, _RET_IP_);  in __ldsem_down_write_nested()
342 	lock_stat(sem, acquired);  in __ldsem_down_write_nested()
350 int __sched ldsem_down_read(struct ld_semaphore *sem, long timeout)  in ldsem_down_read()  argument
353 	return __ldsem_down_read_nested(sem, 0, timeout);  in ldsem_down_read()
359 int ldsem_down_read_trylock(struct ld_semaphore *sem)  in ldsem_down_read_trylock()  argument
361 	long count = atomic_long_read(&sem->count);  in ldsem_down_read_trylock()
364 		if (atomic_long_try_cmpxchg(&sem->count, &count, count + LDSEM_READ_BIAS)) {  in ldsem_down_read_trylock()
365 			lockdep_acquire_read(sem, 0, 1, _RET_IP_);  in ldsem_down_read_trylock()
366 			lock_stat(sem, acquired);  in ldsem_down_read_trylock()
376 int __sched ldsem_down_write(struct ld_semaphore *sem, long timeout)  in ldsem_down_write()  argument
379 	return __ldsem_down_write_nested(sem, 0, timeout);  in ldsem_down_write()
385 int ldsem_down_write_trylock(struct ld_semaphore *sem)  in ldsem_down_write_trylock()  argument
387 	long count = atomic_long_read(&sem->count);  in ldsem_down_write_trylock()
390 		if (atomic_long_try_cmpxchg(&sem->count, &count, count + LDSEM_WRITE_BIAS)) {  in ldsem_down_write_trylock()
391 			lockdep_acquire(sem, 0, 1, _RET_IP_);  in ldsem_down_write_trylock()
392 			lock_stat(sem, acquired);  in ldsem_down_write_trylock()
402 void ldsem_up_read(struct ld_semaphore *sem)  in ldsem_up_read()  argument
406 	lockdep_release(sem, 1, _RET_IP_);  in ldsem_up_read()
408 	count = atomic_long_add_return(-LDSEM_READ_BIAS, &sem->count);  in ldsem_up_read()
410 		ldsem_wake(sem);  in ldsem_up_read()
416 void ldsem_up_write(struct ld_semaphore *sem)  in ldsem_up_write()  argument
420 	lockdep_release(sem, 1, _RET_IP_);  in ldsem_up_write()
422 	count = atomic_long_add_return(-LDSEM_WRITE_BIAS, &sem->count);  in ldsem_up_write()
424 		ldsem_wake(sem);  in ldsem_up_write()
430 int ldsem_down_read_nested(struct ld_semaphore *sem, int subclass, long timeout)  in ldsem_down_read_nested()  argument
433 	return __ldsem_down_read_nested(sem, subclass, timeout);  in ldsem_down_read_nested()
436 int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass,  in ldsem_down_write_nested()  argument
440 	return __ldsem_down_write_nested(sem, subclass, timeout);  in ldsem_down_write_nested()