373 rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)  in rwsem_add_waiter()  argument
376 	list_add_tail(&waiter->list, &sem->wait_list);  in rwsem_add_waiter()
389 rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)  in rwsem_del_waiter()  argument
392 	list_del(&waiter->list);  in rwsem_del_waiter()
418 	struct rwsem_waiter *waiter, *tmp;  in rwsem_mark_wake()  local
428 	waiter = rwsem_first_waiter(sem);  in rwsem_mark_wake()
430 	if (waiter->type == RWSEM_WAITING_FOR_WRITE) {  in rwsem_mark_wake()
439 			wake_q_add(wake_q, waiter->task);  in rwsem_mark_wake()
468 			if (time_after(jiffies, waiter->timeout)) {  in rwsem_mark_wake()
473 				waiter->handoff_set = true;  in rwsem_mark_wake()
485 		owner = waiter->task;  in rwsem_mark_wake()
513 	list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {  in rwsem_mark_wake()
514 		if (waiter->type == RWSEM_WAITING_FOR_WRITE)  in rwsem_mark_wake()
518 		list_move_tail(&waiter->list, &wlist);  in rwsem_mark_wake()
552 	list_for_each_entry_safe(waiter, tmp, &wlist, list) {  in rwsem_mark_wake()
555 		tsk = waiter->task;  in rwsem_mark_wake()
564 		smp_store_release(&waiter->task, NULL);  in rwsem_mark_wake()
580 rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,  in rwsem_del_wake_waiter()  argument
584 	bool first = rwsem_first_waiter(sem) == waiter;  in rwsem_del_wake_waiter()
593 	if (rwsem_del_waiter(sem, waiter) && first)  in rwsem_del_wake_waiter()
608 					struct rwsem_waiter *waiter)  in rwsem_try_write_lock()  argument
625 			if (first->handoff_set && (waiter != first))  in rwsem_try_write_lock()
632 			if (waiter == first)  in rwsem_try_write_lock()
633 				waiter->handoff_set = true;  in rwsem_try_write_lock()
639 			if (has_handoff || (!rt_task(waiter->task) &&  in rwsem_try_write_lock()
640 					    !time_after(jiffies, waiter->timeout)))  in rwsem_try_write_lock()
658 		waiter->handoff_set = true;  in rwsem_try_write_lock()
667 	list_del(&waiter->list);  in rwsem_try_write_lock()
1009 	struct rwsem_waiter waiter;  in rwsem_down_read_slowpath()  local
1044 	waiter.task = current;  in rwsem_down_read_slowpath()
1045 	waiter.type = RWSEM_WAITING_FOR_READ;  in rwsem_down_read_slowpath()
1046 	waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;  in rwsem_down_read_slowpath()
1047 	waiter.handoff_set = false;  in rwsem_down_read_slowpath()
1067 	rwsem_add_waiter(sem, &waiter);  in rwsem_down_read_slowpath()
1083 		if (!smp_load_acquire(&waiter.task)) {  in rwsem_down_read_slowpath()
1089 			if (waiter.task)  in rwsem_down_read_slowpath()
1105 	rwsem_del_wake_waiter(sem, &waiter, &wake_q);  in rwsem_down_read_slowpath()
1118 	struct rwsem_waiter waiter;  in rwsem_down_write_slowpath()  local
1131 	waiter.task = current;  in rwsem_down_write_slowpath()
1132 	waiter.type = RWSEM_WAITING_FOR_WRITE;  in rwsem_down_write_slowpath()
1133 	waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;  in rwsem_down_write_slowpath()
1134 	waiter.handoff_set = false;  in rwsem_down_write_slowpath()
1137 	rwsem_add_waiter(sem, &waiter);  in rwsem_down_write_slowpath()
1140 	if (rwsem_first_waiter(sem) != &waiter) {  in rwsem_down_write_slowpath()
1161 		if (rwsem_try_write_lock(sem, &waiter)) {  in rwsem_down_write_slowpath()
1179 		if (waiter.handoff_set) {  in rwsem_down_write_slowpath()
1205 	rwsem_del_wake_waiter(sem, &waiter, &wake_q);  in rwsem_down_write_slowpath()