404 	struct rwsem_waiter *waiter, *tmp;  in rwsem_mark_wake()  local
414 	waiter = rwsem_first_waiter(sem);  in rwsem_mark_wake()
416 	if (waiter->type == RWSEM_WAITING_FOR_WRITE) {  in rwsem_mark_wake()
425 			wake_q_add(wake_q, waiter->task);  in rwsem_mark_wake()
455 			    time_after(jiffies, waiter->timeout)) {  in rwsem_mark_wake()
469 		owner = waiter->task;  in rwsem_mark_wake()
470 		if (waiter->last_rowner & RWSEM_RD_NONSPINNABLE) {  in rwsem_mark_wake()
501 	list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {  in rwsem_mark_wake()
502 		if (waiter->type == RWSEM_WAITING_FOR_WRITE)  in rwsem_mark_wake()
506 		list_move_tail(&waiter->list, &wlist);  in rwsem_mark_wake()
533 	list_for_each_entry_safe(waiter, tmp, &wlist, list) {  in rwsem_mark_wake()
536 		tsk = waiter->task;  in rwsem_mark_wake()
545 		smp_store_release(&waiter->task, NULL);  in rwsem_mark_wake()
995 	struct rwsem_waiter waiter;  in rwsem_down_read_slowpath()  local
1003 	waiter.last_rowner = atomic_long_read(&sem->owner);  in rwsem_down_read_slowpath()
1004 	if (!(waiter.last_rowner & RWSEM_READER_OWNED))  in rwsem_down_read_slowpath()
1005 		waiter.last_rowner &= RWSEM_RD_NONSPINNABLE;  in rwsem_down_read_slowpath()
1030 	} else if (rwsem_reader_phase_trylock(sem, waiter.last_rowner)) {  in rwsem_down_read_slowpath()
1036 	waiter.task = current;  in rwsem_down_read_slowpath()
1037 	waiter.type = RWSEM_WAITING_FOR_READ;  in rwsem_down_read_slowpath()
1038 	waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;  in rwsem_down_read_slowpath()
1059 	list_add_tail(&waiter.list, &sem->wait_list);  in rwsem_down_read_slowpath()
1087 		if (!smp_load_acquire(&waiter.task)) {  in rwsem_down_read_slowpath()
1093 			if (waiter.task)  in rwsem_down_read_slowpath()
1108 	list_del(&waiter.list);  in rwsem_down_read_slowpath()
1141 	struct rwsem_waiter waiter;  in rwsem_down_write_slowpath()  local
1163 	waiter.task = current;  in rwsem_down_write_slowpath()
1164 	waiter.type = RWSEM_WAITING_FOR_WRITE;  in rwsem_down_write_slowpath()
1165 	waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;  in rwsem_down_write_slowpath()
1172 	list_add_tail(&waiter.list, &sem->wait_list);  in rwsem_down_write_slowpath()
1246 			    (rwsem_first_waiter(sem) == &waiter))  in rwsem_down_write_slowpath()
1258 			    time_after(jiffies, waiter.timeout))) {  in rwsem_down_write_slowpath()
1268 	list_del(&waiter.list);  in rwsem_down_write_slowpath()
1278 	list_del(&waiter.list);  in rwsem_down_write_slowpath()