202 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt()  argument
204 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt()
207 		if (task == xprt->snd_task)  in xprt_reserve_xprt()
213 	xprt->snd_task = task;  in xprt_reserve_xprt()
221 			task->tk_pid, xprt);  in xprt_reserve_xprt()
222 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt()
223 	if  (RPC_IS_SOFT(task))  in xprt_reserve_xprt()
224 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,  in xprt_reserve_xprt()
227 		rpc_sleep_on(&xprt->sending, task, NULL);  in xprt_reserve_xprt()
266 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt_cong()  argument
268 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt_cong()
271 		if (task == xprt->snd_task)  in xprt_reserve_xprt_cong()
276 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
282 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
288 	dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);  in xprt_reserve_xprt_cong()
289 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt_cong()
290 	if (RPC_IS_SOFT(task))  in xprt_reserve_xprt_cong()
291 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,  in xprt_reserve_xprt_cong()
294 		rpc_sleep_on(&xprt->sending, task, NULL);  in xprt_reserve_xprt_cong()
299 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_lock_write()  argument
303 	if (test_bit(XPRT_LOCKED, &xprt->state) && xprt->snd_task == task)  in xprt_lock_write()
306 	retval = xprt->ops->reserve_xprt(xprt, task);  in xprt_lock_write()
311 static bool __xprt_lock_write_func(struct rpc_task *task, void *data)  in __xprt_lock_write_func()  argument
315 	xprt->snd_task = task;  in __xprt_lock_write_func()
354 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt()  argument
356 	if (xprt->snd_task == task) {  in xprt_release_xprt()
371 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt_cong()  argument
373 	if (xprt->snd_task == task) {  in xprt_release_xprt_cong()
380 static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_write()  argument
382 	if (xprt->snd_task != task)  in xprt_release_write()
385 	xprt->ops->release_xprt(xprt, task);  in xprt_release_write()
451 void xprt_release_rqst_cong(struct rpc_task *task)  in xprt_release_rqst_cong()  argument
453 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_release_rqst_cong()
495 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)  in xprt_adjust_cwnd()  argument
497 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_adjust_cwnd()
606 static void xprt_init_majortimeo(struct rpc_task *task, struct rpc_rqst *req)  in xprt_init_majortimeo()  argument
614 		time_init = xprt_abs_ktime_to_jiffies(task->tk_start);  in xprt_init_majortimeo()
615 	req->rq_timeout = task->tk_client->cl_timeout->to_initval;  in xprt_init_majortimeo()
713 xprt_request_retransmit_after_disconnect(struct rpc_task *task)  in xprt_request_retransmit_after_disconnect()  argument
715 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_retransmit_after_disconnect()
780 		struct rpc_task *task,  in xprt_lock_connect()  argument
788 	if (xprt->snd_task != task)  in xprt_lock_connect()
817 void xprt_connect(struct rpc_task *task)  in xprt_connect()  argument
819 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_connect()
821 	dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,  in xprt_connect()
825 		task->tk_status = -EAGAIN;  in xprt_connect()
828 	if (!xprt_lock_write(xprt, task))  in xprt_connect()
835 		task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;  in xprt_connect()
836 		rpc_sleep_on_timeout(&xprt->pending, task, NULL,  in xprt_connect()
837 				xprt_request_timeout(task->tk_rqstp));  in xprt_connect()
846 			xprt->ops->connect(xprt, task);  in xprt_connect()
849 			task->tk_status = 0;  in xprt_connect()
850 			rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_connect()
853 	xprt_release_write(xprt, task);  in xprt_connect()
1025 xprt_request_data_received(struct rpc_task *task)  in xprt_request_data_received()  argument
1027 	return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&  in xprt_request_data_received()
1028 		READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) != 0;  in xprt_request_data_received()
1032 xprt_request_need_enqueue_receive(struct rpc_task *task, struct rpc_rqst *req)  in xprt_request_need_enqueue_receive()  argument
1034 	return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&  in xprt_request_need_enqueue_receive()
1035 		READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) == 0;  in xprt_request_need_enqueue_receive()
1044 xprt_request_enqueue_receive(struct rpc_task *task)  in xprt_request_enqueue_receive()  argument
1046 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_enqueue_receive()
1049 	if (!xprt_request_need_enqueue_receive(task, req))  in xprt_request_enqueue_receive()
1052 	xprt_request_prepare(task->tk_rqstp);  in xprt_request_enqueue_receive()
1061 	set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate);  in xprt_request_enqueue_receive()
1075 xprt_request_dequeue_receive_locked(struct rpc_task *task)  in xprt_request_dequeue_receive_locked()  argument
1077 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_receive_locked()
1079 	if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))  in xprt_request_dequeue_receive_locked()
1089 void xprt_update_rtt(struct rpc_task *task)  in xprt_update_rtt()  argument
1091 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_update_rtt()
1092 	struct rpc_rtt *rtt = task->tk_client->cl_rtt;  in xprt_update_rtt()
1093 	unsigned int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_update_rtt()
1111 void xprt_complete_rqst(struct rpc_task *task, int copied)  in xprt_complete_rqst()  argument
1113 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_complete_rqst()
1117 			task->tk_pid, ntohl(req->rq_xid), copied);  in xprt_complete_rqst()
1127 	xprt_request_dequeue_receive_locked(task);  in xprt_complete_rqst()
1128 	rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_complete_rqst()
1132 static void xprt_timer(struct rpc_task *task)  in xprt_timer()  argument
1134 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_timer()
1137 	if (task->tk_status != -ETIMEDOUT)  in xprt_timer()
1140 	trace_xprt_timer(xprt, req->rq_xid, task->tk_status);  in xprt_timer()
1143 			xprt->ops->timer(xprt, task);  in xprt_timer()
1145 		task->tk_status = 0;  in xprt_timer()
1157 void xprt_wait_for_reply_request_def(struct rpc_task *task)  in xprt_wait_for_reply_request_def()  argument
1159 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_reply_request_def()
1161 	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,  in xprt_wait_for_reply_request_def()
1173 void xprt_wait_for_reply_request_rtt(struct rpc_task *task)  in xprt_wait_for_reply_request_rtt()  argument
1175 	int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_wait_for_reply_request_rtt()
1176 	struct rpc_clnt *clnt = task->tk_client;  in xprt_wait_for_reply_request_rtt()
1178 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_reply_request_rtt()
1186 	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,  in xprt_wait_for_reply_request_rtt()
1196 void xprt_request_wait_receive(struct rpc_task *task)  in xprt_request_wait_receive()  argument
1198 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_wait_receive()
1201 	if (!test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))  in xprt_request_wait_receive()
1209 	if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) {  in xprt_request_wait_receive()
1210 		xprt->ops->wait_for_reply_request(task);  in xprt_request_wait_receive()
1216 		if (xprt_request_retransmit_after_disconnect(task))  in xprt_request_wait_receive()
1218 					task, -ENOTCONN);  in xprt_request_wait_receive()
1224 xprt_request_need_enqueue_transmit(struct rpc_task *task, struct rpc_rqst *req)  in xprt_request_need_enqueue_transmit()  argument
1226 	return !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);  in xprt_request_need_enqueue_transmit()
1236 xprt_request_enqueue_transmit(struct rpc_task *task)  in xprt_request_enqueue_transmit()  argument
1238 	struct rpc_rqst *pos, *req = task->tk_rqstp;  in xprt_request_enqueue_transmit()
1241 	if (xprt_request_need_enqueue_transmit(task, req)) {  in xprt_request_enqueue_transmit()
1256 				trace_xprt_enq_xmit(task, 1);  in xprt_request_enqueue_transmit()
1259 		} else if (RPC_IS_SWAPPER(task)) {  in xprt_request_enqueue_transmit()
1268 				trace_xprt_enq_xmit(task, 2);  in xprt_request_enqueue_transmit()
1273 				if (pos->rq_task->tk_owner != task->tk_owner)  in xprt_request_enqueue_transmit()
1277 				trace_xprt_enq_xmit(task, 3);  in xprt_request_enqueue_transmit()
1283 		trace_xprt_enq_xmit(task, 4);  in xprt_request_enqueue_transmit()
1285 		set_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);  in xprt_request_enqueue_transmit()
1298 xprt_request_dequeue_transmit_locked(struct rpc_task *task)  in xprt_request_dequeue_transmit_locked()  argument
1300 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_transmit_locked()
1302 	if (!test_and_clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_request_dequeue_transmit_locked()
1323 xprt_request_dequeue_transmit(struct rpc_task *task)  in xprt_request_dequeue_transmit()  argument
1325 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_transmit()
1329 	xprt_request_dequeue_transmit_locked(task);  in xprt_request_dequeue_transmit()
1341 xprt_request_dequeue_xprt(struct rpc_task *task)  in xprt_request_dequeue_xprt()  argument
1343 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_dequeue_xprt()
1346 	if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate) ||  in xprt_request_dequeue_xprt()
1347 	    test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) ||  in xprt_request_dequeue_xprt()
1350 		xprt_request_dequeue_transmit_locked(task);  in xprt_request_dequeue_xprt()
1351 		xprt_request_dequeue_receive_locked(task);  in xprt_request_dequeue_xprt()
1353 			set_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);  in xprt_request_dequeue_xprt()
1357 			clear_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);  in xprt_request_dequeue_xprt()
1386 xprt_request_need_retransmit(struct rpc_task *task)  in xprt_request_need_retransmit()  argument
1388 	return xprt_request_retransmit_after_disconnect(task);  in xprt_request_need_retransmit()
1396 bool xprt_prepare_transmit(struct rpc_task *task)  in xprt_prepare_transmit()  argument
1398 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_prepare_transmit()
1401 	dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);  in xprt_prepare_transmit()
1403 	if (!xprt_lock_write(xprt, task)) {  in xprt_prepare_transmit()
1405 		if (!test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_prepare_transmit()
1407 					task, 0);  in xprt_prepare_transmit()
1414 void xprt_end_transmit(struct rpc_task *task)  in xprt_end_transmit()  argument
1416 	xprt_release_write(task->tk_rqstp->rq_xprt, task);  in xprt_end_transmit()
1433 	struct rpc_task *task = req->rq_task;  in xprt_request_transmit()  local
1435 	int is_retrans = RPC_WAS_SENT(task);  in xprt_request_transmit()
1439 		if (xprt_request_data_received(task)) {  in xprt_request_transmit()
1444 		if (rpcauth_xmit_need_reencode(task)) {  in xprt_request_transmit()
1448 		if (RPC_SIGNALLED(task)) {  in xprt_request_transmit()
1470 		task->tk_client->cl_stats->rpcretrans++;  in xprt_request_transmit()
1474 	task->tk_flags |= RPC_TASK_SENT;  in xprt_request_transmit()
1487 	xprt_request_dequeue_transmit(task);  in xprt_request_transmit()
1488 	rpc_wake_up_queued_task_set_status(&xprt->sending, task, status);  in xprt_request_transmit()
1502 xprt_transmit(struct rpc_task *task)  in xprt_transmit()  argument
1504 	struct rpc_rqst *next, *req = task->tk_rqstp;  in xprt_transmit()
1514 		status = xprt_request_transmit(next, task);  in xprt_transmit()
1521 			if (!xprt_request_data_received(task) ||  in xprt_transmit()
1522 			    test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_transmit()
1524 		} else if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_transmit()
1525 			task->tk_status = status;  in xprt_transmit()
1531 static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_add_backlog()  argument
1534 	rpc_sleep_on(&xprt->backlog, task, NULL);  in xprt_add_backlog()
1543 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_throttle_congested()  argument
1551 		rpc_sleep_on(&xprt->backlog, task, NULL);  in xprt_throttle_congested()
1587 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_alloc_slot()  argument
1604 		task->tk_status = -ENOMEM;  in xprt_alloc_slot()
1607 		xprt_add_backlog(xprt, task);  in xprt_alloc_slot()
1611 		task->tk_status = -EAGAIN;  in xprt_alloc_slot()
1620 	task->tk_status = 0;  in xprt_alloc_slot()
1621 	task->tk_rqstp = req;  in xprt_alloc_slot()
1715 xprt_request_init(struct rpc_task *task)  in xprt_request_init()  argument
1717 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_request_init()
1718 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_init()
1720 	req->rq_task	= task;  in xprt_request_init()
1732 	xprt_init_majortimeo(task, req);  in xprt_request_init()
1733 	dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,  in xprt_request_init()
1738 xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_do_reserve()  argument
1740 	xprt->ops->alloc_slot(xprt, task);  in xprt_do_reserve()
1741 	if (task->tk_rqstp != NULL)  in xprt_do_reserve()
1742 		xprt_request_init(task);  in xprt_do_reserve()
1753 void xprt_reserve(struct rpc_task *task)  in xprt_reserve()  argument
1755 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_reserve()
1757 	task->tk_status = 0;  in xprt_reserve()
1758 	if (task->tk_rqstp != NULL)  in xprt_reserve()
1761 	task->tk_status = -EAGAIN;  in xprt_reserve()
1762 	if (!xprt_throttle_congested(xprt, task))  in xprt_reserve()
1763 		xprt_do_reserve(xprt, task);  in xprt_reserve()
1775 void xprt_retry_reserve(struct rpc_task *task)  in xprt_retry_reserve()  argument
1777 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_retry_reserve()
1779 	task->tk_status = 0;  in xprt_retry_reserve()
1780 	if (task->tk_rqstp != NULL)  in xprt_retry_reserve()
1783 	task->tk_status = -EAGAIN;  in xprt_retry_reserve()
1784 	xprt_do_reserve(xprt, task);  in xprt_retry_reserve()
1792 void xprt_release(struct rpc_task *task)  in xprt_release()  argument
1795 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_release()
1798 		if (task->tk_client) {  in xprt_release()
1799 			xprt = task->tk_xprt;  in xprt_release()
1800 			xprt_release_write(xprt, task);  in xprt_release()
1806 	xprt_request_dequeue_xprt(task);  in xprt_release()
1808 	xprt->ops->release_xprt(xprt, task);  in xprt_release()
1810 		xprt->ops->release_request(task);  in xprt_release()
1814 		xprt->ops->buf_free(task);  in xprt_release()
1820 	task->tk_rqstp = NULL;  in xprt_release()
1824 	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);  in xprt_release()
1833 xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task)  in xprt_init_bc_request()  argument
1837 	task->tk_rqstp = req;  in xprt_init_bc_request()
1838 	req->rq_task = task;  in xprt_init_bc_request()