30 	struct smc_connection *conn = cdcpend->conn;  in smc_cdc_tx_handler()  local
34 	smc = container_of(conn, struct smc_sock, conn);  in smc_cdc_tx_handler()
37 		diff = smc_curs_diff(cdcpend->conn->sndbuf_desc->len,  in smc_cdc_tx_handler()
38 				     &cdcpend->conn->tx_curs_fin,  in smc_cdc_tx_handler()
42 		atomic_add(diff, &cdcpend->conn->sndbuf_space);  in smc_cdc_tx_handler()
45 		smc_curs_copy(&conn->tx_curs_fin, &cdcpend->cursor, conn);  in smc_cdc_tx_handler()
46 		smc_curs_copy(&conn->local_tx_ctrl_fin, &cdcpend->p_cursor,  in smc_cdc_tx_handler()
47 			      conn);  in smc_cdc_tx_handler()
48 		conn->tx_cdc_seq_fin = cdcpend->ctrl_seq;  in smc_cdc_tx_handler()
51 	if (atomic_dec_and_test(&conn->cdc_pend_tx_wr)) {  in smc_cdc_tx_handler()
57 			conn->tx_in_release_sock = true;  in smc_cdc_tx_handler()
59 			smc_tx_pending(conn);  in smc_cdc_tx_handler()
61 		if (unlikely(wq_has_sleeper(&conn->cdc_pend_tx_wq)))  in smc_cdc_tx_handler()
62 			wake_up(&conn->cdc_pend_tx_wq);  in smc_cdc_tx_handler()
64 	WARN_ON(atomic_read(&conn->cdc_pend_tx_wr) < 0);  in smc_cdc_tx_handler()
70 int smc_cdc_get_free_slot(struct smc_connection *conn,  in smc_cdc_get_free_slot()  argument
81 	if (conn->killed) {  in smc_cdc_get_free_slot()
91 static inline void smc_cdc_add_pending_send(struct smc_connection *conn,  in smc_cdc_add_pending_send()  argument
103 	pend->conn = conn;  in smc_cdc_add_pending_send()
104 	pend->cursor = conn->tx_curs_sent;  in smc_cdc_add_pending_send()
105 	pend->p_cursor = conn->local_tx_ctrl.prod;  in smc_cdc_add_pending_send()
106 	pend->ctrl_seq = conn->tx_cdc_seq;  in smc_cdc_add_pending_send()
109 int smc_cdc_msg_send(struct smc_connection *conn,  in smc_cdc_msg_send()  argument
113 	struct smc_link *link = conn->lnk;  in smc_cdc_msg_send()
117 	smc_cdc_add_pending_send(conn, pend);  in smc_cdc_msg_send()
119 	conn->tx_cdc_seq++;  in smc_cdc_msg_send()
120 	conn->local_tx_ctrl.seqno = conn->tx_cdc_seq;  in smc_cdc_msg_send()
121 	smc_host_msg_to_cdc((struct smc_cdc_msg *)wr_buf, conn, &cfed);  in smc_cdc_msg_send()
123 	atomic_inc(&conn->cdc_pend_tx_wr);  in smc_cdc_msg_send()
128 		smc_curs_copy(&conn->rx_curs_confirmed, &cfed, conn);  in smc_cdc_msg_send()
129 		conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;  in smc_cdc_msg_send()
131 		conn->tx_cdc_seq--;  in smc_cdc_msg_send()
132 		conn->local_tx_ctrl.seqno = conn->tx_cdc_seq;  in smc_cdc_msg_send()
133 		atomic_dec(&conn->cdc_pend_tx_wr);  in smc_cdc_msg_send()
139 /* send a validation msg indicating the move of a conn to an other QP link */
140 int smcr_cdc_msg_send_validation(struct smc_connection *conn,  in smcr_cdc_msg_send_validation()  argument
144 	struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;  in smcr_cdc_msg_send_validation()
145 	struct smc_link *link = conn->lnk;  in smcr_cdc_msg_send_validation()
152 	peer->seqno = htons(conn->tx_cdc_seq_fin); /* seqno last compl. tx */  in smcr_cdc_msg_send_validation()
156 	/* We need to set pend->conn here to make sure smc_cdc_tx_handler()  in smcr_cdc_msg_send_validation()
159 	smc_cdc_add_pending_send(conn, pend);  in smcr_cdc_msg_send_validation()
161 	atomic_inc(&conn->cdc_pend_tx_wr);  in smcr_cdc_msg_send_validation()
166 		atomic_dec(&conn->cdc_pend_tx_wr);  in smcr_cdc_msg_send_validation()
171 static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn)  in smcr_cdc_get_slot_and_msg_send()  argument
180 	link = conn->lnk;  in smcr_cdc_get_slot_and_msg_send()
183 	rc = smc_cdc_get_free_slot(conn, link, &wr_buf, NULL, &pend);  in smcr_cdc_get_slot_and_msg_send()
187 	spin_lock_bh(&conn->send_lock);  in smcr_cdc_get_slot_and_msg_send()
188 	if (link != conn->lnk) {  in smcr_cdc_get_slot_and_msg_send()
190 		spin_unlock_bh(&conn->send_lock);  in smcr_cdc_get_slot_and_msg_send()
199 	rc = smc_cdc_msg_send(conn, wr_buf, pend);  in smcr_cdc_get_slot_and_msg_send()
200 	spin_unlock_bh(&conn->send_lock);  in smcr_cdc_get_slot_and_msg_send()
206 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn)  in smc_cdc_get_slot_and_msg_send()  argument
210 	if (!smc_conn_lgr_valid(conn) ||  in smc_cdc_get_slot_and_msg_send()
211 	    (conn->lgr->is_smcd && conn->lgr->peer_shutdown))  in smc_cdc_get_slot_and_msg_send()
214 	if (conn->lgr->is_smcd) {  in smc_cdc_get_slot_and_msg_send()
215 		spin_lock_bh(&conn->send_lock);  in smc_cdc_get_slot_and_msg_send()
216 		rc = smcd_cdc_msg_send(conn);  in smc_cdc_get_slot_and_msg_send()
217 		spin_unlock_bh(&conn->send_lock);  in smc_cdc_get_slot_and_msg_send()
219 		rc = smcr_cdc_get_slot_and_msg_send(conn);  in smc_cdc_get_slot_and_msg_send()
225 void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn)  in smc_cdc_wait_pend_tx_wr()  argument
227 	wait_event(conn->cdc_pend_tx_wq, !atomic_read(&conn->cdc_pend_tx_wr));  in smc_cdc_wait_pend_tx_wr()
234 int smcd_cdc_msg_send(struct smc_connection *conn)  in smcd_cdc_msg_send()  argument
236 	struct smc_sock *smc = container_of(conn, struct smc_sock, conn);  in smcd_cdc_msg_send()
243 	curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.prod.acurs);  in smcd_cdc_msg_send()
246 	curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.cons.acurs);  in smcd_cdc_msg_send()
249 	cdc.cons.prod_flags = conn->local_tx_ctrl.prod_flags;  in smcd_cdc_msg_send()
250 	cdc.cons.conn_state_flags = conn->local_tx_ctrl.conn_state_flags;  in smcd_cdc_msg_send()
251 	rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1);  in smcd_cdc_msg_send()
254 	smc_curs_copy(&conn->rx_curs_confirmed, &curs, conn);  in smcd_cdc_msg_send()
255 	conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;  in smcd_cdc_msg_send()
257 	diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,  in smcd_cdc_msg_send()
258 			     &conn->tx_curs_sent);  in smcd_cdc_msg_send()
261 	atomic_add(diff, &conn->sndbuf_space);  in smcd_cdc_msg_send()
264 	smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn);  in smcd_cdc_msg_send()
280 	struct smc_connection *conn = &smc->conn;  in smc_cdc_handle_urg_data_arrival()  local
284 	smc_curs_copy(&conn->urg_curs, &conn->local_rx_ctrl.prod, conn);  in smc_cdc_handle_urg_data_arrival()
285 	conn->urg_state = SMC_URG_VALID;  in smc_cdc_handle_urg_data_arrival()
289 	base = (char *)conn->rmb_desc->cpu_addr + conn->rx_off;  in smc_cdc_handle_urg_data_arrival()
290 	if (conn->urg_curs.count)  in smc_cdc_handle_urg_data_arrival()
291 		conn->urg_rx_byte = *(base + conn->urg_curs.count - 1);  in smc_cdc_handle_urg_data_arrival()
293 		conn->urg_rx_byte = *(base + conn->rmb_desc->len - 1);  in smc_cdc_handle_urg_data_arrival()
300 	struct smc_connection *conn = &smc->conn;  in smc_cdc_msg_validate()  local
305 	diff = conn->local_rx_ctrl.seqno - recv_seq;  in smc_cdc_msg_validate()
308 		conn->out_of_sync = 1;	/* prevent any further receives */  in smc_cdc_msg_validate()
309 		spin_lock_bh(&conn->send_lock);  in smc_cdc_msg_validate()
310 		conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;  in smc_cdc_msg_validate()
311 		conn->lnk = link;  in smc_cdc_msg_validate()
312 		spin_unlock_bh(&conn->send_lock);  in smc_cdc_msg_validate()
314 		if (!queue_work(smc_close_wq, &conn->abort_work))  in smc_cdc_msg_validate()
323 	struct smc_connection *conn = &smc->conn;  in smc_cdc_msg_recv_action()  local
326 	smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn);  in smc_cdc_msg_recv_action()
327 	smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn);  in smc_cdc_msg_recv_action()
328 	smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn);  in smc_cdc_msg_recv_action()
330 	diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old,  in smc_cdc_msg_recv_action()
331 				  &conn->local_rx_ctrl.cons);  in smc_cdc_msg_recv_action()
337 		atomic_add(diff_cons, &conn->peer_rmbe_space);  in smc_cdc_msg_recv_action()
342 	diff_prod = smc_curs_diff(conn->rmb_desc->len, &prod_old,  in smc_cdc_msg_recv_action()
343 				  &conn->local_rx_ctrl.prod);  in smc_cdc_msg_recv_action()
345 		if (conn->local_rx_ctrl.prod_flags.urg_data_present)  in smc_cdc_msg_recv_action()
349 		atomic_add(diff_prod, &conn->bytes_to_rcv);  in smc_cdc_msg_recv_action()
354 		if (conn->local_rx_ctrl.prod_flags.write_blocked)  in smc_cdc_msg_recv_action()
356 		if (conn->local_rx_ctrl.prod_flags.urg_data_pending)  in smc_cdc_msg_recv_action()
357 			conn->urg_state = SMC_URG_NOTYET;  in smc_cdc_msg_recv_action()
361 	if ((diff_cons && smc_tx_prepared_sends(conn)) ||  in smc_cdc_msg_recv_action()
362 	    conn->local_rx_ctrl.prod_flags.cons_curs_upd_req ||  in smc_cdc_msg_recv_action()
363 	    conn->local_rx_ctrl.prod_flags.urg_data_pending) {  in smc_cdc_msg_recv_action()
365 			smc_tx_pending(conn);  in smc_cdc_msg_recv_action()
367 			conn->tx_in_release_sock = true;  in smc_cdc_msg_recv_action()
370 	if (diff_cons && conn->urg_tx_pend &&  in smc_cdc_msg_recv_action()
371 	    atomic_read(&conn->peer_rmbe_space) == conn->peer_rmbe_size) {  in smc_cdc_msg_recv_action()
373 		conn->urg_tx_pend = false;  in smc_cdc_msg_recv_action()
377 	if (conn->local_rx_ctrl.conn_state_flags.peer_conn_abort) {  in smc_cdc_msg_recv_action()
379 		conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;  in smc_cdc_msg_recv_action()
381 	if (smc_cdc_rxed_any_close_or_senddone(conn)) {  in smc_cdc_msg_recv_action()
387 		if (!queue_work(smc_close_wq, &conn->close_work))  in smc_cdc_msg_recv_action()
410 	struct smc_connection *conn = from_tasklet(conn, t, rx_tsklet);  in smcd_cdc_rx_tsklet()  local
415 	if (!conn || conn->killed)  in smcd_cdc_rx_tsklet()
418 	data_cdc = (struct smcd_cdc_msg *)conn->rmb_desc->cpu_addr;  in smcd_cdc_rx_tsklet()
419 	smcd_curs_copy(&cdc.prod, &data_cdc->prod, conn);  in smcd_cdc_rx_tsklet()
420 	smcd_curs_copy(&cdc.cons, &data_cdc->cons, conn);  in smcd_cdc_rx_tsklet()
421 	smc = container_of(conn, struct smc_sock, conn);  in smcd_cdc_rx_tsklet()
428 void smcd_cdc_rx_init(struct smc_connection *conn)  in smcd_cdc_rx_init()  argument
430 	tasklet_setup(&conn->rx_tsklet, smcd_cdc_rx_tsklet);  in smcd_cdc_rx_init()
439 	struct smc_connection *conn;  in smc_cdc_rx_handler()  local
451 	conn = smc_lgr_find_conn(ntohl(cdc->token), lgr);  in smc_cdc_rx_handler()
453 	if (!conn || conn->out_of_sync)  in smc_cdc_rx_handler()
455 	smc = container_of(conn, struct smc_sock, conn);  in smc_cdc_rx_handler()
462 			   conn->local_rx_ctrl.seqno))  in smc_cdc_rx_handler()