Lines Matching +full:smc +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
11 * - support for alternate links postponed
19 #define KMSG_COMPONENT "smc"
33 #include <net/smc.h>
40 #include "smc.h"
75 if (cb_ctx->pos[0]) in smc_nl_dump_hs_limitation()
78 hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, in smc_nl_dump_hs_limitation()
82 return -ENOMEM; in smc_nl_dump_hs_limitation()
85 sock_net(skb->sk)->smc.limit_smc_hs)) in smc_nl_dump_hs_limitation()
89 cb_ctx->pos[0] = 1; in smc_nl_dump_hs_limitation()
91 return skb->len; in smc_nl_dump_hs_limitation()
94 return -EMSGSIZE; in smc_nl_dump_hs_limitation()
99 sock_net(skb->sk)->smc.limit_smc_hs = true; in smc_nl_enable_hs_limitation()
105 sock_net(skb->sk)->smc.limit_smc_hs = false; in smc_nl_disable_hs_limitation()
111 struct smc_sock *smc = smc_sk(sk); in smc_set_keepalive() local
113 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); in smc_set_keepalive()
123 struct smc_sock *smc; in smc_tcp_syn_recv_sock() local
126 smc = smc_clcsock_user_data(sk); in smc_tcp_syn_recv_sock()
128 if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) > in smc_tcp_syn_recv_sock()
129 sk->sk_max_ack_backlog) in smc_tcp_syn_recv_sock()
132 if (sk_acceptq_is_full(&smc->sk)) { in smc_tcp_syn_recv_sock()
138 child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash, in smc_tcp_syn_recv_sock()
140 /* child must not inherit smc or its ops */ in smc_tcp_syn_recv_sock()
144 /* v4-mapped sockets don't inherit parent ops. Don't restore. */ in smc_tcp_syn_recv_sock()
145 if (inet_csk(child)->icsk_af_ops == inet_csk(sk)->icsk_af_ops) in smc_tcp_syn_recv_sock()
146 inet_csk(child)->icsk_af_ops = smc->ori_af_ops; in smc_tcp_syn_recv_sock()
158 const struct smc_sock *smc; in smc_hs_congested() local
160 smc = smc_clcsock_user_data(sk); in smc_hs_congested()
162 if (!smc) in smc_hs_congested()
181 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; in smc_hash_sk()
184 head = &h->ht; in smc_hash_sk()
186 write_lock_bh(&h->lock); in smc_hash_sk()
188 write_unlock_bh(&h->lock); in smc_hash_sk()
189 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in smc_hash_sk()
197 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; in smc_unhash_sk()
199 write_lock_bh(&h->lock); in smc_unhash_sk()
201 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in smc_unhash_sk()
202 write_unlock_bh(&h->lock); in smc_unhash_sk()
212 struct smc_sock *smc = smc_sk(sk); in smc_release_cb() local
214 if (smc->conn.tx_in_release_sock) { in smc_release_cb()
215 smc_tx_pending(&smc->conn); in smc_release_cb()
216 smc->conn.tx_in_release_sock = false; in smc_release_cb()
221 .name = "SMC",
246 static void smc_fback_restore_callbacks(struct smc_sock *smc) in smc_fback_restore_callbacks() argument
248 struct sock *clcsk = smc->clcsock->sk; in smc_fback_restore_callbacks()
250 write_lock_bh(&clcsk->sk_callback_lock); in smc_fback_restore_callbacks()
251 clcsk->sk_user_data = NULL; in smc_fback_restore_callbacks()
253 smc_clcsock_restore_cb(&clcsk->sk_state_change, &smc->clcsk_state_change); in smc_fback_restore_callbacks()
254 smc_clcsock_restore_cb(&clcsk->sk_data_ready, &smc->clcsk_data_ready); in smc_fback_restore_callbacks()
255 smc_clcsock_restore_cb(&clcsk->sk_write_space, &smc->clcsk_write_space); in smc_fback_restore_callbacks()
256 smc_clcsock_restore_cb(&clcsk->sk_error_report, &smc->clcsk_error_report); in smc_fback_restore_callbacks()
258 write_unlock_bh(&clcsk->sk_callback_lock); in smc_fback_restore_callbacks()
261 static void smc_restore_fallback_changes(struct smc_sock *smc) in smc_restore_fallback_changes() argument
263 if (smc->clcsock->file) { /* non-accepted sockets have no file yet */ in smc_restore_fallback_changes()
264 smc->clcsock->file->private_data = smc->sk.sk_socket; in smc_restore_fallback_changes()
265 smc->clcsock->file = NULL; in smc_restore_fallback_changes()
266 smc_fback_restore_callbacks(smc); in smc_restore_fallback_changes()
270 static int __smc_release(struct smc_sock *smc) in __smc_release() argument
272 struct sock *sk = &smc->sk; in __smc_release()
275 if (!smc->use_fallback) { in __smc_release()
276 rc = smc_close_active(smc); in __smc_release()
278 sk->sk_shutdown |= SHUTDOWN_MASK; in __smc_release()
280 if (sk->sk_state != SMC_CLOSED) { in __smc_release()
281 if (sk->sk_state != SMC_LISTEN && in __smc_release()
282 sk->sk_state != SMC_INIT) in __smc_release()
284 if (sk->sk_state == SMC_LISTEN) { in __smc_release()
286 rc = kernel_sock_shutdown(smc->clcsock, in __smc_release()
289 sk->sk_state = SMC_CLOSED; in __smc_release()
290 sk->sk_state_change(sk); in __smc_release()
292 smc_restore_fallback_changes(smc); in __smc_release()
295 sk->sk_prot->unhash(sk); in __smc_release()
297 if (sk->sk_state == SMC_CLOSED) { in __smc_release()
298 if (smc->clcsock) { in __smc_release()
300 smc_clcsock_release(smc); in __smc_release()
303 if (!smc->use_fallback) in __smc_release()
304 smc_conn_free(&smc->conn); in __smc_release()
312 struct sock *sk = sock->sk; in smc_release()
313 struct smc_sock *smc; in smc_release() local
320 smc = smc_sk(sk); in smc_release()
322 old_state = sk->sk_state; in smc_release()
324 /* cleanup for a dangling non-blocking connect */ in smc_release()
325 if (smc->connect_nonblock && old_state == SMC_INIT) in smc_release()
326 tcp_abort(smc->clcsock->sk, ECONNABORTED); in smc_release()
328 if (cancel_work_sync(&smc->connect_work)) in smc_release()
329 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */ in smc_release()
331 if (sk->sk_state == SMC_LISTEN) in smc_release()
339 if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE && in smc_release()
340 !smc->use_fallback) in smc_release()
341 smc_close_active_abort(smc); in smc_release()
343 rc = __smc_release(smc); in smc_release()
347 sock->sk = NULL; in smc_release()
358 if (sk->sk_state != SMC_CLOSED) in smc_destruct()
369 struct smc_sock *smc; in smc_sock_alloc() local
379 sk->sk_state = SMC_INIT; in smc_sock_alloc()
380 sk->sk_destruct = smc_destruct; in smc_sock_alloc()
381 sk->sk_protocol = protocol; in smc_sock_alloc()
382 WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(net->smc.sysctl_wmem)); in smc_sock_alloc()
383 WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(net->smc.sysctl_rmem)); in smc_sock_alloc()
384 smc = smc_sk(sk); in smc_sock_alloc()
385 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); in smc_sock_alloc()
386 INIT_WORK(&smc->connect_work, smc_connect_work); in smc_sock_alloc()
387 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work); in smc_sock_alloc()
388 INIT_LIST_HEAD(&smc->accept_q); in smc_sock_alloc()
389 spin_lock_init(&smc->accept_q_lock); in smc_sock_alloc()
390 spin_lock_init(&smc->conn.send_lock); in smc_sock_alloc()
391 sk->sk_prot->hash(sk); in smc_sock_alloc()
393 mutex_init(&smc->clcsock_release_lock); in smc_sock_alloc()
394 smc_init_saved_callbacks(smc); in smc_sock_alloc()
403 struct sock *sk = sock->sk; in smc_bind()
404 struct smc_sock *smc; in smc_bind() local
407 smc = smc_sk(sk); in smc_bind()
410 rc = -EINVAL; in smc_bind()
414 rc = -EAFNOSUPPORT; in smc_bind()
415 if (addr->sin_family != AF_INET && in smc_bind()
416 addr->sin_family != AF_INET6 && in smc_bind()
417 addr->sin_family != AF_UNSPEC) in smc_bind()
420 if (addr->sin_family == AF_UNSPEC && in smc_bind()
421 addr->sin_addr.s_addr != htonl(INADDR_ANY)) in smc_bind()
427 rc = -EINVAL; in smc_bind()
428 if (sk->sk_state != SMC_INIT || smc->connect_nonblock) in smc_bind()
431 smc->clcsock->sk->sk_reuse = sk->sk_reuse; in smc_bind()
432 smc->clcsock->sk->sk_reuseport = sk->sk_reuseport; in smc_bind()
433 rc = kernel_bind(smc->clcsock, uaddr, addr_len); in smc_bind()
445 nsk->sk_type = osk->sk_type; in smc_copy_sock_settings()
446 nsk->sk_sndbuf = osk->sk_sndbuf; in smc_copy_sock_settings()
447 nsk->sk_rcvbuf = osk->sk_rcvbuf; in smc_copy_sock_settings()
448 nsk->sk_sndtimeo = osk->sk_sndtimeo; in smc_copy_sock_settings()
449 nsk->sk_rcvtimeo = osk->sk_rcvtimeo; in smc_copy_sock_settings()
450 nsk->sk_mark = osk->sk_mark; in smc_copy_sock_settings()
451 nsk->sk_priority = osk->sk_priority; in smc_copy_sock_settings()
452 nsk->sk_rcvlowat = osk->sk_rcvlowat; in smc_copy_sock_settings()
453 nsk->sk_bound_dev_if = osk->sk_bound_dev_if; in smc_copy_sock_settings()
454 nsk->sk_err = osk->sk_err; in smc_copy_sock_settings()
456 nsk->sk_flags &= ~mask; in smc_copy_sock_settings()
457 nsk->sk_flags |= osk->sk_flags & mask; in smc_copy_sock_settings()
475 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
476 * clc socket (since smc is not called for these options from net/core)
478 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) in smc_copy_sock_settings_to_clc() argument
480 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); in smc_copy_sock_settings_to_clc()
487 /* copy only settings and flags relevant for smc from clc to smc socket */
488 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) in smc_copy_sock_settings_to_smc() argument
490 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); in smc_copy_sock_settings_to_smc()
497 struct smc_link_group *lgr = link->lgr; in smcr_lgr_reg_sndbufs()
500 if (!snd_desc->is_vm) in smcr_lgr_reg_sndbufs()
501 return -EINVAL; in smcr_lgr_reg_sndbufs()
504 mutex_lock(&lgr->llc_conf_mutex); in smcr_lgr_reg_sndbufs()
506 if (!smc_link_active(&lgr->lnk[i])) in smcr_lgr_reg_sndbufs()
508 rc = smcr_link_reg_buf(&lgr->lnk[i], snd_desc); in smcr_lgr_reg_sndbufs()
512 mutex_unlock(&lgr->llc_conf_mutex); in smcr_lgr_reg_sndbufs()
520 struct smc_link_group *lgr = link->lgr; in smcr_lgr_reg_rmbs()
529 mutex_lock(&lgr->llc_conf_mutex); in smcr_lgr_reg_rmbs()
531 if (!smc_link_active(&lgr->lnk[i])) in smcr_lgr_reg_rmbs()
533 rc = smcr_link_reg_buf(&lgr->lnk[i], rmb_desc); in smcr_lgr_reg_rmbs()
541 rc = -EFAULT; in smcr_lgr_reg_rmbs()
544 rmb_desc->is_conf_rkey = true; in smcr_lgr_reg_rmbs()
546 mutex_unlock(&lgr->llc_conf_mutex); in smcr_lgr_reg_rmbs()
547 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); in smcr_lgr_reg_rmbs()
551 static int smcr_clnt_conf_first_link(struct smc_sock *smc) in smcr_clnt_conf_first_link() argument
553 struct smc_link *link = smc->conn.lnk; in smcr_clnt_conf_first_link()
558 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME, in smcr_clnt_conf_first_link()
563 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_clnt_conf_first_link()
565 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc; in smcr_clnt_conf_first_link()
569 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl); in smcr_clnt_conf_first_link()
580 if (smc->conn.sndbuf_desc->is_vm) { in smcr_clnt_conf_first_link()
581 if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc)) in smcr_clnt_conf_first_link()
586 if (smcr_link_reg_buf(link, smc->conn.rmb_desc)) in smcr_clnt_conf_first_link()
590 smc->conn.rmb_desc->is_conf_rkey = true; in smcr_clnt_conf_first_link()
598 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE); in smcr_clnt_conf_first_link()
601 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME, in smcr_clnt_conf_first_link()
606 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_clnt_conf_first_link()
608 if (rc == -EAGAIN) in smcr_clnt_conf_first_link()
612 smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl); in smcr_clnt_conf_first_link()
627 static void smc_conn_save_peer_info_fce(struct smc_sock *smc, in smc_conn_save_peer_info_fce() argument
635 if (clc->hdr.version == SMC_V1 || in smc_conn_save_peer_info_fce()
636 !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) in smc_conn_save_peer_info_fce()
639 if (smc->conn.lgr->is_smcd) { in smc_conn_save_peer_info_fce()
640 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid, in smc_conn_save_peer_info_fce()
645 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid, in smc_conn_save_peer_info_fce()
651 smc->conn.lgr->peer_os = fce->os_type; in smc_conn_save_peer_info_fce()
652 smc->conn.lgr->peer_smc_release = fce->release; in smc_conn_save_peer_info_fce()
653 if (smc_isascii(fce->hostname)) in smc_conn_save_peer_info_fce()
654 memcpy(smc->conn.lgr->peer_hostname, fce->hostname, in smc_conn_save_peer_info_fce()
658 static void smcr_conn_save_peer_info(struct smc_sock *smc, in smcr_conn_save_peer_info() argument
661 int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size); in smcr_conn_save_peer_info()
663 smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx; in smcr_conn_save_peer_info()
664 smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token); in smcr_conn_save_peer_info()
665 smc->conn.peer_rmbe_size = bufsize; in smcr_conn_save_peer_info()
666 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); in smcr_conn_save_peer_info()
667 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1); in smcr_conn_save_peer_info()
670 static void smcd_conn_save_peer_info(struct smc_sock *smc, in smcd_conn_save_peer_info() argument
673 int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size); in smcd_conn_save_peer_info()
675 smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx; in smcd_conn_save_peer_info()
676 smc->conn.peer_token = clc->d0.token; in smcd_conn_save_peer_info()
678 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg); in smcd_conn_save_peer_info()
679 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); in smcd_conn_save_peer_info()
680 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx; in smcd_conn_save_peer_info()
683 static void smc_conn_save_peer_info(struct smc_sock *smc, in smc_conn_save_peer_info() argument
686 if (smc->conn.lgr->is_smcd) in smc_conn_save_peer_info()
687 smcd_conn_save_peer_info(smc, clc); in smc_conn_save_peer_info()
689 smcr_conn_save_peer_info(smc, clc); in smc_conn_save_peer_info()
690 smc_conn_save_peer_info_fce(smc, clc); in smc_conn_save_peer_info()
697 link->peer_qpn = ntoh24(clc->r0.qpn); in smc_link_save_peer_info()
698 memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE); in smc_link_save_peer_info()
699 memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac)); in smc_link_save_peer_info()
700 link->peer_psn = ntoh24(clc->r0.psn); in smc_link_save_peer_info()
701 link->peer_mtu = clc->r0.qp_mtu; in smc_link_save_peer_info()
704 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc, in smc_stat_inc_fback_rsn_cnt() argument
710 if (fback_arr[cnt].fback_code == smc->fallback_rsn) { in smc_stat_inc_fback_rsn_cnt()
715 fback_arr[cnt].fback_code = smc->fallback_rsn; in smc_stat_inc_fback_rsn_cnt()
722 static void smc_stat_fallback(struct smc_sock *smc) in smc_stat_fallback() argument
724 struct net *net = sock_net(&smc->sk); in smc_stat_fallback()
726 mutex_lock(&net->smc.mutex_fback_rsn); in smc_stat_fallback()
727 if (smc->listen_smc) { in smc_stat_fallback()
728 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv); in smc_stat_fallback()
729 net->smc.fback_rsn->srv_fback_cnt++; in smc_stat_fallback()
731 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt); in smc_stat_fallback()
732 net->smc.fback_rsn->clnt_fback_cnt++; in smc_stat_fallback()
734 mutex_unlock(&net->smc.mutex_fback_rsn); in smc_stat_fallback()
738 static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key) in smc_fback_wakeup_waitqueue() argument
743 wq = rcu_dereference(smc->sk.sk_wq); in smc_fback_wakeup_waitqueue()
747 /* wake up smc sk->sk_wq */ in smc_fback_wakeup_waitqueue()
750 wake_up_interruptible_all(&wq->wait); in smc_fback_wakeup_waitqueue()
755 wake_up_interruptible_sync_poll(&wq->wait, flags); in smc_fback_wakeup_waitqueue()
758 wake_up_interruptible_poll(&wq->wait, flags); in smc_fback_wakeup_waitqueue()
768 mark->woken = true; in smc_fback_mark_woken()
769 mark->key = key; in smc_fback_mark_woken()
773 static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk, in smc_fback_forward_wakeup() argument
782 wq = rcu_dereference(clcsk->sk_wq); in smc_fback_forward_wakeup()
790 smc_fback_wakeup_waitqueue(smc, mark.key); in smc_fback_forward_wakeup()
797 struct smc_sock *smc; in smc_fback_state_change() local
799 read_lock_bh(&clcsk->sk_callback_lock); in smc_fback_state_change()
800 smc = smc_clcsock_user_data(clcsk); in smc_fback_state_change()
801 if (smc) in smc_fback_state_change()
802 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_state_change()
803 smc->clcsk_state_change); in smc_fback_state_change()
804 read_unlock_bh(&clcsk->sk_callback_lock); in smc_fback_state_change()
809 struct smc_sock *smc; in smc_fback_data_ready() local
811 read_lock_bh(&clcsk->sk_callback_lock); in smc_fback_data_ready()
812 smc = smc_clcsock_user_data(clcsk); in smc_fback_data_ready()
813 if (smc) in smc_fback_data_ready()
814 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_data_ready()
815 smc->clcsk_data_ready); in smc_fback_data_ready()
816 read_unlock_bh(&clcsk->sk_callback_lock); in smc_fback_data_ready()
821 struct smc_sock *smc; in smc_fback_write_space() local
823 read_lock_bh(&clcsk->sk_callback_lock); in smc_fback_write_space()
824 smc = smc_clcsock_user_data(clcsk); in smc_fback_write_space()
825 if (smc) in smc_fback_write_space()
826 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_write_space()
827 smc->clcsk_write_space); in smc_fback_write_space()
828 read_unlock_bh(&clcsk->sk_callback_lock); in smc_fback_write_space()
833 struct smc_sock *smc; in smc_fback_error_report() local
835 read_lock_bh(&clcsk->sk_callback_lock); in smc_fback_error_report()
836 smc = smc_clcsock_user_data(clcsk); in smc_fback_error_report()
837 if (smc) in smc_fback_error_report()
838 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_error_report()
839 smc->clcsk_error_report); in smc_fback_error_report()
840 read_unlock_bh(&clcsk->sk_callback_lock); in smc_fback_error_report()
843 static void smc_fback_replace_callbacks(struct smc_sock *smc) in smc_fback_replace_callbacks() argument
845 struct sock *clcsk = smc->clcsock->sk; in smc_fback_replace_callbacks()
847 write_lock_bh(&clcsk->sk_callback_lock); in smc_fback_replace_callbacks()
848 clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); in smc_fback_replace_callbacks()
850 smc_clcsock_replace_cb(&clcsk->sk_state_change, smc_fback_state_change, in smc_fback_replace_callbacks()
851 &smc->clcsk_state_change); in smc_fback_replace_callbacks()
852 smc_clcsock_replace_cb(&clcsk->sk_data_ready, smc_fback_data_ready, in smc_fback_replace_callbacks()
853 &smc->clcsk_data_ready); in smc_fback_replace_callbacks()
854 smc_clcsock_replace_cb(&clcsk->sk_write_space, smc_fback_write_space, in smc_fback_replace_callbacks()
855 &smc->clcsk_write_space); in smc_fback_replace_callbacks()
856 smc_clcsock_replace_cb(&clcsk->sk_error_report, smc_fback_error_report, in smc_fback_replace_callbacks()
857 &smc->clcsk_error_report); in smc_fback_replace_callbacks()
859 write_unlock_bh(&clcsk->sk_callback_lock); in smc_fback_replace_callbacks()
862 static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code) in smc_switch_to_fallback() argument
866 mutex_lock(&smc->clcsock_release_lock); in smc_switch_to_fallback()
867 if (!smc->clcsock) { in smc_switch_to_fallback()
868 rc = -EBADF; in smc_switch_to_fallback()
872 smc->use_fallback = true; in smc_switch_to_fallback()
873 smc->fallback_rsn = reason_code; in smc_switch_to_fallback()
874 smc_stat_fallback(smc); in smc_switch_to_fallback()
875 trace_smc_switch_to_fallback(smc, reason_code); in smc_switch_to_fallback()
876 if (smc->sk.sk_socket && smc->sk.sk_socket->file) { in smc_switch_to_fallback()
877 smc->clcsock->file = smc->sk.sk_socket->file; in smc_switch_to_fallback()
878 smc->clcsock->file->private_data = smc->clcsock; in smc_switch_to_fallback()
879 smc->clcsock->wq.fasync_list = in smc_switch_to_fallback()
880 smc->sk.sk_socket->wq.fasync_list; in smc_switch_to_fallback()
883 * in smc sk->sk_wq and they should be woken up in smc_switch_to_fallback()
886 smc_fback_replace_callbacks(smc); in smc_switch_to_fallback()
889 mutex_unlock(&smc->clcsock_release_lock); in smc_switch_to_fallback()
894 static int smc_connect_fallback(struct smc_sock *smc, int reason_code) in smc_connect_fallback() argument
896 struct net *net = sock_net(&smc->sk); in smc_connect_fallback()
899 rc = smc_switch_to_fallback(smc, reason_code); in smc_connect_fallback()
901 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_fallback()
902 if (smc->sk.sk_state == SMC_INIT) in smc_connect_fallback()
903 sock_put(&smc->sk); /* passive closing */ in smc_connect_fallback()
906 smc_copy_sock_settings_to_clc(smc); in smc_connect_fallback()
907 smc->connect_nonblock = 0; in smc_connect_fallback()
908 if (smc->sk.sk_state == SMC_INIT) in smc_connect_fallback()
909 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_fallback()
914 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code, in smc_connect_decline_fallback() argument
917 struct net *net = sock_net(&smc->sk); in smc_connect_decline_fallback()
921 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_decline_fallback()
922 if (smc->sk.sk_state == SMC_INIT) in smc_connect_decline_fallback()
923 sock_put(&smc->sk); /* passive closing */ in smc_connect_decline_fallback()
927 rc = smc_clc_send_decline(smc, reason_code, version); in smc_connect_decline_fallback()
929 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_decline_fallback()
930 if (smc->sk.sk_state == SMC_INIT) in smc_connect_decline_fallback()
931 sock_put(&smc->sk); /* passive closing */ in smc_connect_decline_fallback()
935 return smc_connect_fallback(smc, reason_code); in smc_connect_decline_fallback()
938 static void smc_conn_abort(struct smc_sock *smc, int local_first) in smc_conn_abort() argument
940 struct smc_connection *conn = &smc->conn; in smc_conn_abort()
941 struct smc_link_group *lgr = conn->lgr; in smc_conn_abort()
954 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini) in smc_find_rdma_device() argument
960 smc_pnet_find_roce_resource(smc->clcsock->sk, ini); in smc_find_rdma_device()
961 if (!ini->check_smcrv2 && !ini->ib_dev) in smc_find_rdma_device()
963 if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2) in smc_find_rdma_device()
970 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini) in smc_find_ism_device() argument
973 smc_pnet_find_ism_resource(smc->clcsock->sk, ini); in smc_find_ism_device()
974 if (!ini->ism_dev[0]) in smc_find_ism_device()
977 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]); in smc_find_ism_device()
985 int i = (!ini->ism_dev[0]) ? 1 : 0; in smc_find_ism_v2_is_unique_chid()
988 if (ini->ism_chid[i] == chid) in smc_find_ism_v2_is_unique_chid()
996 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc, in smc_find_ism_v2_device_clnt() argument
1004 if (smcd_indicated(ini->smc_type_v1)) in smc_find_ism_v2_device_clnt()
1008 if (smcd->going_away || smcd == ini->ism_dev[0]) in smc_find_ism_v2_device_clnt()
1013 if (!smc_pnet_is_pnetid_set(smcd->pnetid) || in smc_find_ism_v2_device_clnt()
1014 smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) { in smc_find_ism_v2_device_clnt()
1015 ini->ism_dev[i] = smcd; in smc_find_ism_v2_device_clnt()
1016 ini->ism_chid[i] = chid; in smc_find_ism_v2_device_clnt()
1017 ini->is_smcd = true; in smc_find_ism_v2_device_clnt()
1025 ini->ism_offered_cnt = i - 1; in smc_find_ism_v2_device_clnt()
1026 if (!ini->ism_dev[0] && !ini->ism_dev[1]) in smc_find_ism_v2_device_clnt()
1027 ini->smcd_version = 0; in smc_find_ism_v2_device_clnt()
1032 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
1033 static int smc_connect_ism_vlan_setup(struct smc_sock *smc, in smc_connect_ism_vlan_setup() argument
1036 if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id)) in smc_connect_ism_vlan_setup()
1041 static int smc_find_proposal_devices(struct smc_sock *smc, in smc_find_proposal_devices() argument
1047 if (!(ini->smcd_version & SMC_V1) || in smc_find_proposal_devices()
1048 smc_find_ism_device(smc, ini) || in smc_find_proposal_devices()
1049 smc_connect_ism_vlan_setup(smc, ini)) in smc_find_proposal_devices()
1050 ini->smcd_version &= ~SMC_V1; in smc_find_proposal_devices()
1054 if (!(ini->smcr_version & SMC_V1) || in smc_find_proposal_devices()
1055 smc_find_rdma_device(smc, ini)) in smc_find_proposal_devices()
1056 ini->smcr_version &= ~SMC_V1; in smc_find_proposal_devices()
1059 ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1, in smc_find_proposal_devices()
1060 ini->smcr_version & SMC_V1); in smc_find_proposal_devices()
1063 if (!(ini->smcd_version & SMC_V2) || in smc_find_proposal_devices()
1065 smc_find_ism_v2_device_clnt(smc, ini)) in smc_find_proposal_devices()
1066 ini->smcd_version &= ~SMC_V2; in smc_find_proposal_devices()
1069 ini->check_smcrv2 = true; in smc_find_proposal_devices()
1070 ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr; in smc_find_proposal_devices()
1071 if (!(ini->smcr_version & SMC_V2) || in smc_find_proposal_devices()
1072 smc->clcsock->sk->sk_family != AF_INET || in smc_find_proposal_devices()
1074 smc_find_rdma_device(smc, ini)) in smc_find_proposal_devices()
1075 ini->smcr_version &= ~SMC_V2; in smc_find_proposal_devices()
1076 ini->check_smcrv2 = false; in smc_find_proposal_devices()
1078 ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2, in smc_find_proposal_devices()
1079 ini->smcr_version & SMC_V2); in smc_find_proposal_devices()
1082 if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N) in smc_find_proposal_devices()
1088 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
1089 * used, the VLAN ID will be registered again during the connection setup.
1091 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, in smc_connect_ism_vlan_cleanup() argument
1094 if (!smcd_indicated(ini->smc_type_v1)) in smc_connect_ism_vlan_cleanup()
1096 if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id)) in smc_connect_ism_vlan_cleanup()
1107 static int smc_connect_clc(struct smc_sock *smc, in smc_connect_clc() argument
1114 rc = smc_clc_send_proposal(smc, ini); in smc_connect_clc()
1117 /* receive SMC Accept CLC message */ in smc_connect_clc()
1118 return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN, in smc_connect_clc()
1129 memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE); in smc_fill_gid_list()
1135 alt_ini->vlan_id = lgr->vlan_id; in smc_fill_gid_list()
1136 alt_ini->check_smcrv2 = true; in smc_fill_gid_list()
1137 alt_ini->smcrv2.saddr = lgr->saddr; in smc_fill_gid_list()
1140 if (!alt_ini->smcrv2.ib_dev_v2) in smc_fill_gid_list()
1143 memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2, in smc_fill_gid_list()
1150 static int smc_connect_rdma_v2_prepare(struct smc_sock *smc, in smc_connect_rdma_v2_prepare() argument
1160 if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1) in smc_connect_rdma_v2_prepare()
1163 if (fce->v2_direct) { in smc_connect_rdma_v2_prepare()
1164 memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN); in smc_connect_rdma_v2_prepare()
1165 ini->smcrv2.uses_gateway = false; in smc_connect_rdma_v2_prepare()
1167 if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr, in smc_connect_rdma_v2_prepare()
1168 smc_ib_gid_to_ipv4(aclc->r0.lcl.gid), in smc_connect_rdma_v2_prepare()
1169 ini->smcrv2.nexthop_mac, in smc_connect_rdma_v2_prepare()
1170 &ini->smcrv2.uses_gateway)) in smc_connect_rdma_v2_prepare()
1172 if (!ini->smcrv2.uses_gateway) { in smc_connect_rdma_v2_prepare()
1181 static int smc_connect_rdma(struct smc_sock *smc, in smc_connect_rdma() argument
1189 ini->is_smcd = false; in smc_connect_rdma()
1190 ini->ib_clcqpn = ntoh24(aclc->r0.qpn); in smc_connect_rdma()
1191 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; in smc_connect_rdma()
1192 memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN); in smc_connect_rdma()
1193 memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE); in smc_connect_rdma()
1194 memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN); in smc_connect_rdma()
1196 reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini); in smc_connect_rdma()
1201 reason_code = smc_conn_create(smc, ini); in smc_connect_rdma()
1207 smc_conn_save_peer_info(smc, aclc); in smc_connect_rdma()
1209 if (ini->first_contact_local) { in smc_connect_rdma()
1210 link = smc->conn.lnk; in smc_connect_rdma()
1215 struct smc_link *l = &smc->conn.lgr->lnk[i]; in smc_connect_rdma()
1217 if (l->peer_qpn == ntoh24(aclc->r0.qpn) && in smc_connect_rdma()
1218 !memcmp(l->peer_gid, &aclc->r0.lcl.gid, in smc_connect_rdma()
1220 (aclc->hdr.version > SMC_V1 || in smc_connect_rdma()
1221 !memcmp(l->peer_mac, &aclc->r0.lcl.mac, in smc_connect_rdma()
1222 sizeof(l->peer_mac)))) { in smc_connect_rdma()
1231 smc_switch_link_and_count(&smc->conn, link); in smc_connect_rdma()
1235 if (smc_buf_create(smc, false)) { in smc_connect_rdma()
1240 if (ini->first_contact_local) in smc_connect_rdma()
1243 if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) { in smc_connect_rdma()
1248 smc_close_init(smc); in smc_connect_rdma()
1249 smc_rx_init(smc); in smc_connect_rdma()
1251 if (ini->first_contact_local) { in smc_connect_rdma()
1258 if (smc->conn.sndbuf_desc->is_vm) { in smc_connect_rdma()
1259 if (smcr_lgr_reg_sndbufs(link, smc->conn.sndbuf_desc)) { in smc_connect_rdma()
1264 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) { in smc_connect_rdma()
1270 if (aclc->hdr.version > SMC_V1) { in smc_connect_rdma()
1274 eid = clc_v2->r1.eid; in smc_connect_rdma()
1275 if (ini->first_contact_local) in smc_connect_rdma()
1276 smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist, in smc_connect_rdma()
1277 link->smcibdev, link->gid); in smc_connect_rdma()
1280 reason_code = smc_clc_send_confirm(smc, ini->first_contact_local, in smc_connect_rdma()
1281 aclc->hdr.version, eid, ini); in smc_connect_rdma()
1285 smc_tx_init(smc); in smc_connect_rdma()
1287 if (ini->first_contact_local) { in smc_connect_rdma()
1289 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK); in smc_connect_rdma()
1290 reason_code = smcr_clnt_conf_first_link(smc); in smc_connect_rdma()
1291 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl); in smc_connect_rdma()
1297 smc_copy_sock_settings_to_clc(smc); in smc_connect_rdma()
1298 smc->connect_nonblock = 0; in smc_connect_rdma()
1299 if (smc->sk.sk_state == SMC_INIT) in smc_connect_rdma()
1300 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_rdma()
1304 smc_conn_abort(smc, ini->first_contact_local); in smc_connect_rdma()
1306 smc->connect_nonblock = 0; in smc_connect_rdma()
1320 for (i = 0; i < ini->ism_offered_cnt + 1; i++) { in smc_v2_determine_accepted_chid()
1321 if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) { in smc_v2_determine_accepted_chid()
1322 ini->ism_selected = i; in smc_v2_determine_accepted_chid()
1327 return -EPROTO; in smc_v2_determine_accepted_chid()
1331 static int smc_connect_ism(struct smc_sock *smc, in smc_connect_ism() argument
1338 ini->is_smcd = true; in smc_connect_ism()
1339 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; in smc_connect_ism()
1341 if (aclc->hdr.version == SMC_V2) { in smc_connect_ism()
1349 ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid; in smc_connect_ism()
1351 /* there is only one lgr role for SMC-D; use server lock */ in smc_connect_ism()
1353 rc = smc_conn_create(smc, ini); in smc_connect_ism()
1360 rc = smc_buf_create(smc, true); in smc_connect_ism()
1362 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM; in smc_connect_ism()
1366 smc_conn_save_peer_info(smc, aclc); in smc_connect_ism()
1367 smc_close_init(smc); in smc_connect_ism()
1368 smc_rx_init(smc); in smc_connect_ism()
1369 smc_tx_init(smc); in smc_connect_ism()
1371 if (aclc->hdr.version > SMC_V1) { in smc_connect_ism()
1375 eid = clc_v2->d1.eid; in smc_connect_ism()
1378 rc = smc_clc_send_confirm(smc, ini->first_contact_local, in smc_connect_ism()
1379 aclc->hdr.version, eid, NULL); in smc_connect_ism()
1384 smc_copy_sock_settings_to_clc(smc); in smc_connect_ism()
1385 smc->connect_nonblock = 0; in smc_connect_ism()
1386 if (smc->sk.sk_state == SMC_INIT) in smc_connect_ism()
1387 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_ism()
1391 smc_conn_abort(smc, ini->first_contact_local); in smc_connect_ism()
1393 smc->connect_nonblock = 0; in smc_connect_ism()
1402 if (aclc->hdr.typev1 != SMC_TYPE_R && in smc_connect_check_aclc()
1403 aclc->hdr.typev1 != SMC_TYPE_D) in smc_connect_check_aclc()
1406 if (aclc->hdr.version >= SMC_V2) { in smc_connect_check_aclc()
1407 if ((aclc->hdr.typev1 == SMC_TYPE_R && in smc_connect_check_aclc()
1408 !smcr_indicated(ini->smc_type_v2)) || in smc_connect_check_aclc()
1409 (aclc->hdr.typev1 == SMC_TYPE_D && in smc_connect_check_aclc()
1410 !smcd_indicated(ini->smc_type_v2))) in smc_connect_check_aclc()
1413 if ((aclc->hdr.typev1 == SMC_TYPE_R && in smc_connect_check_aclc()
1414 !smcr_indicated(ini->smc_type_v1)) || in smc_connect_check_aclc()
1415 (aclc->hdr.typev1 == SMC_TYPE_D && in smc_connect_check_aclc()
1416 !smcd_indicated(ini->smc_type_v1))) in smc_connect_check_aclc()
1424 static int __smc_connect(struct smc_sock *smc) in __smc_connect() argument
1433 if (smc->use_fallback) in __smc_connect()
1434 return smc_connect_fallback(smc, smc->fallback_rsn); in __smc_connect()
1436 /* if peer has not signalled SMC-capability, fall back */ in __smc_connect()
1437 if (!tcp_sk(smc->clcsock->sk)->syn_smc) in __smc_connect()
1438 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC); in __smc_connect()
1440 /* IPSec connections opt out of SMC optimizations */ in __smc_connect()
1441 if (using_ipsec(smc)) in __smc_connect()
1442 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC, in __smc_connect()
1447 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM, in __smc_connect()
1450 ini->smcd_version = SMC_V1 | SMC_V2; in __smc_connect()
1451 ini->smcr_version = SMC_V1 | SMC_V2; in __smc_connect()
1452 ini->smc_type_v1 = SMC_TYPE_B; in __smc_connect()
1453 ini->smc_type_v2 = SMC_TYPE_B; in __smc_connect()
1455 /* get vlan id from IP device */ in __smc_connect()
1456 if (smc_vlan_by_tcpsk(smc->clcsock, ini)) { in __smc_connect()
1457 ini->smcd_version &= ~SMC_V1; in __smc_connect()
1458 ini->smcr_version = 0; in __smc_connect()
1459 ini->smc_type_v1 = SMC_TYPE_N; in __smc_connect()
1460 if (!ini->smcd_version) { in __smc_connect()
1466 rc = smc_find_proposal_devices(smc, ini); in __smc_connect()
1479 rc = smc_connect_clc(smc, aclc2, ini); in __smc_connect()
1481 /* -EAGAIN on timeout, see tcp_recvmsg() */ in __smc_connect()
1482 if (rc == -EAGAIN) { in __smc_connect()
1483 rc = -ETIMEDOUT; in __smc_connect()
1484 smc->sk.sk_err = ETIMEDOUT; in __smc_connect()
1489 /* check if smc modes and versions of CLC proposal and accept match */ in __smc_connect()
1491 version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2; in __smc_connect()
1496 if (aclc->hdr.typev1 == SMC_TYPE_R) { in __smc_connect()
1497 ini->smcr_version = version; in __smc_connect()
1498 rc = smc_connect_rdma(smc, aclc, ini); in __smc_connect()
1499 } else if (aclc->hdr.typev1 == SMC_TYPE_D) { in __smc_connect()
1500 ini->smcd_version = version; in __smc_connect()
1501 rc = smc_connect_ism(smc, aclc, ini); in __smc_connect()
1506 SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc); in __smc_connect()
1507 smc_connect_ism_vlan_cleanup(smc, ini); in __smc_connect()
1513 smc_connect_ism_vlan_cleanup(smc, ini); in __smc_connect()
1517 return smc_connect_decline_fallback(smc, rc, version); in __smc_connect()
1522 struct smc_sock *smc = container_of(work, struct smc_sock, in smc_connect_work() local
1524 long timeo = smc->sk.sk_sndtimeo; in smc_connect_work()
1529 lock_sock(smc->clcsock->sk); in smc_connect_work()
1530 if (smc->clcsock->sk->sk_err) { in smc_connect_work()
1531 smc->sk.sk_err = smc->clcsock->sk->sk_err; in smc_connect_work()
1532 } else if ((1 << smc->clcsock->sk->sk_state) & in smc_connect_work()
1534 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo); in smc_connect_work()
1535 if ((rc == -EPIPE) && in smc_connect_work()
1536 ((1 << smc->clcsock->sk->sk_state) & in smc_connect_work()
1540 release_sock(smc->clcsock->sk); in smc_connect_work()
1541 lock_sock(&smc->sk); in smc_connect_work()
1542 if (rc != 0 || smc->sk.sk_err) { in smc_connect_work()
1543 smc->sk.sk_state = SMC_CLOSED; in smc_connect_work()
1544 if (rc == -EPIPE || rc == -EAGAIN) in smc_connect_work()
1545 smc->sk.sk_err = EPIPE; in smc_connect_work()
1546 else if (rc == -ECONNREFUSED) in smc_connect_work()
1547 smc->sk.sk_err = ECONNREFUSED; in smc_connect_work()
1549 smc->sk.sk_err = -sock_intr_errno(timeo); in smc_connect_work()
1550 sock_put(&smc->sk); /* passive closing */ in smc_connect_work()
1554 rc = __smc_connect(smc); in smc_connect_work()
1556 smc->sk.sk_err = -rc; in smc_connect_work()
1559 if (!sock_flag(&smc->sk, SOCK_DEAD)) { in smc_connect_work()
1560 if (smc->sk.sk_err) { in smc_connect_work()
1561 smc->sk.sk_state_change(&smc->sk); in smc_connect_work()
1563 smc->clcsock->sk->sk_write_space(smc->clcsock->sk); in smc_connect_work()
1564 smc->sk.sk_write_space(&smc->sk); in smc_connect_work()
1567 release_sock(&smc->sk); in smc_connect_work()
1573 struct sock *sk = sock->sk; in smc_connect()
1574 struct smc_sock *smc; in smc_connect() local
1575 int rc = -EINVAL; in smc_connect()
1577 smc = smc_sk(sk); in smc_connect()
1579 /* separate smc parameter checking to be safe */ in smc_connect()
1580 if (alen < sizeof(addr->sa_family)) in smc_connect()
1582 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) in smc_connect()
1586 switch (sock->state) { in smc_connect()
1588 rc = -EINVAL; in smc_connect()
1591 rc = sk->sk_state == SMC_ACTIVE ? -EISCONN : -EINVAL; in smc_connect()
1594 if (sk->sk_state == SMC_ACTIVE) in smc_connect()
1598 sock->state = SS_CONNECTING; in smc_connect()
1602 switch (sk->sk_state) { in smc_connect()
1606 rc = sock_error(sk) ? : -ECONNABORTED; in smc_connect()
1607 sock->state = SS_UNCONNECTED; in smc_connect()
1610 rc = -EISCONN; in smc_connect()
1616 smc_copy_sock_settings_to_clc(smc); in smc_connect()
1617 tcp_sk(smc->clcsock->sk)->syn_smc = 1; in smc_connect()
1618 if (smc->connect_nonblock) { in smc_connect()
1619 rc = -EALREADY; in smc_connect()
1622 rc = kernel_connect(smc->clcsock, addr, alen, flags); in smc_connect()
1623 if (rc && rc != -EINPROGRESS) in smc_connect()
1626 if (smc->use_fallback) { in smc_connect()
1627 sock->state = rc ? SS_CONNECTING : SS_CONNECTED; in smc_connect()
1630 sock_hold(&smc->sk); /* sock put in passive closing */ in smc_connect()
1632 if (queue_work(smc_hs_wq, &smc->connect_work)) in smc_connect()
1633 smc->connect_nonblock = 1; in smc_connect()
1634 rc = -EINPROGRESS; in smc_connect()
1637 rc = __smc_connect(smc); in smc_connect()
1644 sock->state = SS_CONNECTED; in smc_connect()
1654 struct sock *lsk = &lsmc->sk; in smc_clcsock_accept()
1656 int rc = -EINVAL; in smc_clcsock_accept()
1659 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol); in smc_clcsock_accept()
1661 rc = -ENOMEM; in smc_clcsock_accept()
1662 lsk->sk_err = ENOMEM; in smc_clcsock_accept()
1669 mutex_lock(&lsmc->clcsock_release_lock); in smc_clcsock_accept()
1670 if (lsmc->clcsock) in smc_clcsock_accept()
1671 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK); in smc_clcsock_accept()
1672 mutex_unlock(&lsmc->clcsock_release_lock); in smc_clcsock_accept()
1674 if (rc < 0 && rc != -EAGAIN) in smc_clcsock_accept()
1675 lsk->sk_err = -rc; in smc_clcsock_accept()
1676 if (rc < 0 || lsk->sk_state == SMC_CLOSED) { in smc_clcsock_accept()
1677 new_sk->sk_prot->unhash(new_sk); in smc_clcsock_accept()
1680 new_sk->sk_state = SMC_CLOSED; in smc_clcsock_accept()
1687 /* new clcsock has inherited the smc listen-specific sk_data_ready in smc_clcsock_accept()
1690 new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready; in smc_clcsock_accept()
1692 /* if new clcsock has also inherited the fallback-specific callback in smc_clcsock_accept()
1695 if (lsmc->use_fallback) { in smc_clcsock_accept()
1696 if (lsmc->clcsk_state_change) in smc_clcsock_accept()
1697 new_clcsock->sk->sk_state_change = lsmc->clcsk_state_change; in smc_clcsock_accept()
1698 if (lsmc->clcsk_write_space) in smc_clcsock_accept()
1699 new_clcsock->sk->sk_write_space = lsmc->clcsk_write_space; in smc_clcsock_accept()
1700 if (lsmc->clcsk_error_report) in smc_clcsock_accept()
1701 new_clcsock->sk->sk_error_report = lsmc->clcsk_error_report; in smc_clcsock_accept()
1704 (*new_smc)->clcsock = new_clcsock; in smc_clcsock_accept()
1717 spin_lock(&par->accept_q_lock); in smc_accept_enqueue()
1718 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q); in smc_accept_enqueue()
1719 spin_unlock(&par->accept_q_lock); in smc_accept_enqueue()
1726 struct smc_sock *par = smc_sk(sk)->listen_smc; in smc_accept_unlink()
1728 spin_lock(&par->accept_q_lock); in smc_accept_unlink()
1729 list_del_init(&smc_sk(sk)->accept_q); in smc_accept_unlink()
1730 spin_unlock(&par->accept_q_lock); in smc_accept_unlink()
1731 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk); in smc_accept_unlink()
1744 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) { in smc_accept_dequeue()
1748 if (new_sk->sk_state == SMC_CLOSED) { in smc_accept_dequeue()
1749 new_sk->sk_prot->unhash(new_sk); in smc_accept_dequeue()
1750 if (isk->clcsock) { in smc_accept_dequeue()
1751 sock_release(isk->clcsock); in smc_accept_dequeue()
1752 isk->clcsock = NULL; in smc_accept_dequeue()
1759 new_sock->state = SS_CONNECTED; in smc_accept_dequeue()
1760 if (isk->use_fallback) { in smc_accept_dequeue()
1761 smc_sk(new_sk)->clcsock->file = new_sock->file; in smc_accept_dequeue()
1762 isk->clcsock->file->private_data = isk->clcsock; in smc_accept_dequeue()
1773 struct smc_sock *smc = smc_sk(sk); in smc_close_non_accepted() local
1777 if (!sk->sk_lingertime) in smc_close_non_accepted()
1779 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT; in smc_close_non_accepted()
1780 __smc_release(smc); in smc_close_non_accepted()
1786 static int smcr_serv_conf_first_link(struct smc_sock *smc) in smcr_serv_conf_first_link() argument
1788 struct smc_link *link = smc->conn.lnk; in smcr_serv_conf_first_link()
1793 if (smc->conn.sndbuf_desc->is_vm) { in smcr_serv_conf_first_link()
1794 if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc)) in smcr_serv_conf_first_link()
1799 if (smcr_link_reg_buf(link, smc->conn.rmb_desc)) in smcr_serv_conf_first_link()
1808 qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME, in smcr_serv_conf_first_link()
1813 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_serv_conf_first_link()
1815 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc; in smcr_serv_conf_first_link()
1819 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl); in smcr_serv_conf_first_link()
1824 smc->conn.rmb_desc->is_conf_rkey = true; in smcr_serv_conf_first_link()
1827 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE); in smcr_serv_conf_first_link()
1829 /* initial contact - try to establish second link */ in smcr_serv_conf_first_link()
1837 struct smc_sock *lsmc = new_smc->listen_smc; in smc_listen_out()
1838 struct sock *newsmcsk = &new_smc->sk; in smc_listen_out()
1840 if (tcp_sk(new_smc->clcsock->sk)->syn_smc) in smc_listen_out()
1841 atomic_dec(&lsmc->queued_smc_hs); in smc_listen_out()
1843 if (lsmc->sk.sk_state == SMC_LISTEN) { in smc_listen_out()
1844 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); in smc_listen_out()
1845 smc_accept_enqueue(&lsmc->sk, newsmcsk); in smc_listen_out()
1846 release_sock(&lsmc->sk); in smc_listen_out()
1852 lsmc->sk.sk_data_ready(&lsmc->sk); in smc_listen_out()
1853 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */ in smc_listen_out()
1859 struct sock *newsmcsk = &new_smc->sk; in smc_listen_out_connected()
1861 if (newsmcsk->sk_state == SMC_INIT) in smc_listen_out_connected()
1862 newsmcsk->sk_state = SMC_ACTIVE; in smc_listen_out_connected()
1870 struct sock *newsmcsk = &new_smc->sk; in smc_listen_out_err()
1873 this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt); in smc_listen_out_err()
1874 if (newsmcsk->sk_state == SMC_INIT) in smc_listen_out_err()
1875 sock_put(&new_smc->sk); /* passive closing */ in smc_listen_out_err()
1876 newsmcsk->sk_state = SMC_CLOSED; in smc_listen_out_err()
1911 ini->smc_type_v1 = pclc->hdr.typev1; in smc_listen_v2_check()
1912 ini->smc_type_v2 = pclc->hdr.typev2; in smc_listen_v2_check()
1913 ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0; in smc_listen_v2_check()
1914 ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0; in smc_listen_v2_check()
1915 if (pclc->hdr.version > SMC_V1) { in smc_listen_v2_check()
1916 if (smcd_indicated(ini->smc_type_v2)) in smc_listen_v2_check()
1917 ini->smcd_version |= SMC_V2; in smc_listen_v2_check()
1918 if (smcr_indicated(ini->smc_type_v2)) in smc_listen_v2_check()
1919 ini->smcr_version |= SMC_V2; in smc_listen_v2_check()
1921 if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) { in smc_listen_v2_check()
1927 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1928 ini->smcr_version &= ~SMC_V2; in smc_listen_v2_check()
1933 if (ini->smcd_version & SMC_V2) { in smc_listen_v2_check()
1935 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1938 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1940 } else if (!pclc_v2_ext->hdr.eid_cnt && in smc_listen_v2_check()
1941 !pclc_v2_ext->hdr.flag.seid) { in smc_listen_v2_check()
1942 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1946 if (ini->smcr_version & SMC_V2) { in smc_listen_v2_check()
1947 if (!pclc_v2_ext->hdr.eid_cnt) { in smc_listen_v2_check()
1948 ini->smcr_version &= ~SMC_V2; in smc_listen_v2_check()
1954 if (!ini->smcd_version && !ini->smcr_version) in smc_listen_v2_check()
1965 struct socket *newclcsock = new_smc->clcsock; in smc_listen_prfx_check()
1967 if (pclc->hdr.typev1 == SMC_TYPE_N) in smc_listen_prfx_check()
1994 /* listen worker: initialize connection and buffers for SMC-D */
2007 smc_conn_abort(new_smc, ini->first_contact_local); in smc_listen_ism_init()
2008 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : in smc_listen_ism_init()
2022 if (smcd == ini->ism_dev[i]) in smc_is_already_selected()
2036 if (smcd->going_away) in smc_check_ism_v2_match()
2042 ini->ism_peer_gid[*matches] = proposed_gid; in smc_check_ism_v2_match()
2043 ini->ism_dev[*matches] = smcd; in smc_check_ism_v2_match()
2052 if (!ini->rc) in smc_find_ism_store_rc()
2053 ini->rc = rc; in smc_find_ism_store_rc()
2068 if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2)) in smc_find_ism_v2_device_serv()
2076 if (pclc_smcd->ism.chid) in smc_find_ism_v2_device_serv()
2078 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid), in smc_find_ism_v2_device_serv()
2079 ntohll(pclc_smcd->ism.gid), &matches); in smc_find_ism_v2_device_serv()
2080 for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) { in smc_find_ism_v2_device_serv()
2081 /* check for ISM devices matching proposed non-native ISM in smc_find_ism_v2_device_serv()
2085 ntohs(smcd_v2_ext->gidchid[i - 1].chid), in smc_find_ism_v2_device_serv()
2086 ntohll(smcd_v2_ext->gidchid[i - 1].gid), in smc_find_ism_v2_device_serv()
2091 if (!ini->ism_dev[0]) { in smc_find_ism_v2_device_serv()
2097 if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, in smc_find_ism_v2_device_serv()
2098 smcd_v2_ext->system_eid, eid)) in smc_find_ism_v2_device_serv()
2101 /* separate - outside the smcd_dev_list.lock */ in smc_find_ism_v2_device_serv()
2102 smcd_version = ini->smcd_version; in smc_find_ism_v2_device_serv()
2104 ini->smcd_version = SMC_V2; in smc_find_ism_v2_device_serv()
2105 ini->is_smcd = true; in smc_find_ism_v2_device_serv()
2106 ini->ism_selected = i; in smc_find_ism_v2_device_serv()
2116 ini->smcd_version = smcd_version; /* restore original value */ in smc_find_ism_v2_device_serv()
2117 ini->negotiated_eid[0] = 0; in smc_find_ism_v2_device_serv()
2120 ini->smcd_version &= ~SMC_V2; in smc_find_ism_v2_device_serv()
2121 ini->ism_dev[0] = NULL; in smc_find_ism_v2_device_serv()
2122 ini->is_smcd = false; in smc_find_ism_v2_device_serv()
2133 if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1)) in smc_find_ism_v1_device_serv()
2135 ini->is_smcd = true; /* prepare ISM check */ in smc_find_ism_v1_device_serv()
2136 ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid); in smc_find_ism_v1_device_serv()
2140 ini->ism_selected = 0; in smc_find_ism_v1_device_serv()
2147 ini->smcd_version &= ~SMC_V1; in smc_find_ism_v1_device_serv()
2148 ini->ism_dev[0] = NULL; in smc_find_ism_v1_device_serv()
2149 ini->is_smcd = false; in smc_find_ism_v1_device_serv()
2155 struct smc_connection *conn = &new_smc->conn; in smc_listen_rdma_reg()
2159 if (conn->sndbuf_desc->is_vm) { in smc_listen_rdma_reg()
2160 if (smcr_lgr_reg_sndbufs(conn->lnk, in smc_listen_rdma_reg()
2161 conn->sndbuf_desc)) in smc_listen_rdma_reg()
2164 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc)) in smc_listen_rdma_reg()
2179 if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2)) in smc_find_rdma_v2_device_serv()
2183 if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL)) in smc_find_rdma_v2_device_serv()
2187 memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN); in smc_find_rdma_v2_device_serv()
2188 memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE); in smc_find_rdma_v2_device_serv()
2189 memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN); in smc_find_rdma_v2_device_serv()
2190 ini->check_smcrv2 = true; in smc_find_rdma_v2_device_serv()
2191 ini->smcrv2.clc_sk = new_smc->clcsock->sk; in smc_find_rdma_v2_device_serv()
2192 ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr; in smc_find_rdma_v2_device_serv()
2193 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce); in smc_find_rdma_v2_device_serv()
2199 if (!ini->smcrv2.uses_gateway) in smc_find_rdma_v2_device_serv()
2200 memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN); in smc_find_rdma_v2_device_serv()
2202 smcr_version = ini->smcr_version; in smc_find_rdma_v2_device_serv()
2203 ini->smcr_version = SMC_V2; in smc_find_rdma_v2_device_serv()
2206 rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local); in smc_find_rdma_v2_device_serv()
2209 ini->smcr_version = smcr_version; in smc_find_rdma_v2_device_serv()
2213 ini->smcr_version &= ~SMC_V2; in smc_find_rdma_v2_device_serv()
2214 ini->smcrv2.ib_dev_v2 = NULL; in smc_find_rdma_v2_device_serv()
2215 ini->check_smcrv2 = false; in smc_find_rdma_v2_device_serv()
2224 if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1)) in smc_find_rdma_v1_device_serv()
2228 memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN); in smc_find_rdma_v1_device_serv()
2229 memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE); in smc_find_rdma_v1_device_serv()
2230 memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN); in smc_find_rdma_v1_device_serv()
2239 return smc_listen_rdma_reg(new_smc, ini->first_contact_local); in smc_find_rdma_v1_device_serv()
2251 if (ini->ism_dev[0]) in smc_listen_find_device()
2259 /* get vlan id from IP device */ in smc_listen_find_device()
2260 if (smc_vlan_by_tcpsk(new_smc->clcsock, ini)) in smc_listen_find_device()
2261 return ini->rc ?: SMC_CLC_DECL_GETVLANERR; in smc_listen_find_device()
2266 if (ini->ism_dev[0]) in smc_listen_find_device()
2269 if (!smcr_indicated(pclc->hdr.typev1) && in smc_listen_find_device()
2270 !smcr_indicated(pclc->hdr.typev2)) in smc_listen_find_device()
2272 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV; in smc_listen_find_device()
2276 if (ini->smcrv2.ib_dev_v2) in smc_listen_find_device()
2285 return (!rc) ? 0 : ini->rc; in smc_listen_find_device()
2296 struct smc_link *link = new_smc->conn.lnk; in smc_listen_rdma_finish()
2302 if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc)) in smc_listen_rdma_finish()
2309 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK); in smc_listen_rdma_finish()
2311 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl); in smc_listen_rdma_finish()
2321 struct socket *newclcsock = new_smc->clcsock; in smc_listen_work()
2330 if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN) in smc_listen_work()
2333 if (new_smc->use_fallback) { in smc_listen_work()
2338 /* check if peer is smc capable */ in smc_listen_work()
2339 if (!tcp_sk(newclcsock->sk)->syn_smc) { in smc_listen_work()
2348 /* do inband token exchange - in smc_listen_work()
2349 * wait for and receive SMC Proposal CLC message in smc_listen_work()
2362 if (pclc->hdr.version > SMC_V1) in smc_listen_work()
2365 /* IPSec connections opt out of SMC optimizations */ in smc_listen_work()
2392 /* send SMC Accept CLC message */ in smc_listen_work()
2393 accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version; in smc_listen_work()
2394 rc = smc_clc_send_accept(new_smc, ini->first_contact_local, in smc_listen_work()
2395 accept_version, ini->negotiated_eid); in smc_listen_work()
2399 /* SMC-D does not need this lock any more */ in smc_listen_work()
2400 if (ini->is_smcd) in smc_listen_work()
2403 /* receive SMC Confirm CLC message */ in smc_listen_work()
2409 if (!ini->is_smcd) in smc_listen_work()
2415 if (!ini->is_smcd) { in smc_listen_work()
2417 ini->first_contact_local, ini); in smc_listen_work()
2424 SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini); in smc_listen_work()
2430 smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0, in smc_listen_work()
2441 struct sock *lsk = &lsmc->sk; in smc_tcp_listen_work()
2446 while (lsk->sk_state == SMC_LISTEN) { in smc_tcp_listen_work()
2453 if (tcp_sk(new_smc->clcsock->sk)->syn_smc) in smc_tcp_listen_work()
2454 atomic_inc(&lsmc->queued_smc_hs); in smc_tcp_listen_work()
2456 new_smc->listen_smc = lsmc; in smc_tcp_listen_work()
2457 new_smc->use_fallback = lsmc->use_fallback; in smc_tcp_listen_work()
2458 new_smc->fallback_rsn = lsmc->fallback_rsn; in smc_tcp_listen_work()
2460 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work); in smc_tcp_listen_work()
2462 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf; in smc_tcp_listen_work()
2463 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf; in smc_tcp_listen_work()
2464 sock_hold(&new_smc->sk); /* sock_put in passive closing */ in smc_tcp_listen_work()
2465 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work)) in smc_tcp_listen_work()
2466 sock_put(&new_smc->sk); in smc_tcp_listen_work()
2471 sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */ in smc_tcp_listen_work()
2478 read_lock_bh(&listen_clcsock->sk_callback_lock); in smc_clcsock_data_ready()
2482 lsmc->clcsk_data_ready(listen_clcsock); in smc_clcsock_data_ready()
2483 if (lsmc->sk.sk_state == SMC_LISTEN) { in smc_clcsock_data_ready()
2484 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */ in smc_clcsock_data_ready()
2485 if (!queue_work(smc_tcp_ls_wq, &lsmc->tcp_listen_work)) in smc_clcsock_data_ready()
2486 sock_put(&lsmc->sk); in smc_clcsock_data_ready()
2489 read_unlock_bh(&listen_clcsock->sk_callback_lock); in smc_clcsock_data_ready()
2494 struct sock *sk = sock->sk; in smc_listen()
2495 struct smc_sock *smc; in smc_listen() local
2498 smc = smc_sk(sk); in smc_listen()
2501 rc = -EINVAL; in smc_listen()
2502 if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) || in smc_listen()
2503 smc->connect_nonblock || sock->state != SS_UNCONNECTED) in smc_listen()
2507 if (sk->sk_state == SMC_LISTEN) { in smc_listen()
2508 sk->sk_max_ack_backlog = backlog; in smc_listen()
2512 * them to the clc socket -- copy smc socket options to clc socket in smc_listen()
2514 smc_copy_sock_settings_to_clc(smc); in smc_listen()
2515 if (!smc->use_fallback) in smc_listen()
2516 tcp_sk(smc->clcsock->sk)->syn_smc = 1; in smc_listen()
2519 * smc-specific sk_data_ready function in smc_listen()
2521 write_lock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2522 smc->clcsock->sk->sk_user_data = in smc_listen()
2523 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); in smc_listen()
2524 smc_clcsock_replace_cb(&smc->clcsock->sk->sk_data_ready, in smc_listen()
2525 smc_clcsock_data_ready, &smc->clcsk_data_ready); in smc_listen()
2526 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2529 smc->ori_af_ops = inet_csk(smc->clcsock->sk)->icsk_af_ops; in smc_listen()
2531 smc->af_ops = *smc->ori_af_ops; in smc_listen()
2532 smc->af_ops.syn_recv_sock = smc_tcp_syn_recv_sock; in smc_listen()
2534 inet_csk(smc->clcsock->sk)->icsk_af_ops = &smc->af_ops; in smc_listen()
2536 if (smc->limit_smc_hs) in smc_listen()
2537 tcp_sk(smc->clcsock->sk)->smc_hs_congested = smc_hs_congested; in smc_listen()
2539 rc = kernel_listen(smc->clcsock, backlog); in smc_listen()
2541 write_lock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2542 smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, in smc_listen()
2543 &smc->clcsk_data_ready); in smc_listen()
2544 smc->clcsock->sk->sk_user_data = NULL; in smc_listen()
2545 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2548 sk->sk_max_ack_backlog = backlog; in smc_listen()
2549 sk->sk_ack_backlog = 0; in smc_listen()
2550 sk->sk_state = SMC_LISTEN; in smc_listen()
2560 struct sock *sk = sock->sk, *nsk; in smc_accept()
2570 if (lsmc->sk.sk_state != SMC_LISTEN) { in smc_accept()
2571 rc = -EINVAL; in smc_accept()
2582 rc = -EAGAIN; in smc_accept()
2604 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) { in smc_accept()
2606 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept * in smc_accept()
2608 if (smc_sk(nsk)->use_fallback) { in smc_accept()
2609 struct sock *clcsk = smc_sk(nsk)->clcsock->sk; in smc_accept()
2612 if (skb_queue_empty(&clcsk->sk_receive_queue)) in smc_accept()
2615 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) { in smc_accept()
2630 struct smc_sock *smc; in smc_getname() local
2632 if (peer && (sock->sk->sk_state != SMC_ACTIVE) && in smc_getname()
2633 (sock->sk->sk_state != SMC_APPCLOSEWAIT1)) in smc_getname()
2634 return -ENOTCONN; in smc_getname()
2636 smc = smc_sk(sock->sk); in smc_getname()
2638 return smc->clcsock->ops->getname(smc->clcsock, addr, peer); in smc_getname()
2643 struct sock *sk = sock->sk; in smc_sendmsg()
2644 struct smc_sock *smc; in smc_sendmsg() local
2645 int rc = -EPIPE; in smc_sendmsg()
2647 smc = smc_sk(sk); in smc_sendmsg()
2649 if ((sk->sk_state != SMC_ACTIVE) && in smc_sendmsg()
2650 (sk->sk_state != SMC_APPCLOSEWAIT1) && in smc_sendmsg()
2651 (sk->sk_state != SMC_INIT)) in smc_sendmsg()
2654 if (msg->msg_flags & MSG_FASTOPEN) { in smc_sendmsg()
2655 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { in smc_sendmsg()
2656 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); in smc_sendmsg()
2660 rc = -EINVAL; in smc_sendmsg()
2665 if (smc->use_fallback) { in smc_sendmsg()
2666 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len); in smc_sendmsg()
2668 rc = smc_tx_sendmsg(smc, msg, len); in smc_sendmsg()
2669 SMC_STAT_TX_PAYLOAD(smc, len, rc); in smc_sendmsg()
2679 struct sock *sk = sock->sk; in smc_recvmsg()
2680 struct smc_sock *smc; in smc_recvmsg() local
2681 int rc = -ENOTCONN; in smc_recvmsg()
2683 smc = smc_sk(sk); in smc_recvmsg()
2685 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) { in smc_recvmsg()
2690 if ((sk->sk_state == SMC_INIT) || in smc_recvmsg()
2691 (sk->sk_state == SMC_LISTEN) || in smc_recvmsg()
2692 (sk->sk_state == SMC_CLOSED)) in smc_recvmsg()
2695 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { in smc_recvmsg()
2700 if (smc->use_fallback) { in smc_recvmsg()
2701 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags); in smc_recvmsg()
2703 msg->msg_namelen = 0; in smc_recvmsg()
2704 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags); in smc_recvmsg()
2705 SMC_STAT_RX_PAYLOAD(smc, rc, rc); in smc_recvmsg()
2718 spin_lock(&isk->accept_q_lock); in smc_accept_poll()
2719 if (!list_empty(&isk->accept_q)) in smc_accept_poll()
2721 spin_unlock(&isk->accept_q_lock); in smc_accept_poll()
2729 struct sock *sk = sock->sk; in smc_poll()
2730 struct smc_sock *smc; in smc_poll() local
2736 smc = smc_sk(sock->sk); in smc_poll()
2737 if (smc->use_fallback) { in smc_poll()
2739 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait); in smc_poll()
2740 sk->sk_err = smc->clcsock->sk->sk_err; in smc_poll()
2742 if (sk->sk_state != SMC_CLOSED) in smc_poll()
2744 if (sk->sk_err) in smc_poll()
2746 if ((sk->sk_shutdown == SHUTDOWN_MASK) || in smc_poll()
2747 (sk->sk_state == SMC_CLOSED)) in smc_poll()
2749 if (sk->sk_state == SMC_LISTEN) { in smc_poll()
2752 } else if (smc->use_fallback) { /* as result of connect_work()*/ in smc_poll()
2753 mask |= smc->clcsock->ops->poll(file, smc->clcsock, in smc_poll()
2755 sk->sk_err = smc->clcsock->sk->sk_err; in smc_poll()
2757 if ((sk->sk_state != SMC_INIT && in smc_poll()
2758 atomic_read(&smc->conn.sndbuf_space)) || in smc_poll()
2759 sk->sk_shutdown & SEND_SHUTDOWN) { in smc_poll()
2763 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in smc_poll()
2765 if (atomic_read(&smc->conn.bytes_to_rcv)) in smc_poll()
2767 if (sk->sk_shutdown & RCV_SHUTDOWN) in smc_poll()
2769 if (sk->sk_state == SMC_APPCLOSEWAIT1) in smc_poll()
2771 if (smc->conn.urg_state == SMC_URG_VALID) in smc_poll()
2781 struct sock *sk = sock->sk; in smc_shutdown()
2783 struct smc_sock *smc; in smc_shutdown() local
2784 int rc = -EINVAL; in smc_shutdown()
2788 smc = smc_sk(sk); in smc_shutdown()
2795 if (sock->state == SS_CONNECTING) { in smc_shutdown()
2796 if (sk->sk_state == SMC_ACTIVE) in smc_shutdown()
2797 sock->state = SS_CONNECTED; in smc_shutdown()
2798 else if (sk->sk_state == SMC_PEERCLOSEWAIT1 || in smc_shutdown()
2799 sk->sk_state == SMC_PEERCLOSEWAIT2 || in smc_shutdown()
2800 sk->sk_state == SMC_APPCLOSEWAIT1 || in smc_shutdown()
2801 sk->sk_state == SMC_APPCLOSEWAIT2 || in smc_shutdown()
2802 sk->sk_state == SMC_APPFINCLOSEWAIT) in smc_shutdown()
2803 sock->state = SS_DISCONNECTING; in smc_shutdown()
2806 rc = -ENOTCONN; in smc_shutdown()
2807 if ((sk->sk_state != SMC_ACTIVE) && in smc_shutdown()
2808 (sk->sk_state != SMC_PEERCLOSEWAIT1) && in smc_shutdown()
2809 (sk->sk_state != SMC_PEERCLOSEWAIT2) && in smc_shutdown()
2810 (sk->sk_state != SMC_APPCLOSEWAIT1) && in smc_shutdown()
2811 (sk->sk_state != SMC_APPCLOSEWAIT2) && in smc_shutdown()
2812 (sk->sk_state != SMC_APPFINCLOSEWAIT)) in smc_shutdown()
2814 if (smc->use_fallback) { in smc_shutdown()
2815 rc = kernel_sock_shutdown(smc->clcsock, how); in smc_shutdown()
2816 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown; in smc_shutdown()
2817 if (sk->sk_shutdown == SHUTDOWN_MASK) { in smc_shutdown()
2818 sk->sk_state = SMC_CLOSED; in smc_shutdown()
2819 sk->sk_socket->state = SS_UNCONNECTED; in smc_shutdown()
2826 old_state = sk->sk_state; in smc_shutdown()
2827 rc = smc_close_active(smc); in smc_shutdown()
2829 sk->sk_state == SMC_PEERCLOSEWAIT1) in smc_shutdown()
2833 rc = smc_close_shutdown_write(smc); in smc_shutdown()
2840 if (do_shutdown && smc->clcsock) in smc_shutdown()
2841 rc1 = kernel_sock_shutdown(smc->clcsock, how); in smc_shutdown()
2843 sk->sk_shutdown |= how + 1; in smc_shutdown()
2845 if (sk->sk_state == SMC_CLOSED) in smc_shutdown()
2846 sock->state = SS_UNCONNECTED; in smc_shutdown()
2848 sock->state = SS_DISCONNECTING; in smc_shutdown()
2857 struct smc_sock *smc; in __smc_getsockopt() local
2860 smc = smc_sk(sock->sk); in __smc_getsockopt()
2863 return -EFAULT; in __smc_getsockopt()
2868 return -EINVAL; in __smc_getsockopt()
2872 val = smc->limit_smc_hs; in __smc_getsockopt()
2875 return -EOPNOTSUPP; in __smc_getsockopt()
2879 return -EFAULT; in __smc_getsockopt()
2881 return -EFAULT; in __smc_getsockopt()
2889 struct sock *sk = sock->sk; in __smc_setsockopt()
2890 struct smc_sock *smc; in __smc_setsockopt() local
2893 smc = smc_sk(sk); in __smc_setsockopt()
2899 rc = -EINVAL; in __smc_setsockopt()
2903 rc = -EFAULT; in __smc_setsockopt()
2907 smc->limit_smc_hs = !!val; in __smc_setsockopt()
2911 rc = -EOPNOTSUPP; in __smc_setsockopt()
2922 struct sock *sk = sock->sk; in smc_setsockopt()
2923 struct smc_sock *smc; in smc_setsockopt() local
2927 return -EOPNOTSUPP; in smc_setsockopt()
2931 smc = smc_sk(sk); in smc_setsockopt()
2936 mutex_lock(&smc->clcsock_release_lock); in smc_setsockopt()
2937 if (!smc->clcsock) { in smc_setsockopt()
2938 mutex_unlock(&smc->clcsock_release_lock); in smc_setsockopt()
2939 return -EBADF; in smc_setsockopt()
2941 if (unlikely(!smc->clcsock->ops->setsockopt)) in smc_setsockopt()
2942 rc = -EOPNOTSUPP; in smc_setsockopt()
2944 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname, in smc_setsockopt()
2946 if (smc->clcsock->sk->sk_err) { in smc_setsockopt()
2947 sk->sk_err = smc->clcsock->sk->sk_err; in smc_setsockopt()
2950 mutex_unlock(&smc->clcsock_release_lock); in smc_setsockopt()
2953 return -EINVAL; in smc_setsockopt()
2955 return -EFAULT; in smc_setsockopt()
2958 if (rc || smc->use_fallback) in smc_setsockopt()
2965 /* option not supported by SMC */ in smc_setsockopt()
2966 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { in smc_setsockopt()
2967 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); in smc_setsockopt()
2969 rc = -EINVAL; in smc_setsockopt()
2973 if (sk->sk_state != SMC_INIT && in smc_setsockopt()
2974 sk->sk_state != SMC_LISTEN && in smc_setsockopt()
2975 sk->sk_state != SMC_CLOSED) { in smc_setsockopt()
2977 SMC_STAT_INC(smc, ndly_cnt); in smc_setsockopt()
2978 smc_tx_pending(&smc->conn); in smc_setsockopt()
2979 cancel_delayed_work(&smc->conn.tx_work); in smc_setsockopt()
2984 if (sk->sk_state != SMC_INIT && in smc_setsockopt()
2985 sk->sk_state != SMC_LISTEN && in smc_setsockopt()
2986 sk->sk_state != SMC_CLOSED) { in smc_setsockopt()
2988 SMC_STAT_INC(smc, cork_cnt); in smc_setsockopt()
2989 smc_tx_pending(&smc->conn); in smc_setsockopt()
2990 cancel_delayed_work(&smc->conn.tx_work); in smc_setsockopt()
2995 smc->sockopt_defer_accept = val; in smc_setsockopt()
3009 struct smc_sock *smc; in smc_getsockopt() local
3015 smc = smc_sk(sock->sk); in smc_getsockopt()
3016 mutex_lock(&smc->clcsock_release_lock); in smc_getsockopt()
3017 if (!smc->clcsock) { in smc_getsockopt()
3018 mutex_unlock(&smc->clcsock_release_lock); in smc_getsockopt()
3019 return -EBADF; in smc_getsockopt()
3022 if (unlikely(!smc->clcsock->ops->getsockopt)) { in smc_getsockopt()
3023 mutex_unlock(&smc->clcsock_release_lock); in smc_getsockopt()
3024 return -EOPNOTSUPP; in smc_getsockopt()
3026 rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname, in smc_getsockopt()
3028 mutex_unlock(&smc->clcsock_release_lock); in smc_getsockopt()
3037 struct smc_sock *smc; in smc_ioctl() local
3040 smc = smc_sk(sock->sk); in smc_ioctl()
3041 conn = &smc->conn; in smc_ioctl()
3042 lock_sock(&smc->sk); in smc_ioctl()
3043 if (smc->use_fallback) { in smc_ioctl()
3044 if (!smc->clcsock) { in smc_ioctl()
3045 release_sock(&smc->sk); in smc_ioctl()
3046 return -EBADF; in smc_ioctl()
3048 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg); in smc_ioctl()
3049 release_sock(&smc->sk); in smc_ioctl()
3054 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3055 release_sock(&smc->sk); in smc_ioctl()
3056 return -EINVAL; in smc_ioctl()
3058 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3059 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
3062 answ = atomic_read(&smc->conn.bytes_to_rcv); in smc_ioctl()
3066 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3067 release_sock(&smc->sk); in smc_ioctl()
3068 return -EINVAL; in smc_ioctl()
3070 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3071 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
3074 answ = smc->conn.sndbuf_desc->len - in smc_ioctl()
3075 atomic_read(&smc->conn.sndbuf_space); in smc_ioctl()
3079 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3080 release_sock(&smc->sk); in smc_ioctl()
3081 return -EINVAL; in smc_ioctl()
3083 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3084 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
3087 answ = smc_tx_prepared_sends(&smc->conn); in smc_ioctl()
3090 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3091 release_sock(&smc->sk); in smc_ioctl()
3092 return -EINVAL; in smc_ioctl()
3094 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3095 smc->sk.sk_state == SMC_CLOSED) { in smc_ioctl()
3098 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); in smc_ioctl()
3099 smc_curs_copy(&urg, &conn->urg_curs, conn); in smc_ioctl()
3100 answ = smc_curs_diff(conn->rmb_desc->len, in smc_ioctl()
3105 release_sock(&smc->sk); in smc_ioctl()
3106 return -ENOIOCTLCMD; in smc_ioctl()
3108 release_sock(&smc->sk); in smc_ioctl()
3116 struct sock *sk = sock->sk; in smc_sendpage()
3117 struct smc_sock *smc; in smc_sendpage() local
3118 int rc = -EPIPE; in smc_sendpage()
3120 smc = smc_sk(sk); in smc_sendpage()
3122 if (sk->sk_state != SMC_ACTIVE) { in smc_sendpage()
3127 if (smc->use_fallback) { in smc_sendpage()
3128 rc = kernel_sendpage(smc->clcsock, page, offset, in smc_sendpage()
3132 rc = smc_tx_sendpage(smc, page, offset, size, flags); in smc_sendpage()
3134 SMC_STAT_INC(smc, sendpage_cnt); in smc_sendpage()
3142 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
3151 struct sock *sk = sock->sk; in smc_splice_read()
3152 struct smc_sock *smc; in smc_splice_read() local
3153 int rc = -ENOTCONN; in smc_splice_read()
3155 smc = smc_sk(sk); in smc_splice_read()
3157 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) { in smc_splice_read()
3162 if (sk->sk_state == SMC_INIT || in smc_splice_read()
3163 sk->sk_state == SMC_LISTEN || in smc_splice_read()
3164 sk->sk_state == SMC_CLOSED) in smc_splice_read()
3167 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { in smc_splice_read()
3172 if (smc->use_fallback) { in smc_splice_read()
3173 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos, in smc_splice_read()
3177 rc = -ESPIPE; in smc_splice_read()
3184 SMC_STAT_INC(smc, splice_cnt); in smc_splice_read()
3185 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags); in smc_splice_read()
3220 struct smc_sock *smc; in __smc_create() local
3224 rc = -ESOCKTNOSUPPORT; in __smc_create()
3225 if (sock->type != SOCK_STREAM) in __smc_create()
3228 rc = -EPROTONOSUPPORT; in __smc_create()
3232 rc = -ENOBUFS; in __smc_create()
3233 sock->ops = &smc_sock_ops; in __smc_create()
3234 sock->state = SS_UNCONNECTED; in __smc_create()
3240 smc = smc_sk(sk); in __smc_create()
3241 smc->use_fallback = false; /* assume rdma capability first */ in __smc_create()
3242 smc->fallback_rsn = 0; in __smc_create()
3245 smc->limit_smc_hs = net->smc.limit_smc_hs; in __smc_create()
3250 &smc->clcsock); in __smc_create()
3256 smc->clcsock = clcsock; in __smc_create()
3277 struct socket *tcp = sk->sk_socket; in smc_ulp_init()
3283 if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP || in smc_ulp_init()
3284 (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)) in smc_ulp_init()
3285 return -ESOCKTNOSUPPORT; in smc_ulp_init()
3287 if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list) in smc_ulp_init()
3288 return -ENOTCONN; in smc_ulp_init()
3290 if (sk->sk_family == AF_INET) in smc_ulp_init()
3297 return -ENFILE; in smc_ulp_init()
3299 smcsock->type = SOCK_STREAM; in smc_ulp_init()
3307 /* replace tcp socket to smc */ in smc_ulp_init()
3308 smcsock->file = tcp->file; in smc_ulp_init()
3309 smcsock->file->private_data = smcsock; in smc_ulp_init()
3310 smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */ in smc_ulp_init()
3311 smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */ in smc_ulp_init()
3312 tcp->file = NULL; in smc_ulp_init()
3323 icsk->icsk_ulp_ops = NULL; in smc_ulp_clone()
3327 .name = "smc",
3364 .id = &smc_net_id,
3396 rc = -ENOMEM; in smc_init()
3515 MODULE_DESCRIPTION("smc socket address family");
3518 MODULE_ALIAS_TCP_ULP("smc");