Lines Matching +full:smc +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
11 * - support for alternate links postponed
19 #define KMSG_COMPONENT "smc"
33 #include <net/smc.h>
40 #include "smc.h"
69 struct smc_sock *smc = smc_sk(sk); in smc_set_keepalive() local
71 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); in smc_set_keepalive()
84 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; in smc_hash_sk()
87 head = &h->ht; in smc_hash_sk()
89 write_lock_bh(&h->lock); in smc_hash_sk()
91 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in smc_hash_sk()
92 write_unlock_bh(&h->lock); in smc_hash_sk()
100 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; in smc_unhash_sk()
102 write_lock_bh(&h->lock); in smc_unhash_sk()
104 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in smc_unhash_sk()
105 write_unlock_bh(&h->lock); in smc_unhash_sk()
110 .name = "SMC",
133 static void smc_restore_fallback_changes(struct smc_sock *smc) in smc_restore_fallback_changes() argument
135 if (smc->clcsock->file) { /* non-accepted sockets have no file yet */ in smc_restore_fallback_changes()
136 smc->clcsock->file->private_data = smc->sk.sk_socket; in smc_restore_fallback_changes()
137 smc->clcsock->file = NULL; in smc_restore_fallback_changes()
141 static int __smc_release(struct smc_sock *smc) in __smc_release() argument
143 struct sock *sk = &smc->sk; in __smc_release()
146 if (!smc->use_fallback) { in __smc_release()
147 rc = smc_close_active(smc); in __smc_release()
149 sk->sk_shutdown |= SHUTDOWN_MASK; in __smc_release()
151 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT) in __smc_release()
153 if (sk->sk_state == SMC_LISTEN) { in __smc_release()
155 rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR); in __smc_release()
157 sk->sk_state = SMC_CLOSED; in __smc_release()
158 sk->sk_state_change(sk); in __smc_release()
159 smc_restore_fallback_changes(smc); in __smc_release()
162 sk->sk_prot->unhash(sk); in __smc_release()
164 if (sk->sk_state == SMC_CLOSED) { in __smc_release()
165 if (smc->clcsock) { in __smc_release()
167 smc_clcsock_release(smc); in __smc_release()
170 if (!smc->use_fallback) in __smc_release()
171 smc_conn_free(&smc->conn); in __smc_release()
179 struct sock *sk = sock->sk; in smc_release()
180 struct smc_sock *smc; in smc_release() local
187 smc = smc_sk(sk); in smc_release()
189 /* cleanup for a dangling non-blocking connect */ in smc_release()
190 if (smc->connect_nonblock && sk->sk_state == SMC_INIT) in smc_release()
191 tcp_abort(smc->clcsock->sk, ECONNABORTED); in smc_release()
192 flush_work(&smc->connect_work); in smc_release()
194 if (sk->sk_state == SMC_LISTEN) in smc_release()
202 rc = __smc_release(smc); in smc_release()
206 sock->sk = NULL; in smc_release()
217 if (sk->sk_state != SMC_CLOSED) in smc_destruct()
228 struct smc_sock *smc; in smc_sock_alloc() local
238 sk->sk_state = SMC_INIT; in smc_sock_alloc()
239 sk->sk_destruct = smc_destruct; in smc_sock_alloc()
240 sk->sk_protocol = protocol; in smc_sock_alloc()
241 smc = smc_sk(sk); in smc_sock_alloc()
242 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); in smc_sock_alloc()
243 INIT_WORK(&smc->connect_work, smc_connect_work); in smc_sock_alloc()
244 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work); in smc_sock_alloc()
245 INIT_LIST_HEAD(&smc->accept_q); in smc_sock_alloc()
246 spin_lock_init(&smc->accept_q_lock); in smc_sock_alloc()
247 spin_lock_init(&smc->conn.send_lock); in smc_sock_alloc()
248 sk->sk_prot->hash(sk); in smc_sock_alloc()
250 mutex_init(&smc->clcsock_release_lock); in smc_sock_alloc()
259 struct sock *sk = sock->sk; in smc_bind()
260 struct smc_sock *smc; in smc_bind() local
263 smc = smc_sk(sk); in smc_bind()
266 rc = -EINVAL; in smc_bind()
270 rc = -EAFNOSUPPORT; in smc_bind()
271 if (addr->sin_family != AF_INET && in smc_bind()
272 addr->sin_family != AF_INET6 && in smc_bind()
273 addr->sin_family != AF_UNSPEC) in smc_bind()
276 if (addr->sin_family == AF_UNSPEC && in smc_bind()
277 addr->sin_addr.s_addr != htonl(INADDR_ANY)) in smc_bind()
283 rc = -EINVAL; in smc_bind()
284 if (sk->sk_state != SMC_INIT || smc->connect_nonblock) in smc_bind()
287 smc->clcsock->sk->sk_reuse = sk->sk_reuse; in smc_bind()
288 rc = kernel_bind(smc->clcsock, uaddr, addr_len); in smc_bind()
300 nsk->sk_type = osk->sk_type; in smc_copy_sock_settings()
301 nsk->sk_sndbuf = osk->sk_sndbuf; in smc_copy_sock_settings()
302 nsk->sk_rcvbuf = osk->sk_rcvbuf; in smc_copy_sock_settings()
303 nsk->sk_sndtimeo = osk->sk_sndtimeo; in smc_copy_sock_settings()
304 nsk->sk_rcvtimeo = osk->sk_rcvtimeo; in smc_copy_sock_settings()
305 nsk->sk_mark = osk->sk_mark; in smc_copy_sock_settings()
306 nsk->sk_priority = osk->sk_priority; in smc_copy_sock_settings()
307 nsk->sk_rcvlowat = osk->sk_rcvlowat; in smc_copy_sock_settings()
308 nsk->sk_bound_dev_if = osk->sk_bound_dev_if; in smc_copy_sock_settings()
309 nsk->sk_err = osk->sk_err; in smc_copy_sock_settings()
311 nsk->sk_flags &= ~mask; in smc_copy_sock_settings()
312 nsk->sk_flags |= osk->sk_flags & mask; in smc_copy_sock_settings()
330 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
331 * clc socket (since smc is not called for these options from net/core)
333 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) in smc_copy_sock_settings_to_clc() argument
335 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); in smc_copy_sock_settings_to_clc()
342 /* copy only settings and flags relevant for smc from clc to smc socket */
343 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) in smc_copy_sock_settings_to_smc() argument
345 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); in smc_copy_sock_settings_to_smc()
352 struct smc_link_group *lgr = link->lgr; in smcr_lgr_reg_rmbs()
361 mutex_lock(&lgr->llc_conf_mutex); in smcr_lgr_reg_rmbs()
363 if (!smc_link_active(&lgr->lnk[i])) in smcr_lgr_reg_rmbs()
365 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc); in smcr_lgr_reg_rmbs()
373 rc = -EFAULT; in smcr_lgr_reg_rmbs()
376 rmb_desc->is_conf_rkey = true; in smcr_lgr_reg_rmbs()
378 mutex_unlock(&lgr->llc_conf_mutex); in smcr_lgr_reg_rmbs()
379 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); in smcr_lgr_reg_rmbs()
383 static int smcr_clnt_conf_first_link(struct smc_sock *smc) in smcr_clnt_conf_first_link() argument
385 struct smc_link *link = smc->conn.lnk; in smcr_clnt_conf_first_link()
390 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME, in smcr_clnt_conf_first_link()
395 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_clnt_conf_first_link()
397 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc; in smcr_clnt_conf_first_link()
401 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl); in smcr_clnt_conf_first_link()
411 if (smcr_link_reg_rmb(link, smc->conn.rmb_desc)) in smcr_clnt_conf_first_link()
415 smc->conn.rmb_desc->is_conf_rkey = true; in smcr_clnt_conf_first_link()
423 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE); in smcr_clnt_conf_first_link()
426 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME, in smcr_clnt_conf_first_link()
431 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_clnt_conf_first_link()
433 if (rc == -EAGAIN) in smcr_clnt_conf_first_link()
437 smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl); in smcr_clnt_conf_first_link()
442 static void smcr_conn_save_peer_info(struct smc_sock *smc, in smcr_conn_save_peer_info() argument
445 int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size); in smcr_conn_save_peer_info()
447 smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx; in smcr_conn_save_peer_info()
448 smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token); in smcr_conn_save_peer_info()
449 smc->conn.peer_rmbe_size = bufsize; in smcr_conn_save_peer_info()
450 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); in smcr_conn_save_peer_info()
451 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1); in smcr_conn_save_peer_info()
464 static void smcd_conn_save_peer_info(struct smc_sock *smc, in smcd_conn_save_peer_info() argument
467 int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size); in smcd_conn_save_peer_info()
469 smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx; in smcd_conn_save_peer_info()
470 smc->conn.peer_token = clc->d0.token; in smcd_conn_save_peer_info()
472 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg); in smcd_conn_save_peer_info()
473 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); in smcd_conn_save_peer_info()
474 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx; in smcd_conn_save_peer_info()
475 if (clc->hdr.version > SMC_V1 && in smcd_conn_save_peer_info()
476 (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) { in smcd_conn_save_peer_info()
483 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid, in smcd_conn_save_peer_info()
485 smc->conn.lgr->peer_os = fce->os_type; in smcd_conn_save_peer_info()
486 smc->conn.lgr->peer_smc_release = fce->release; in smcd_conn_save_peer_info()
487 if (smc_isascii(fce->hostname)) in smcd_conn_save_peer_info()
488 memcpy(smc->conn.lgr->peer_hostname, fce->hostname, in smcd_conn_save_peer_info()
493 static void smc_conn_save_peer_info(struct smc_sock *smc, in smc_conn_save_peer_info() argument
496 if (smc->conn.lgr->is_smcd) in smc_conn_save_peer_info()
497 smcd_conn_save_peer_info(smc, clc); in smc_conn_save_peer_info()
499 smcr_conn_save_peer_info(smc, clc); in smc_conn_save_peer_info()
505 link->peer_qpn = ntoh24(clc->r0.qpn); in smc_link_save_peer_info()
506 memcpy(link->peer_gid, clc->r0.lcl.gid, SMC_GID_SIZE); in smc_link_save_peer_info()
507 memcpy(link->peer_mac, clc->r0.lcl.mac, sizeof(link->peer_mac)); in smc_link_save_peer_info()
508 link->peer_psn = ntoh24(clc->r0.psn); in smc_link_save_peer_info()
509 link->peer_mtu = clc->r0.qp_mtu; in smc_link_save_peer_info()
512 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc, in smc_stat_inc_fback_rsn_cnt() argument
518 if (fback_arr[cnt].fback_code == smc->fallback_rsn) { in smc_stat_inc_fback_rsn_cnt()
523 fback_arr[cnt].fback_code = smc->fallback_rsn; in smc_stat_inc_fback_rsn_cnt()
530 static void smc_stat_fallback(struct smc_sock *smc) in smc_stat_fallback() argument
532 struct net *net = sock_net(&smc->sk); in smc_stat_fallback()
534 mutex_lock(&net->smc.mutex_fback_rsn); in smc_stat_fallback()
535 if (smc->listen_smc) { in smc_stat_fallback()
536 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv); in smc_stat_fallback()
537 net->smc.fback_rsn->srv_fback_cnt++; in smc_stat_fallback()
539 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt); in smc_stat_fallback()
540 net->smc.fback_rsn->clnt_fback_cnt++; in smc_stat_fallback()
542 mutex_unlock(&net->smc.mutex_fback_rsn); in smc_stat_fallback()
545 static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code) in smc_switch_to_fallback() argument
547 smc->use_fallback = true; in smc_switch_to_fallback()
548 smc->fallback_rsn = reason_code; in smc_switch_to_fallback()
549 smc_stat_fallback(smc); in smc_switch_to_fallback()
550 if (smc->sk.sk_socket && smc->sk.sk_socket->file) { in smc_switch_to_fallback()
551 smc->clcsock->file = smc->sk.sk_socket->file; in smc_switch_to_fallback()
552 smc->clcsock->file->private_data = smc->clcsock; in smc_switch_to_fallback()
553 smc->clcsock->wq.fasync_list = in smc_switch_to_fallback()
554 smc->sk.sk_socket->wq.fasync_list; in smc_switch_to_fallback()
559 static int smc_connect_fallback(struct smc_sock *smc, int reason_code) in smc_connect_fallback() argument
561 smc_switch_to_fallback(smc, reason_code); in smc_connect_fallback()
562 smc_copy_sock_settings_to_clc(smc); in smc_connect_fallback()
563 smc->connect_nonblock = 0; in smc_connect_fallback()
564 if (smc->sk.sk_state == SMC_INIT) in smc_connect_fallback()
565 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_fallback()
570 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code, in smc_connect_decline_fallback() argument
573 struct net *net = sock_net(&smc->sk); in smc_connect_decline_fallback()
577 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_decline_fallback()
578 if (smc->sk.sk_state == SMC_INIT) in smc_connect_decline_fallback()
579 sock_put(&smc->sk); /* passive closing */ in smc_connect_decline_fallback()
583 rc = smc_clc_send_decline(smc, reason_code, version); in smc_connect_decline_fallback()
585 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_decline_fallback()
586 if (smc->sk.sk_state == SMC_INIT) in smc_connect_decline_fallback()
587 sock_put(&smc->sk); /* passive closing */ in smc_connect_decline_fallback()
591 return smc_connect_fallback(smc, reason_code); in smc_connect_decline_fallback()
594 static void smc_conn_abort(struct smc_sock *smc, int local_first) in smc_conn_abort() argument
597 smc_lgr_cleanup_early(&smc->conn); in smc_conn_abort()
599 smc_conn_free(&smc->conn); in smc_conn_abort()
604 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini) in smc_find_rdma_device() argument
610 smc_pnet_find_roce_resource(smc->clcsock->sk, ini); in smc_find_rdma_device()
611 if (!ini->ib_dev) in smc_find_rdma_device()
618 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini) in smc_find_ism_device() argument
621 smc_pnet_find_ism_resource(smc->clcsock->sk, ini); in smc_find_ism_device()
622 if (!ini->ism_dev[0]) in smc_find_ism_device()
625 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]); in smc_find_ism_device()
633 int i = (!ini->ism_dev[0]) ? 1 : 0; in smc_find_ism_v2_is_unique_chid()
636 if (ini->ism_chid[i] == chid) in smc_find_ism_v2_is_unique_chid()
644 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc, in smc_find_ism_v2_device_clnt() argument
652 if (smcd_indicated(ini->smc_type_v1)) in smc_find_ism_v2_device_clnt()
656 if (smcd->going_away || smcd == ini->ism_dev[0]) in smc_find_ism_v2_device_clnt()
661 if (!smc_pnet_is_pnetid_set(smcd->pnetid) || in smc_find_ism_v2_device_clnt()
662 smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) { in smc_find_ism_v2_device_clnt()
663 ini->ism_dev[i] = smcd; in smc_find_ism_v2_device_clnt()
664 ini->ism_chid[i] = chid; in smc_find_ism_v2_device_clnt()
665 ini->is_smcd = true; in smc_find_ism_v2_device_clnt()
673 ini->ism_offered_cnt = i - 1; in smc_find_ism_v2_device_clnt()
674 if (!ini->ism_dev[0] && !ini->ism_dev[1]) in smc_find_ism_v2_device_clnt()
675 ini->smcd_version = 0; in smc_find_ism_v2_device_clnt()
680 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
681 static int smc_connect_ism_vlan_setup(struct smc_sock *smc, in smc_connect_ism_vlan_setup() argument
684 if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id)) in smc_connect_ism_vlan_setup()
689 static int smc_find_proposal_devices(struct smc_sock *smc, in smc_find_proposal_devices() argument
695 if (ini->smcd_version & SMC_V1) { in smc_find_proposal_devices()
696 if (smc_find_ism_device(smc, ini) || in smc_find_proposal_devices()
697 smc_connect_ism_vlan_setup(smc, ini)) { in smc_find_proposal_devices()
698 if (ini->smc_type_v1 == SMC_TYPE_B) in smc_find_proposal_devices()
699 ini->smc_type_v1 = SMC_TYPE_R; in smc_find_proposal_devices()
701 ini->smc_type_v1 = SMC_TYPE_N; in smc_find_proposal_devices()
703 if (smc_find_rdma_device(smc, ini)) { in smc_find_proposal_devices()
704 if (ini->smc_type_v1 == SMC_TYPE_B) in smc_find_proposal_devices()
705 ini->smc_type_v1 = SMC_TYPE_D; in smc_find_proposal_devices()
707 ini->smc_type_v1 = SMC_TYPE_N; in smc_find_proposal_devices()
710 if (smc_ism_is_v2_capable() && smc_find_ism_v2_device_clnt(smc, ini)) in smc_find_proposal_devices()
711 ini->smc_type_v2 = SMC_TYPE_N; in smc_find_proposal_devices()
714 if (!smcr_indicated(ini->smc_type_v1) && in smc_find_proposal_devices()
715 ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N) in smc_find_proposal_devices()
721 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
722 * used, the VLAN ID will be registered again during the connection setup.
724 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, in smc_connect_ism_vlan_cleanup() argument
727 if (!smcd_indicated(ini->smc_type_v1)) in smc_connect_ism_vlan_cleanup()
729 if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id)) in smc_connect_ism_vlan_cleanup()
740 static int smc_connect_clc(struct smc_sock *smc, in smc_connect_clc() argument
747 rc = smc_clc_send_proposal(smc, ini); in smc_connect_clc()
750 /* receive SMC Accept CLC message */ in smc_connect_clc()
751 return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN, in smc_connect_clc()
756 static int smc_connect_rdma(struct smc_sock *smc, in smc_connect_rdma() argument
763 ini->is_smcd = false; in smc_connect_rdma()
764 ini->ib_lcl = &aclc->r0.lcl; in smc_connect_rdma()
765 ini->ib_clcqpn = ntoh24(aclc->r0.qpn); in smc_connect_rdma()
766 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; in smc_connect_rdma()
769 reason_code = smc_conn_create(smc, ini); in smc_connect_rdma()
775 smc_conn_save_peer_info(smc, aclc); in smc_connect_rdma()
777 if (ini->first_contact_local) { in smc_connect_rdma()
778 link = smc->conn.lnk; in smc_connect_rdma()
783 struct smc_link *l = &smc->conn.lgr->lnk[i]; in smc_connect_rdma()
785 if (l->peer_qpn == ntoh24(aclc->r0.qpn) && in smc_connect_rdma()
786 !memcmp(l->peer_gid, &aclc->r0.lcl.gid, in smc_connect_rdma()
788 !memcmp(l->peer_mac, &aclc->r0.lcl.mac, in smc_connect_rdma()
789 sizeof(l->peer_mac))) { in smc_connect_rdma()
798 smc_switch_link_and_count(&smc->conn, link); in smc_connect_rdma()
802 if (smc_buf_create(smc, false)) { in smc_connect_rdma()
807 if (ini->first_contact_local) in smc_connect_rdma()
810 if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) { in smc_connect_rdma()
815 smc_close_init(smc); in smc_connect_rdma()
816 smc_rx_init(smc); in smc_connect_rdma()
818 if (ini->first_contact_local) { in smc_connect_rdma()
824 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) { in smc_connect_rdma()
829 smc_rmb_sync_sg_for_device(&smc->conn); in smc_connect_rdma()
831 reason_code = smc_clc_send_confirm(smc, ini->first_contact_local, in smc_connect_rdma()
836 smc_tx_init(smc); in smc_connect_rdma()
838 if (ini->first_contact_local) { in smc_connect_rdma()
840 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK); in smc_connect_rdma()
841 reason_code = smcr_clnt_conf_first_link(smc); in smc_connect_rdma()
842 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl); in smc_connect_rdma()
848 smc_copy_sock_settings_to_clc(smc); in smc_connect_rdma()
849 smc->connect_nonblock = 0; in smc_connect_rdma()
850 if (smc->sk.sk_state == SMC_INIT) in smc_connect_rdma()
851 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_rdma()
855 smc_conn_abort(smc, ini->first_contact_local); in smc_connect_rdma()
857 smc->connect_nonblock = 0; in smc_connect_rdma()
871 for (i = 0; i < ini->ism_offered_cnt + 1; i++) { in smc_v2_determine_accepted_chid()
872 if (ini->ism_chid[i] == ntohs(aclc->chid)) { in smc_v2_determine_accepted_chid()
873 ini->ism_selected = i; in smc_v2_determine_accepted_chid()
878 return -EPROTO; in smc_v2_determine_accepted_chid()
882 static int smc_connect_ism(struct smc_sock *smc, in smc_connect_ism() argument
888 ini->is_smcd = true; in smc_connect_ism()
889 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; in smc_connect_ism()
891 if (aclc->hdr.version == SMC_V2) { in smc_connect_ism()
899 ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid; in smc_connect_ism()
901 /* there is only one lgr role for SMC-D; use server lock */ in smc_connect_ism()
903 rc = smc_conn_create(smc, ini); in smc_connect_ism()
910 rc = smc_buf_create(smc, true); in smc_connect_ism()
912 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM; in smc_connect_ism()
916 smc_conn_save_peer_info(smc, aclc); in smc_connect_ism()
917 smc_close_init(smc); in smc_connect_ism()
918 smc_rx_init(smc); in smc_connect_ism()
919 smc_tx_init(smc); in smc_connect_ism()
921 rc = smc_clc_send_confirm(smc, ini->first_contact_local, in smc_connect_ism()
922 aclc->hdr.version); in smc_connect_ism()
927 smc_copy_sock_settings_to_clc(smc); in smc_connect_ism()
928 smc->connect_nonblock = 0; in smc_connect_ism()
929 if (smc->sk.sk_state == SMC_INIT) in smc_connect_ism()
930 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_ism()
934 smc_conn_abort(smc, ini->first_contact_local); in smc_connect_ism()
936 smc->connect_nonblock = 0; in smc_connect_ism()
945 if ((aclc->hdr.typev1 == SMC_TYPE_R && in smc_connect_check_aclc()
946 !smcr_indicated(ini->smc_type_v1)) || in smc_connect_check_aclc()
947 (aclc->hdr.typev1 == SMC_TYPE_D && in smc_connect_check_aclc()
948 ((!smcd_indicated(ini->smc_type_v1) && in smc_connect_check_aclc()
949 !smcd_indicated(ini->smc_type_v2)) || in smc_connect_check_aclc()
950 (aclc->hdr.version == SMC_V1 && in smc_connect_check_aclc()
951 !smcd_indicated(ini->smc_type_v1)) || in smc_connect_check_aclc()
952 (aclc->hdr.version == SMC_V2 && in smc_connect_check_aclc()
953 !smcd_indicated(ini->smc_type_v2))))) in smc_connect_check_aclc()
960 static int __smc_connect(struct smc_sock *smc) in __smc_connect() argument
969 if (smc->use_fallback) in __smc_connect()
970 return smc_connect_fallback(smc, smc->fallback_rsn); in __smc_connect()
972 /* if peer has not signalled SMC-capability, fall back */ in __smc_connect()
973 if (!tcp_sk(smc->clcsock->sk)->syn_smc) in __smc_connect()
974 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC); in __smc_connect()
976 /* IPSec connections opt out of SMC optimizations */ in __smc_connect()
977 if (using_ipsec(smc)) in __smc_connect()
978 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC, in __smc_connect()
983 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM, in __smc_connect()
986 ini->smcd_version = SMC_V1; in __smc_connect()
987 ini->smcd_version |= smc_ism_is_v2_capable() ? SMC_V2 : 0; in __smc_connect()
988 ini->smc_type_v1 = SMC_TYPE_B; in __smc_connect()
989 ini->smc_type_v2 = smc_ism_is_v2_capable() ? SMC_TYPE_D : SMC_TYPE_N; in __smc_connect()
991 /* get vlan id from IP device */ in __smc_connect()
992 if (smc_vlan_by_tcpsk(smc->clcsock, ini)) { in __smc_connect()
993 ini->smcd_version &= ~SMC_V1; in __smc_connect()
994 ini->smc_type_v1 = SMC_TYPE_N; in __smc_connect()
995 if (!ini->smcd_version) { in __smc_connect()
1001 rc = smc_find_proposal_devices(smc, ini); in __smc_connect()
1014 rc = smc_connect_clc(smc, aclc2, ini); in __smc_connect()
1018 /* check if smc modes and versions of CLC proposal and accept match */ in __smc_connect()
1020 version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2; in __smc_connect()
1021 ini->smcd_version = version; in __smc_connect()
1026 if (aclc->hdr.typev1 == SMC_TYPE_R) in __smc_connect()
1027 rc = smc_connect_rdma(smc, aclc, ini); in __smc_connect()
1028 else if (aclc->hdr.typev1 == SMC_TYPE_D) in __smc_connect()
1029 rc = smc_connect_ism(smc, aclc, ini); in __smc_connect()
1033 SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc); in __smc_connect()
1034 smc_connect_ism_vlan_cleanup(smc, ini); in __smc_connect()
1040 smc_connect_ism_vlan_cleanup(smc, ini); in __smc_connect()
1044 return smc_connect_decline_fallback(smc, rc, version); in __smc_connect()
1049 struct smc_sock *smc = container_of(work, struct smc_sock, in smc_connect_work() local
1051 long timeo = smc->sk.sk_sndtimeo; in smc_connect_work()
1056 lock_sock(smc->clcsock->sk); in smc_connect_work()
1057 if (smc->clcsock->sk->sk_err) { in smc_connect_work()
1058 smc->sk.sk_err = smc->clcsock->sk->sk_err; in smc_connect_work()
1059 } else if ((1 << smc->clcsock->sk->sk_state) & in smc_connect_work()
1061 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo); in smc_connect_work()
1062 if ((rc == -EPIPE) && in smc_connect_work()
1063 ((1 << smc->clcsock->sk->sk_state) & in smc_connect_work()
1067 release_sock(smc->clcsock->sk); in smc_connect_work()
1068 lock_sock(&smc->sk); in smc_connect_work()
1069 if (rc != 0 || smc->sk.sk_err) { in smc_connect_work()
1070 smc->sk.sk_state = SMC_CLOSED; in smc_connect_work()
1071 if (rc == -EPIPE || rc == -EAGAIN) in smc_connect_work()
1072 smc->sk.sk_err = EPIPE; in smc_connect_work()
1074 smc->sk.sk_err = -sock_intr_errno(timeo); in smc_connect_work()
1075 sock_put(&smc->sk); /* passive closing */ in smc_connect_work()
1079 rc = __smc_connect(smc); in smc_connect_work()
1081 smc->sk.sk_err = -rc; in smc_connect_work()
1084 if (!sock_flag(&smc->sk, SOCK_DEAD)) { in smc_connect_work()
1085 if (smc->sk.sk_err) { in smc_connect_work()
1086 smc->sk.sk_state_change(&smc->sk); in smc_connect_work()
1088 smc->clcsock->sk->sk_write_space(smc->clcsock->sk); in smc_connect_work()
1089 smc->sk.sk_write_space(&smc->sk); in smc_connect_work()
1092 release_sock(&smc->sk); in smc_connect_work()
1098 struct sock *sk = sock->sk; in smc_connect()
1099 struct smc_sock *smc; in smc_connect() local
1100 int rc = -EINVAL; in smc_connect()
1102 smc = smc_sk(sk); in smc_connect()
1104 /* separate smc parameter checking to be safe */ in smc_connect()
1105 if (alen < sizeof(addr->sa_family)) in smc_connect()
1107 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) in smc_connect()
1111 switch (sk->sk_state) { in smc_connect()
1115 rc = -EISCONN; in smc_connect()
1121 smc_copy_sock_settings_to_clc(smc); in smc_connect()
1122 tcp_sk(smc->clcsock->sk)->syn_smc = 1; in smc_connect()
1123 if (smc->connect_nonblock) { in smc_connect()
1124 rc = -EALREADY; in smc_connect()
1127 rc = kernel_connect(smc->clcsock, addr, alen, flags); in smc_connect()
1128 if (rc && rc != -EINPROGRESS) in smc_connect()
1131 sock_hold(&smc->sk); /* sock put in passive closing */ in smc_connect()
1132 if (smc->use_fallback) in smc_connect()
1135 if (queue_work(smc_hs_wq, &smc->connect_work)) in smc_connect()
1136 smc->connect_nonblock = 1; in smc_connect()
1137 rc = -EINPROGRESS; in smc_connect()
1139 rc = __smc_connect(smc); in smc_connect()
1155 struct sock *lsk = &lsmc->sk; in smc_clcsock_accept()
1157 int rc = -EINVAL; in smc_clcsock_accept()
1160 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol); in smc_clcsock_accept()
1162 rc = -ENOMEM; in smc_clcsock_accept()
1163 lsk->sk_err = ENOMEM; in smc_clcsock_accept()
1170 mutex_lock(&lsmc->clcsock_release_lock); in smc_clcsock_accept()
1171 if (lsmc->clcsock) in smc_clcsock_accept()
1172 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK); in smc_clcsock_accept()
1173 mutex_unlock(&lsmc->clcsock_release_lock); in smc_clcsock_accept()
1175 if (rc < 0 && rc != -EAGAIN) in smc_clcsock_accept()
1176 lsk->sk_err = -rc; in smc_clcsock_accept()
1177 if (rc < 0 || lsk->sk_state == SMC_CLOSED) { in smc_clcsock_accept()
1178 new_sk->sk_prot->unhash(new_sk); in smc_clcsock_accept()
1181 new_sk->sk_state = SMC_CLOSED; in smc_clcsock_accept()
1188 /* new clcsock has inherited the smc listen-specific sk_data_ready in smc_clcsock_accept()
1191 new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready; in smc_clcsock_accept()
1192 (*new_smc)->clcsock = new_clcsock; in smc_clcsock_accept()
1205 spin_lock(&par->accept_q_lock); in smc_accept_enqueue()
1206 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q); in smc_accept_enqueue()
1207 spin_unlock(&par->accept_q_lock); in smc_accept_enqueue()
1214 struct smc_sock *par = smc_sk(sk)->listen_smc; in smc_accept_unlink()
1216 spin_lock(&par->accept_q_lock); in smc_accept_unlink()
1217 list_del_init(&smc_sk(sk)->accept_q); in smc_accept_unlink()
1218 spin_unlock(&par->accept_q_lock); in smc_accept_unlink()
1219 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk); in smc_accept_unlink()
1232 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) { in smc_accept_dequeue()
1236 if (new_sk->sk_state == SMC_CLOSED) { in smc_accept_dequeue()
1237 new_sk->sk_prot->unhash(new_sk); in smc_accept_dequeue()
1238 if (isk->clcsock) { in smc_accept_dequeue()
1239 sock_release(isk->clcsock); in smc_accept_dequeue()
1240 isk->clcsock = NULL; in smc_accept_dequeue()
1247 if (isk->use_fallback) { in smc_accept_dequeue()
1248 smc_sk(new_sk)->clcsock->file = new_sock->file; in smc_accept_dequeue()
1249 isk->clcsock->file->private_data = isk->clcsock; in smc_accept_dequeue()
1260 struct smc_sock *smc = smc_sk(sk); in smc_close_non_accepted() local
1264 if (!sk->sk_lingertime) in smc_close_non_accepted()
1266 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT; in smc_close_non_accepted()
1267 __smc_release(smc); in smc_close_non_accepted()
1273 static int smcr_serv_conf_first_link(struct smc_sock *smc) in smcr_serv_conf_first_link() argument
1275 struct smc_link *link = smc->conn.lnk; in smcr_serv_conf_first_link()
1279 if (smcr_link_reg_rmb(link, smc->conn.rmb_desc)) in smcr_serv_conf_first_link()
1288 qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME, in smcr_serv_conf_first_link()
1293 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_serv_conf_first_link()
1295 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc; in smcr_serv_conf_first_link()
1299 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl); in smcr_serv_conf_first_link()
1304 smc->conn.rmb_desc->is_conf_rkey = true; in smcr_serv_conf_first_link()
1307 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE); in smcr_serv_conf_first_link()
1309 /* initial contact - try to establish second link */ in smcr_serv_conf_first_link()
1317 struct smc_sock *lsmc = new_smc->listen_smc; in smc_listen_out()
1318 struct sock *newsmcsk = &new_smc->sk; in smc_listen_out()
1320 if (lsmc->sk.sk_state == SMC_LISTEN) { in smc_listen_out()
1321 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); in smc_listen_out()
1322 smc_accept_enqueue(&lsmc->sk, newsmcsk); in smc_listen_out()
1323 release_sock(&lsmc->sk); in smc_listen_out()
1329 lsmc->sk.sk_data_ready(&lsmc->sk); in smc_listen_out()
1330 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */ in smc_listen_out()
1336 struct sock *newsmcsk = &new_smc->sk; in smc_listen_out_connected()
1339 if (newsmcsk->sk_state == SMC_INIT) in smc_listen_out_connected()
1340 newsmcsk->sk_state = SMC_ACTIVE; in smc_listen_out_connected()
1348 struct sock *newsmcsk = &new_smc->sk; in smc_listen_out_err()
1351 this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt); in smc_listen_out_err()
1352 if (newsmcsk->sk_state == SMC_INIT) in smc_listen_out_err()
1353 sock_put(&new_smc->sk); /* passive closing */ in smc_listen_out_err()
1354 newsmcsk->sk_state = SMC_CLOSED; in smc_listen_out_err()
1388 ini->smc_type_v1 = pclc->hdr.typev1; in smc_listen_v2_check()
1389 ini->smc_type_v2 = pclc->hdr.typev2; in smc_listen_v2_check()
1390 ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0; in smc_listen_v2_check()
1391 if (pclc->hdr.version > SMC_V1) in smc_listen_v2_check()
1392 ini->smcd_version |= in smc_listen_v2_check()
1393 ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0; in smc_listen_v2_check()
1394 if (!(ini->smcd_version & SMC_V2)) { in smc_listen_v2_check()
1399 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1405 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1411 ini->smcd_version &= ~SMC_V2; in smc_listen_v2_check()
1416 if (!ini->smcd_version) in smc_listen_v2_check()
1427 struct socket *newclcsock = new_smc->clcsock; in smc_listen_prfx_check()
1429 if (pclc->hdr.typev1 == SMC_TYPE_N) in smc_listen_prfx_check()
1456 /* listen worker: initialize connection and buffers for SMC-D */
1469 smc_conn_abort(new_smc, ini->first_contact_local); in smc_listen_ism_init()
1470 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : in smc_listen_ism_init()
1484 if (smcd == ini->ism_dev[i]) in smc_is_already_selected()
1498 if (smcd->going_away) in smc_check_ism_v2_match()
1504 ini->ism_peer_gid[*matches] = proposed_gid; in smc_check_ism_v2_match()
1505 ini->ism_dev[*matches] = smcd; in smc_check_ism_v2_match()
1514 if (!ini->rc) in smc_find_ism_store_rc()
1515 ini->rc = rc; in smc_find_ism_store_rc()
1530 if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2)) in smc_find_ism_v2_device_serv()
1537 !smc_v2_ext->hdr.flag.seid) { /* no system EID support for SMCD */ in smc_find_ism_v2_device_serv()
1543 if (pclc_smcd->ism.chid) in smc_find_ism_v2_device_serv()
1545 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid), in smc_find_ism_v2_device_serv()
1546 ntohll(pclc_smcd->ism.gid), &matches); in smc_find_ism_v2_device_serv()
1547 for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) { in smc_find_ism_v2_device_serv()
1548 /* check for ISM devices matching proposed non-native ISM in smc_find_ism_v2_device_serv()
1552 ntohs(smcd_v2_ext->gidchid[i - 1].chid), in smc_find_ism_v2_device_serv()
1553 ntohll(smcd_v2_ext->gidchid[i - 1].gid), in smc_find_ism_v2_device_serv()
1558 if (ini->ism_dev[0]) { in smc_find_ism_v2_device_serv()
1559 smc_ism_get_system_eid(ini->ism_dev[0], &eid); in smc_find_ism_v2_device_serv()
1560 if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN)) in smc_find_ism_v2_device_serv()
1566 /* separate - outside the smcd_dev_list.lock */ in smc_find_ism_v2_device_serv()
1567 smcd_version = ini->smcd_version; in smc_find_ism_v2_device_serv()
1569 ini->smcd_version = SMC_V2; in smc_find_ism_v2_device_serv()
1570 ini->is_smcd = true; in smc_find_ism_v2_device_serv()
1571 ini->ism_selected = i; in smc_find_ism_v2_device_serv()
1581 ini->smcd_version = smcd_version; /* restore original value */ in smc_find_ism_v2_device_serv()
1584 ini->smcd_version &= ~SMC_V2; in smc_find_ism_v2_device_serv()
1585 ini->ism_dev[0] = NULL; in smc_find_ism_v2_device_serv()
1586 ini->is_smcd = false; in smc_find_ism_v2_device_serv()
1597 if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1)) in smc_find_ism_v1_device_serv()
1599 ini->is_smcd = true; /* prepare ISM check */ in smc_find_ism_v1_device_serv()
1600 ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid); in smc_find_ism_v1_device_serv()
1604 ini->ism_selected = 0; in smc_find_ism_v1_device_serv()
1611 ini->ism_dev[0] = NULL; in smc_find_ism_v1_device_serv()
1612 ini->is_smcd = false; in smc_find_ism_v1_device_serv()
1618 struct smc_connection *conn = &new_smc->conn; in smc_listen_rdma_reg()
1621 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc)) in smc_listen_rdma_reg()
1624 smc_rmb_sync_sg_for_device(&new_smc->conn); in smc_listen_rdma_reg()
1635 if (!smcr_indicated(ini->smc_type_v1)) in smc_find_rdma_v1_device_serv()
1639 ini->ib_lcl = &pclc->lcl; in smc_find_rdma_v1_device_serv()
1643 if (ini->smc_type_v1 == SMC_TYPE_B) in smc_find_rdma_v1_device_serv()
1651 return smc_listen_rdma_reg(new_smc, ini->first_contact_local); in smc_find_rdma_v1_device_serv()
1663 if (ini->ism_dev[0]) in smc_listen_find_device()
1666 if (!(ini->smcd_version & SMC_V1)) in smc_listen_find_device()
1667 return ini->rc ?: SMC_CLC_DECL_NOSMCD2DEV; in smc_listen_find_device()
1672 return ini->rc ?: rc; in smc_listen_find_device()
1674 /* get vlan id from IP device */ in smc_listen_find_device()
1675 if (smc_vlan_by_tcpsk(new_smc->clcsock, ini)) in smc_listen_find_device()
1676 return ini->rc ?: SMC_CLC_DECL_GETVLANERR; in smc_listen_find_device()
1680 if (ini->ism_dev[0]) in smc_listen_find_device()
1683 if (pclc->hdr.typev1 == SMC_TYPE_D) in smc_listen_find_device()
1685 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV; in smc_listen_find_device()
1691 return (!rc) ? 0 : ini->rc; in smc_listen_find_device()
1699 struct smc_link *link = new_smc->conn.lnk; in smc_listen_rdma_finish()
1705 if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc)) in smc_listen_rdma_finish()
1712 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK); in smc_listen_rdma_finish()
1714 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl); in smc_listen_rdma_finish()
1725 struct socket *newclcsock = new_smc->clcsock; in smc_listen_work()
1732 if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN) in smc_listen_work()
1735 if (new_smc->use_fallback) { in smc_listen_work()
1740 /* check if peer is smc capable */ in smc_listen_work()
1741 if (!tcp_sk(newclcsock->sk)->syn_smc) { in smc_listen_work()
1747 /* do inband token exchange - in smc_listen_work()
1748 * wait for and receive SMC Proposal CLC message in smc_listen_work()
1760 version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version; in smc_listen_work()
1762 /* IPSec connections opt out of SMC optimizations */ in smc_listen_work()
1789 /* send SMC Accept CLC message */ in smc_listen_work()
1790 rc = smc_clc_send_accept(new_smc, ini->first_contact_local, in smc_listen_work()
1791 ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1); in smc_listen_work()
1795 /* SMC-D does not need this lock any more */ in smc_listen_work()
1796 if (ini->is_smcd) in smc_listen_work()
1799 /* receive SMC Confirm CLC message */ in smc_listen_work()
1805 if (!ini->is_smcd) in smc_listen_work()
1811 if (!ini->is_smcd) { in smc_listen_work()
1813 ini->first_contact_local); in smc_listen_work()
1820 SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini); in smc_listen_work()
1826 smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0, in smc_listen_work()
1837 struct sock *lsk = &lsmc->sk; in smc_tcp_listen_work()
1842 while (lsk->sk_state == SMC_LISTEN) { in smc_tcp_listen_work()
1849 new_smc->listen_smc = lsmc; in smc_tcp_listen_work()
1850 new_smc->use_fallback = lsmc->use_fallback; in smc_tcp_listen_work()
1851 new_smc->fallback_rsn = lsmc->fallback_rsn; in smc_tcp_listen_work()
1853 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work); in smc_tcp_listen_work()
1855 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf; in smc_tcp_listen_work()
1856 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf; in smc_tcp_listen_work()
1857 sock_hold(&new_smc->sk); /* sock_put in passive closing */ in smc_tcp_listen_work()
1858 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work)) in smc_tcp_listen_work()
1859 sock_put(&new_smc->sk); in smc_tcp_listen_work()
1864 sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */ in smc_tcp_listen_work()
1872 ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY); in smc_clcsock_data_ready()
1875 lsmc->clcsk_data_ready(listen_clcsock); in smc_clcsock_data_ready()
1876 if (lsmc->sk.sk_state == SMC_LISTEN) { in smc_clcsock_data_ready()
1877 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */ in smc_clcsock_data_ready()
1878 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work)) in smc_clcsock_data_ready()
1879 sock_put(&lsmc->sk); in smc_clcsock_data_ready()
1885 struct sock *sk = sock->sk; in smc_listen()
1886 struct smc_sock *smc; in smc_listen() local
1889 smc = smc_sk(sk); in smc_listen()
1892 rc = -EINVAL; in smc_listen()
1893 if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) || in smc_listen()
1894 smc->connect_nonblock) in smc_listen()
1898 if (sk->sk_state == SMC_LISTEN) { in smc_listen()
1899 sk->sk_max_ack_backlog = backlog; in smc_listen()
1903 * them to the clc socket -- copy smc socket options to clc socket in smc_listen()
1905 smc_copy_sock_settings_to_clc(smc); in smc_listen()
1906 if (!smc->use_fallback) in smc_listen()
1907 tcp_sk(smc->clcsock->sk)->syn_smc = 1; in smc_listen()
1910 * smc-specific sk_data_ready function in smc_listen()
1912 smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready; in smc_listen()
1913 smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready; in smc_listen()
1914 smc->clcsock->sk->sk_user_data = in smc_listen()
1915 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); in smc_listen()
1916 rc = kernel_listen(smc->clcsock, backlog); in smc_listen()
1919 sk->sk_max_ack_backlog = backlog; in smc_listen()
1920 sk->sk_ack_backlog = 0; in smc_listen()
1921 sk->sk_state = SMC_LISTEN; in smc_listen()
1931 struct sock *sk = sock->sk, *nsk; in smc_accept()
1941 if (lsmc->sk.sk_state != SMC_LISTEN) { in smc_accept()
1942 rc = -EINVAL; in smc_accept()
1953 rc = -EAGAIN; in smc_accept()
1975 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) { in smc_accept()
1977 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept * in smc_accept()
1979 if (smc_sk(nsk)->use_fallback) { in smc_accept()
1980 struct sock *clcsk = smc_sk(nsk)->clcsock->sk; in smc_accept()
1983 if (skb_queue_empty(&clcsk->sk_receive_queue)) in smc_accept()
1986 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) { in smc_accept()
2001 struct smc_sock *smc; in smc_getname() local
2003 if (peer && (sock->sk->sk_state != SMC_ACTIVE) && in smc_getname()
2004 (sock->sk->sk_state != SMC_APPCLOSEWAIT1)) in smc_getname()
2005 return -ENOTCONN; in smc_getname()
2007 smc = smc_sk(sock->sk); in smc_getname()
2009 return smc->clcsock->ops->getname(smc->clcsock, addr, peer); in smc_getname()
2014 struct sock *sk = sock->sk; in smc_sendmsg()
2015 struct smc_sock *smc; in smc_sendmsg() local
2016 int rc = -EPIPE; in smc_sendmsg()
2018 smc = smc_sk(sk); in smc_sendmsg()
2020 if ((sk->sk_state != SMC_ACTIVE) && in smc_sendmsg()
2021 (sk->sk_state != SMC_APPCLOSEWAIT1) && in smc_sendmsg()
2022 (sk->sk_state != SMC_INIT)) in smc_sendmsg()
2025 if (msg->msg_flags & MSG_FASTOPEN) { in smc_sendmsg()
2026 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { in smc_sendmsg()
2027 smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); in smc_sendmsg()
2029 rc = -EINVAL; in smc_sendmsg()
2034 if (smc->use_fallback) { in smc_sendmsg()
2035 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len); in smc_sendmsg()
2037 rc = smc_tx_sendmsg(smc, msg, len); in smc_sendmsg()
2038 SMC_STAT_TX_PAYLOAD(smc, len, rc); in smc_sendmsg()
2048 struct sock *sk = sock->sk; in smc_recvmsg()
2049 struct smc_sock *smc; in smc_recvmsg() local
2050 int rc = -ENOTCONN; in smc_recvmsg()
2052 smc = smc_sk(sk); in smc_recvmsg()
2054 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) { in smc_recvmsg()
2059 if ((sk->sk_state == SMC_INIT) || in smc_recvmsg()
2060 (sk->sk_state == SMC_LISTEN) || in smc_recvmsg()
2061 (sk->sk_state == SMC_CLOSED)) in smc_recvmsg()
2064 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { in smc_recvmsg()
2069 if (smc->use_fallback) { in smc_recvmsg()
2070 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags); in smc_recvmsg()
2072 msg->msg_namelen = 0; in smc_recvmsg()
2073 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags); in smc_recvmsg()
2074 SMC_STAT_RX_PAYLOAD(smc, rc, rc); in smc_recvmsg()
2087 spin_lock(&isk->accept_q_lock); in smc_accept_poll()
2088 if (!list_empty(&isk->accept_q)) in smc_accept_poll()
2090 spin_unlock(&isk->accept_q_lock); in smc_accept_poll()
2098 struct sock *sk = sock->sk; in smc_poll()
2099 struct smc_sock *smc; in smc_poll() local
2105 smc = smc_sk(sock->sk); in smc_poll()
2106 if (smc->use_fallback) { in smc_poll()
2108 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait); in smc_poll()
2109 sk->sk_err = smc->clcsock->sk->sk_err; in smc_poll()
2111 if (sk->sk_state != SMC_CLOSED) in smc_poll()
2113 if (sk->sk_err) in smc_poll()
2115 if ((sk->sk_shutdown == SHUTDOWN_MASK) || in smc_poll()
2116 (sk->sk_state == SMC_CLOSED)) in smc_poll()
2118 if (sk->sk_state == SMC_LISTEN) { in smc_poll()
2121 } else if (smc->use_fallback) { /* as result of connect_work()*/ in smc_poll()
2122 mask |= smc->clcsock->ops->poll(file, smc->clcsock, in smc_poll()
2124 sk->sk_err = smc->clcsock->sk->sk_err; in smc_poll()
2126 if ((sk->sk_state != SMC_INIT && in smc_poll()
2127 atomic_read(&smc->conn.sndbuf_space)) || in smc_poll()
2128 sk->sk_shutdown & SEND_SHUTDOWN) { in smc_poll()
2132 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in smc_poll()
2134 if (atomic_read(&smc->conn.bytes_to_rcv)) in smc_poll()
2136 if (sk->sk_shutdown & RCV_SHUTDOWN) in smc_poll()
2138 if (sk->sk_state == SMC_APPCLOSEWAIT1) in smc_poll()
2140 if (smc->conn.urg_state == SMC_URG_VALID) in smc_poll()
2150 struct sock *sk = sock->sk; in smc_shutdown()
2151 struct smc_sock *smc; in smc_shutdown() local
2152 int rc = -EINVAL; in smc_shutdown()
2155 smc = smc_sk(sk); in smc_shutdown()
2162 rc = -ENOTCONN; in smc_shutdown()
2163 if ((sk->sk_state != SMC_ACTIVE) && in smc_shutdown()
2164 (sk->sk_state != SMC_PEERCLOSEWAIT1) && in smc_shutdown()
2165 (sk->sk_state != SMC_PEERCLOSEWAIT2) && in smc_shutdown()
2166 (sk->sk_state != SMC_APPCLOSEWAIT1) && in smc_shutdown()
2167 (sk->sk_state != SMC_APPCLOSEWAIT2) && in smc_shutdown()
2168 (sk->sk_state != SMC_APPFINCLOSEWAIT)) in smc_shutdown()
2170 if (smc->use_fallback) { in smc_shutdown()
2171 rc = kernel_sock_shutdown(smc->clcsock, how); in smc_shutdown()
2172 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown; in smc_shutdown()
2173 if (sk->sk_shutdown == SHUTDOWN_MASK) in smc_shutdown()
2174 sk->sk_state = SMC_CLOSED; in smc_shutdown()
2179 rc = smc_close_active(smc); in smc_shutdown()
2182 rc = smc_close_shutdown_write(smc); in smc_shutdown()
2189 if (smc->clcsock) in smc_shutdown()
2190 rc1 = kernel_sock_shutdown(smc->clcsock, how); in smc_shutdown()
2192 sk->sk_shutdown |= how + 1; in smc_shutdown()
2202 struct sock *sk = sock->sk; in smc_setsockopt()
2203 struct smc_sock *smc; in smc_setsockopt() local
2207 return -EOPNOTSUPP; in smc_setsockopt()
2209 smc = smc_sk(sk); in smc_setsockopt()
2214 if (unlikely(!smc->clcsock->ops->setsockopt)) in smc_setsockopt()
2215 rc = -EOPNOTSUPP; in smc_setsockopt()
2217 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname, in smc_setsockopt()
2219 if (smc->clcsock->sk->sk_err) { in smc_setsockopt()
2220 sk->sk_err = smc->clcsock->sk->sk_err; in smc_setsockopt()
2225 return -EINVAL; in smc_setsockopt()
2227 return -EFAULT; in smc_setsockopt()
2230 if (rc || smc->use_fallback) in smc_setsockopt()
2237 /* option not supported by SMC */ in smc_setsockopt()
2238 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { in smc_setsockopt()
2239 smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); in smc_setsockopt()
2241 rc = -EINVAL; in smc_setsockopt()
2245 if (sk->sk_state != SMC_INIT && in smc_setsockopt()
2246 sk->sk_state != SMC_LISTEN && in smc_setsockopt()
2247 sk->sk_state != SMC_CLOSED) { in smc_setsockopt()
2249 SMC_STAT_INC(smc, ndly_cnt); in smc_setsockopt()
2250 mod_delayed_work(smc->conn.lgr->tx_wq, in smc_setsockopt()
2251 &smc->conn.tx_work, 0); in smc_setsockopt()
2256 if (sk->sk_state != SMC_INIT && in smc_setsockopt()
2257 sk->sk_state != SMC_LISTEN && in smc_setsockopt()
2258 sk->sk_state != SMC_CLOSED) { in smc_setsockopt()
2260 SMC_STAT_INC(smc, cork_cnt); in smc_setsockopt()
2261 mod_delayed_work(smc->conn.lgr->tx_wq, in smc_setsockopt()
2262 &smc->conn.tx_work, 0); in smc_setsockopt()
2267 smc->sockopt_defer_accept = val; in smc_setsockopt()
2281 struct smc_sock *smc; in smc_getsockopt() local
2283 smc = smc_sk(sock->sk); in smc_getsockopt()
2285 if (unlikely(!smc->clcsock->ops->getsockopt)) in smc_getsockopt()
2286 return -EOPNOTSUPP; in smc_getsockopt()
2287 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname, in smc_getsockopt()
2296 struct smc_sock *smc; in smc_ioctl() local
2299 smc = smc_sk(sock->sk); in smc_ioctl()
2300 conn = &smc->conn; in smc_ioctl()
2301 lock_sock(&smc->sk); in smc_ioctl()
2302 if (smc->use_fallback) { in smc_ioctl()
2303 if (!smc->clcsock) { in smc_ioctl()
2304 release_sock(&smc->sk); in smc_ioctl()
2305 return -EBADF; in smc_ioctl()
2307 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg); in smc_ioctl()
2308 release_sock(&smc->sk); in smc_ioctl()
2313 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
2314 release_sock(&smc->sk); in smc_ioctl()
2315 return -EINVAL; in smc_ioctl()
2317 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
2318 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
2321 answ = atomic_read(&smc->conn.bytes_to_rcv); in smc_ioctl()
2325 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
2326 release_sock(&smc->sk); in smc_ioctl()
2327 return -EINVAL; in smc_ioctl()
2329 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
2330 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
2333 answ = smc->conn.sndbuf_desc->len - in smc_ioctl()
2334 atomic_read(&smc->conn.sndbuf_space); in smc_ioctl()
2338 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
2339 release_sock(&smc->sk); in smc_ioctl()
2340 return -EINVAL; in smc_ioctl()
2342 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
2343 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
2346 answ = smc_tx_prepared_sends(&smc->conn); in smc_ioctl()
2349 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
2350 release_sock(&smc->sk); in smc_ioctl()
2351 return -EINVAL; in smc_ioctl()
2353 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
2354 smc->sk.sk_state == SMC_CLOSED) { in smc_ioctl()
2357 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); in smc_ioctl()
2358 smc_curs_copy(&urg, &conn->urg_curs, conn); in smc_ioctl()
2359 answ = smc_curs_diff(conn->rmb_desc->len, in smc_ioctl()
2364 release_sock(&smc->sk); in smc_ioctl()
2365 return -ENOIOCTLCMD; in smc_ioctl()
2367 release_sock(&smc->sk); in smc_ioctl()
2375 struct sock *sk = sock->sk; in smc_sendpage()
2376 struct smc_sock *smc; in smc_sendpage() local
2377 int rc = -EPIPE; in smc_sendpage()
2379 smc = smc_sk(sk); in smc_sendpage()
2381 if (sk->sk_state != SMC_ACTIVE) { in smc_sendpage()
2386 if (smc->use_fallback) { in smc_sendpage()
2387 rc = kernel_sendpage(smc->clcsock, page, offset, in smc_sendpage()
2390 SMC_STAT_INC(smc, sendpage_cnt); in smc_sendpage()
2399 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2408 struct sock *sk = sock->sk; in smc_splice_read()
2409 struct smc_sock *smc; in smc_splice_read() local
2410 int rc = -ENOTCONN; in smc_splice_read()
2412 smc = smc_sk(sk); in smc_splice_read()
2414 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) { in smc_splice_read()
2419 if (sk->sk_state == SMC_INIT || in smc_splice_read()
2420 sk->sk_state == SMC_LISTEN || in smc_splice_read()
2421 sk->sk_state == SMC_CLOSED) in smc_splice_read()
2424 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { in smc_splice_read()
2429 if (smc->use_fallback) { in smc_splice_read()
2430 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos, in smc_splice_read()
2434 rc = -ESPIPE; in smc_splice_read()
2441 SMC_STAT_INC(smc, splice_cnt); in smc_splice_read()
2442 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags); in smc_splice_read()
2477 struct smc_sock *smc; in smc_create() local
2481 rc = -ESOCKTNOSUPPORT; in smc_create()
2482 if (sock->type != SOCK_STREAM) in smc_create()
2485 rc = -EPROTONOSUPPORT; in smc_create()
2489 rc = -ENOBUFS; in smc_create()
2490 sock->ops = &smc_sock_ops; in smc_create()
2496 smc = smc_sk(sk); in smc_create()
2497 smc->use_fallback = false; /* assume rdma capability first */ in smc_create()
2498 smc->fallback_rsn = 0; in smc_create()
2500 &smc->clcsock); in smc_create()
2505 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE); in smc_create()
2506 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE); in smc_create()
2543 .id = &smc_net_id,
2575 rc = -ENOMEM; in smc_init()
2674 MODULE_DESCRIPTION("smc socket address family");