71 static void smc_ibdev_cnt_inc(struct smc_link *lnk)  in smc_ibdev_cnt_inc()  argument
73 	atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);  in smc_ibdev_cnt_inc()
76 static void smc_ibdev_cnt_dec(struct smc_link *lnk)  in smc_ibdev_cnt_dec()  argument
78 	atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);  in smc_ibdev_cnt_dec()
131 		struct smc_link *lnk = &conn->lgr->lnk[i];  in smcr_lgr_conn_assign_link()  local
133 		if (lnk->state != expected || lnk->link_is_asym)  in smcr_lgr_conn_assign_link()
136 			conn->lnk = lnk; /* temporary, SMC server assigns link*/  in smcr_lgr_conn_assign_link()
143 				lnk2 = &conn->lgr->lnk[j];  in smcr_lgr_conn_assign_link()
146 					conn->lnk = lnk2;  in smcr_lgr_conn_assign_link()
151 		if (!conn->lnk)  in smcr_lgr_conn_assign_link()
152 			conn->lnk = lnk;  in smcr_lgr_conn_assign_link()
155 	if (!conn->lnk)  in smcr_lgr_conn_assign_link()
157 	atomic_inc(&conn->lnk->conn_cnt);  in smcr_lgr_conn_assign_link()
201 	if (conn->lnk)  in __smc_lgr_unregister_conn()
202 		atomic_dec(&conn->lnk->conn_cnt);  in __smc_lgr_unregister_conn()
460 		if (!smc_link_usable(&lgr->lnk[i]))  in smc_nl_handle_lgr()
462 		if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb))  in smc_nl_handle_lgr()
652 		struct smc_link *lnk = &lgr->lnk[i];  in smcr_lgr_link_deactivate_all()  local
654 		if (smc_link_sendable(lnk))  in smcr_lgr_link_deactivate_all()
655 			lnk->state = SMC_LNK_INACTIVE;  in smcr_lgr_link_deactivate_all()
719 			if (smc_link_usable(&lgr->lnk[i]) &&  in smcr_next_link_id()
720 			    lgr->lnk[i].link_id == link_id)  in smcr_next_link_id()
737 int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,  in smcr_link_init()  argument
745 		lnk->smcibdev = ini->smcrv2.ib_dev_v2;  in smcr_link_init()
746 		lnk->ibport = ini->smcrv2.ib_port_v2;  in smcr_link_init()
748 		lnk->smcibdev = ini->ib_dev;  in smcr_link_init()
749 		lnk->ibport = ini->ib_port;  in smcr_link_init()
751 	get_device(&lnk->smcibdev->ibdev->dev);  in smcr_link_init()
752 	atomic_inc(&lnk->smcibdev->lnk_cnt);  in smcr_link_init()
753 	refcount_set(&lnk->refcnt, 1); /* link refcnt is set to 1 */  in smcr_link_init()
754 	lnk->clearing = 0;  in smcr_link_init()
755 	lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu;  in smcr_link_init()
756 	lnk->link_id = smcr_next_link_id(lgr);  in smcr_link_init()
757 	lnk->lgr = lgr;  in smcr_link_init()
759 	lnk->link_idx = link_idx;  in smcr_link_init()
760 	lnk->wr_rx_id_compl = 0;  in smcr_link_init()
761 	smc_ibdev_cnt_inc(lnk);  in smcr_link_init()
762 	smcr_copy_dev_info_to_link(lnk);  in smcr_link_init()
763 	atomic_set(&lnk->conn_cnt, 0);  in smcr_link_init()
764 	smc_llc_link_set_uid(lnk);  in smcr_link_init()
765 	INIT_WORK(&lnk->link_down_wrk, smc_link_down_work);  in smcr_link_init()
766 	if (!lnk->smcibdev->initialized) {  in smcr_link_init()
767 		rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev);  in smcr_link_init()
772 	lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +  in smcr_link_init()
774 	rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,  in smcr_link_init()
775 				  ini->vlan_id, lnk->gid, &lnk->sgid_index,  in smcr_link_init()
780 	rc = smc_llc_link_init(lnk);  in smcr_link_init()
783 	rc = smc_wr_alloc_link_mem(lnk);  in smcr_link_init()
786 	rc = smc_ib_create_protection_domain(lnk);  in smcr_link_init()
789 	rc = smc_ib_create_queue_pair(lnk);  in smcr_link_init()
792 	rc = smc_wr_create_link(lnk);  in smcr_link_init()
795 	lnk->state = SMC_LNK_ACTIVATING;  in smcr_link_init()
799 	smc_ib_destroy_queue_pair(lnk);  in smcr_link_init()
801 	smc_ib_dealloc_protection_domain(lnk);  in smcr_link_init()
803 	smc_wr_free_link_mem(lnk);  in smcr_link_init()
805 	smc_llc_link_clear(lnk, false);  in smcr_link_init()
807 	smc_ibdev_cnt_dec(lnk);  in smcr_link_init()
808 	put_device(&lnk->smcibdev->ibdev->dev);  in smcr_link_init()
809 	smcibdev = lnk->smcibdev;  in smcr_link_init()
810 	memset(lnk, 0, sizeof(struct smc_link));  in smcr_link_init()
811 	lnk->state = SMC_LNK_UNUSED;  in smcr_link_init()
823 	struct smc_link *lnk;  in smc_lgr_create()  local
905 		lnk = &lgr->lnk[link_idx];  in smc_lgr_create()
906 		rc = smcr_link_init(lgr, lnk, link_idx, ini);  in smc_lgr_create()
911 		lgr->net = smc_ib_net(lnk->smcibdev);  in smc_lgr_create()
998 		smc_wr_tx_put_slot(conn->lnk,  in smc_switch_cursor()
1007 	atomic_dec(&conn->lnk->conn_cnt);  in smc_switch_link_and_count()
1009 	smcr_link_put(conn->lnk);  in smc_switch_link_and_count()
1010 	conn->lnk = to_lnk;  in smc_switch_link_and_count()
1011 	atomic_inc(&conn->lnk->conn_cnt);  in smc_switch_link_and_count()
1013 	smcr_link_hold(conn->lnk);  in smc_switch_link_and_count()
1031 		if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx)  in smc_switch_conns()
1033 		if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev &&  in smc_switch_conns()
1034 		    from_lnk->ibport == lgr->lnk[i].ibport) {  in smc_switch_conns()
1037 		to_lnk = &lgr->lnk[i];  in smc_switch_conns()
1048 		if (conn->lnk != from_lnk)  in smc_switch_conns()
1051 		/* conn->lnk not yet set in SMC_INIT state */  in smc_switch_conns()
1188 		smcr_link_put(conn->lnk); /* link_hold in smc_conn_create() */  in smc_conn_free()
1194 				struct smc_link *lnk)  in smcr_buf_unmap_link()  argument
1197 		buf_desc->is_reg_mr[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1198 	if (!buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_unmap_link()
1202 	    buf_desc->mr[lnk->link_idx]) {  in smcr_buf_unmap_link()
1203 		smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]);  in smcr_buf_unmap_link()
1204 		buf_desc->mr[lnk->link_idx] = NULL;  in smcr_buf_unmap_link()
1207 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE);  in smcr_buf_unmap_link()
1209 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE);  in smcr_buf_unmap_link()
1211 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_unmap_link()
1212 	buf_desc->is_map_ib[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1216 static void smcr_buf_unmap_lgr(struct smc_link *lnk)  in smcr_buf_unmap_lgr()  argument
1218 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_unmap_lgr()
1225 			smcr_buf_unmap_link(buf_desc, true, lnk);  in smcr_buf_unmap_lgr()
1230 			smcr_buf_unmap_link(buf_desc, false, lnk);  in smcr_buf_unmap_lgr()
1235 static void smcr_rtoken_clear_link(struct smc_link *lnk)  in smcr_rtoken_clear_link()  argument
1237 	struct smc_link_group *lgr = lnk->lgr;  in smcr_rtoken_clear_link()
1241 		lgr->rtokens[i][lnk->link_idx].rkey = 0;  in smcr_rtoken_clear_link()
1242 		lgr->rtokens[i][lnk->link_idx].dma_addr = 0;  in smcr_rtoken_clear_link()
1246 static void __smcr_link_clear(struct smc_link *lnk)  in __smcr_link_clear()  argument
1248 	struct smc_link_group *lgr = lnk->lgr;  in __smcr_link_clear()
1251 	smc_wr_free_link_mem(lnk);  in __smcr_link_clear()
1252 	smc_ibdev_cnt_dec(lnk);  in __smcr_link_clear()
1253 	put_device(&lnk->smcibdev->ibdev->dev);  in __smcr_link_clear()
1254 	smcibdev = lnk->smcibdev;  in __smcr_link_clear()
1255 	memset(lnk, 0, sizeof(struct smc_link));  in __smcr_link_clear()
1256 	lnk->state = SMC_LNK_UNUSED;  in __smcr_link_clear()
1263 void smcr_link_clear(struct smc_link *lnk, bool log)  in smcr_link_clear()  argument
1265 	if (!lnk->lgr || lnk->clearing ||  in smcr_link_clear()
1266 	    lnk->state == SMC_LNK_UNUSED)  in smcr_link_clear()
1268 	lnk->clearing = 1;  in smcr_link_clear()
1269 	lnk->peer_qpn = 0;  in smcr_link_clear()
1270 	smc_llc_link_clear(lnk, log);  in smcr_link_clear()
1271 	smcr_buf_unmap_lgr(lnk);  in smcr_link_clear()
1272 	smcr_rtoken_clear_link(lnk);  in smcr_link_clear()
1273 	smc_ib_modify_qp_error(lnk);  in smcr_link_clear()
1274 	smc_wr_free_link(lnk);  in smcr_link_clear()
1275 	smc_ib_destroy_queue_pair(lnk);  in smcr_link_clear()
1276 	smc_ib_dealloc_protection_domain(lnk);  in smcr_link_clear()
1277 	smcr_link_put(lnk); /* theoretically last link_put */  in smcr_link_clear()
1280 void smcr_link_hold(struct smc_link *lnk)  in smcr_link_hold()  argument
1282 	refcount_inc(&lnk->refcnt);  in smcr_link_hold()
1285 void smcr_link_put(struct smc_link *lnk)  in smcr_link_put()  argument
1287 	if (refcount_dec_and_test(&lnk->refcnt))  in smcr_link_put()
1288 		__smcr_link_clear(lnk);  in smcr_link_put()
1297 		smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]);  in smcr_buf_free()
1378 			if (lgr->lnk[i].state != SMC_LNK_UNUSED)  in smc_lgr_free()
1379 				smcr_link_clear(&lgr->lnk[i], false);  in smc_lgr_free()
1568 				if (lgr->lnk[i].smcibdev == smcibdev)  in smc_smcr_terminate_all()
1569 					smcr_link_down_cond_sched(&lgr->lnk[i]);  in smc_smcr_terminate_all()
1598 		if (smc_link_usable(&lgr->lnk[i]))  in smcr_lgr_set_type()
1599 			lgr->lnk[i].link_is_asym = false;  in smcr_lgr_set_type()
1631 	lgr->lnk[asym_lnk_idx].link_is_asym = true;  in smcr_lgr_set_type_asym()
1672 static void smcr_link_down(struct smc_link *lnk)  in smcr_link_down()  argument
1674 	struct smc_link_group *lgr = lnk->lgr;  in smcr_link_down()
1678 	if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list))  in smcr_link_down()
1681 	to_lnk = smc_switch_conns(lgr, lnk, true);  in smcr_link_down()
1683 		smcr_link_clear(lnk, true);  in smcr_link_down()
1687 	del_link_id = lnk->link_id;  in smcr_link_down()
1706 			smcr_link_clear(lnk, true);  in smcr_link_down()
1713 void smcr_link_down_cond(struct smc_link *lnk)  in smcr_link_down_cond()  argument
1715 	if (smc_link_downing(&lnk->state)) {  in smcr_link_down_cond()
1716 		trace_smcr_link_down(lnk, __builtin_return_address(0));  in smcr_link_down_cond()
1717 		smcr_link_down(lnk);  in smcr_link_down_cond()
1722 void smcr_link_down_cond_sched(struct smc_link *lnk)  in smcr_link_down_cond_sched()  argument
1724 	if (smc_link_downing(&lnk->state)) {  in smcr_link_down_cond_sched()
1725 		trace_smcr_link_down(lnk, __builtin_return_address(0));  in smcr_link_down_cond_sched()
1726 		schedule_work(&lnk->link_down_wrk);  in smcr_link_down_cond_sched()
1742 			struct smc_link *lnk = &lgr->lnk[i];  in smcr_port_err()  local
1744 			if (smc_link_usable(lnk) &&  in smcr_port_err()
1745 			    lnk->smcibdev == smcibdev && lnk->ibport == ibport)  in smcr_port_err()
1746 				smcr_link_down_cond_sched(lnk);  in smcr_port_err()
1820 	struct smc_link *lnk;  in smcr_lgr_match()  local
1828 		lnk = &lgr->lnk[i];  in smcr_lgr_match()
1830 		if (!smc_link_active(lnk))  in smcr_lgr_match()
1833 		if (!rdma_dev_access_netns(lnk->smcibdev->ibdev, net))  in smcr_lgr_match()
1835 		if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) &&  in smcr_lgr_match()
1836 		    !memcmp(lnk->peer_gid, peer_gid, SMC_GID_SIZE) &&  in smcr_lgr_match()
1838 		     !memcmp(lnk->peer_mac, peer_mac_v1, ETH_ALEN)))  in smcr_lgr_match()
1928 		smcr_link_hold(conn->lnk); /* link_put in smc_conn_free() */  in smc_conn_create()
2016 			     struct smc_link *lnk)  in smcr_buf_map_link()  argument
2022 	if (buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_map_link()
2034 	rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL);  in smcr_buf_map_link()
2040 		for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) {  in smcr_buf_map_link()
2049 		sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl,  in smcr_buf_map_link()
2054 	rc = smc_ib_buf_map_sg(lnk, buf_desc,  in smcr_buf_map_link()
2063 		smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx;  in smcr_buf_map_link()
2071 		rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags,  in smcr_buf_map_link()
2072 					      buf_desc, lnk->link_idx);  in smcr_buf_map_link()
2075 		smc_ib_sync_sg_for_device(lnk, buf_desc,  in smcr_buf_map_link()
2078 	buf_desc->is_map_ib[lnk->link_idx] = true;  in smcr_buf_map_link()
2082 	smc_ib_buf_unmap_sg(lnk, buf_desc,  in smcr_buf_map_link()
2085 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_map_link()
2110 static int _smcr_buf_map_lgr(struct smc_link *lnk, struct mutex *lock,  in _smcr_buf_map_lgr()  argument
2120 		rc = smcr_buf_map_link(buf_desc, is_rmb, lnk);  in _smcr_buf_map_lgr()
2130 int smcr_buf_map_lgr(struct smc_link *lnk)  in smcr_buf_map_lgr()  argument
2132 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_map_lgr()
2136 		rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock,  in smcr_buf_map_lgr()
2140 		rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock,  in smcr_buf_map_lgr()
2151 int smcr_buf_reg_lgr(struct smc_link *lnk)  in smcr_buf_reg_lgr()  argument
2153 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_reg_lgr()
2163 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2181 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2248 		struct smc_link *lnk = &lgr->lnk[i];  in smcr_buf_map_usable_links()  local
2250 		if (!smc_link_usable(lnk))  in smcr_buf_map_usable_links()
2252 		if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) {  in smcr_buf_map_usable_links()
2395 	    !smc_link_active(conn->lnk))  in smc_sndbuf_sync_sg_for_device()
2397 	smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE);  in smc_sndbuf_sync_sg_for_device()
2409 		if (!smc_link_active(&conn->lgr->lnk[i]))  in smc_rmb_sync_sg_for_cpu()
2411 		smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc,  in smc_rmb_sync_sg_for_cpu()
2489 		if (lgr->lnk[link_idx].link_id == link_id) {  in smc_rtoken_set2()
2501 int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey)  in smc_rtoken_add()  argument
2503 	struct smc_link_group *lgr = smc_get_lgr(lnk);  in smc_rtoken_add()
2509 		if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&  in smc_rtoken_add()
2510 		    lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr &&  in smc_rtoken_add()
2519 	lgr->rtokens[i][lnk->link_idx].rkey = rkey;  in smc_rtoken_add()
2520 	lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr;  in smc_rtoken_add()
2525 int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey)  in smc_rtoken_delete()  argument
2527 	struct smc_link_group *lgr = smc_get_lgr(lnk);  in smc_rtoken_delete()
2532 		if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&  in smc_rtoken_delete()
2547 			    struct smc_link *lnk,  in smc_rmb_rtoken_handling()  argument
2550 	conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr,  in smc_rmb_rtoken_handling()