Lines Matching refs:lgr
52 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
54 static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft);
59 static inline struct list_head *smc_lgr_list_head(struct smc_link_group *lgr, in smc_lgr_list_head() argument
62 if (lgr->is_smcd) { in smc_lgr_list_head()
63 *lgr_lock = &lgr->smcd->lgr_lock; in smc_lgr_list_head()
64 return &lgr->smcd->lgr_list; in smc_lgr_list_head()
81 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr) in smc_lgr_schedule_free_work() argument
87 if (!lgr->freeing) { in smc_lgr_schedule_free_work()
88 mod_delayed_work(system_wq, &lgr->free_work, in smc_lgr_schedule_free_work()
89 (!lgr->is_smcd && lgr->role == SMC_CLNT) ? in smc_lgr_schedule_free_work()
106 link = &conn->lgr->conns_all.rb_node; in smc_lgr_add_alert_token()
119 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); in smc_lgr_add_alert_token()
132 struct smc_link *lnk = &conn->lgr->lnk[i]; in smcr_lgr_conn_assign_link()
136 if (conn->lgr->role == SMC_CLNT) { in smcr_lgr_conn_assign_link()
140 if (conn->lgr->conns_num % 2) { in smcr_lgr_conn_assign_link()
144 lnk2 = &conn->lgr->lnk[j]; in smcr_lgr_conn_assign_link()
173 if (!conn->lgr->is_smcd) { in smc_lgr_register_conn()
176 conn->lgr = NULL; in smc_lgr_register_conn()
186 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) in smc_lgr_register_conn()
190 conn->lgr->conns_num++; in smc_lgr_register_conn()
199 struct smc_link_group *lgr = conn->lgr; in __smc_lgr_unregister_conn() local
201 rb_erase(&conn->alert_node, &lgr->conns_all); in __smc_lgr_unregister_conn()
204 lgr->conns_num--; in __smc_lgr_unregister_conn()
213 struct smc_link_group *lgr = conn->lgr; in smc_lgr_unregister_conn() local
217 write_lock_bh(&lgr->conns_lock); in smc_lgr_unregister_conn()
221 write_unlock_bh(&lgr->conns_lock); in smc_lgr_unregister_conn()
280 static int smc_nl_fill_lgr_v2_common(struct smc_link_group *lgr, in smc_nl_fill_lgr_v2_common() argument
288 if (nla_put_u8(skb, SMC_NLA_LGR_V2_VER, lgr->smc_version)) in smc_nl_fill_lgr_v2_common()
290 if (nla_put_u8(skb, SMC_NLA_LGR_V2_REL, lgr->peer_smc_release)) in smc_nl_fill_lgr_v2_common()
292 if (nla_put_u8(skb, SMC_NLA_LGR_V2_OS, lgr->peer_os)) in smc_nl_fill_lgr_v2_common()
294 memcpy(smc_host, lgr->peer_hostname, SMC_MAX_HOSTNAME_LEN); in smc_nl_fill_lgr_v2_common()
298 memcpy(smc_eid, lgr->negotiated_eid, SMC_MAX_EID_LEN); in smc_nl_fill_lgr_v2_common()
311 static int smc_nl_fill_smcr_lgr_v2(struct smc_link_group *lgr, in smc_nl_fill_smcr_lgr_v2() argument
320 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_DIRECT, !lgr->uses_gateway)) in smc_nl_fill_smcr_lgr_v2()
322 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_MAX_CONNS, lgr->max_conns)) in smc_nl_fill_smcr_lgr_v2()
324 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_MAX_LINKS, lgr->max_links)) in smc_nl_fill_smcr_lgr_v2()
336 static int smc_nl_fill_lgr(struct smc_link_group *lgr, in smc_nl_fill_lgr() argument
347 if (nla_put_u32(skb, SMC_NLA_LGR_R_ID, *((u32 *)&lgr->id))) in smc_nl_fill_lgr()
349 if (nla_put_u32(skb, SMC_NLA_LGR_R_CONNS_NUM, lgr->conns_num)) in smc_nl_fill_lgr()
351 if (nla_put_u8(skb, SMC_NLA_LGR_R_ROLE, lgr->role)) in smc_nl_fill_lgr()
353 if (nla_put_u8(skb, SMC_NLA_LGR_R_TYPE, lgr->type)) in smc_nl_fill_lgr()
355 if (nla_put_u8(skb, SMC_NLA_LGR_R_BUF_TYPE, lgr->buf_type)) in smc_nl_fill_lgr()
357 if (nla_put_u8(skb, SMC_NLA_LGR_R_VLAN_ID, lgr->vlan_id)) in smc_nl_fill_lgr()
360 lgr->net->net_cookie, SMC_NLA_LGR_R_PAD)) in smc_nl_fill_lgr()
362 memcpy(smc_target, lgr->pnet_id, SMC_MAX_PNETID_LEN); in smc_nl_fill_lgr()
366 if (lgr->smc_version > SMC_V1) { in smc_nl_fill_lgr()
370 if (smc_nl_fill_lgr_v2_common(lgr, skb, cb, v2_attrs)) in smc_nl_fill_lgr()
372 if (smc_nl_fill_smcr_lgr_v2(lgr, skb, cb)) in smc_nl_fill_lgr()
384 static int smc_nl_fill_lgr_link(struct smc_link_group *lgr, in smc_nl_fill_lgr_link() argument
445 static int smc_nl_handle_lgr(struct smc_link_group *lgr, in smc_nl_handle_lgr() argument
458 if (smc_nl_fill_lgr(lgr, skb, cb)) in smc_nl_handle_lgr()
465 if (!smc_link_usable(&lgr->lnk[i])) in smc_nl_handle_lgr()
467 if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb)) in smc_nl_handle_lgr()
485 struct smc_link_group *lgr; in smc_nl_fill_lgr_list() local
490 list_for_each_entry(lgr, &smc_lgr->list, list) { in smc_nl_fill_lgr_list()
493 if (smc_nl_handle_lgr(lgr, skb, cb, list_links)) in smc_nl_fill_lgr_list()
503 static int smc_nl_fill_smcd_lgr(struct smc_link_group *lgr, in smc_nl_fill_smcd_lgr() argument
508 struct smcd_dev *smcd = lgr->smcd; in smc_nl_fill_smcd_lgr()
522 if (nla_put_u32(skb, SMC_NLA_LGR_D_ID, *((u32 *)&lgr->id))) in smc_nl_fill_smcd_lgr()
528 if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_GID, lgr->peer_gid, in smc_nl_fill_smcd_lgr()
531 if (nla_put_u8(skb, SMC_NLA_LGR_D_VLAN_ID, lgr->vlan_id)) in smc_nl_fill_smcd_lgr()
533 if (nla_put_u32(skb, SMC_NLA_LGR_D_CONNS_NUM, lgr->conns_num)) in smc_nl_fill_smcd_lgr()
535 if (nla_put_u32(skb, SMC_NLA_LGR_D_CHID, smc_ism_get_chid(lgr->smcd))) in smc_nl_fill_smcd_lgr()
537 memcpy(smc_pnet, lgr->smcd->pnetid, SMC_MAX_PNETID_LEN); in smc_nl_fill_smcd_lgr()
541 if (lgr->smc_version > SMC_V1) { in smc_nl_fill_smcd_lgr()
547 if (smc_nl_fill_lgr_v2_common(lgr, skb, cb, v2_attrs)) in smc_nl_fill_smcd_lgr()
567 struct smc_link_group *lgr; in smc_nl_handle_smcd_lgr() local
572 list_for_each_entry(lgr, &dev->lgr_list, list) { in smc_nl_handle_smcd_lgr()
573 if (!lgr->is_smcd) in smc_nl_handle_smcd_lgr()
577 rc = smc_nl_fill_smcd_lgr(lgr, skb, cb); in smc_nl_handle_smcd_lgr()
638 void smc_lgr_cleanup_early(struct smc_link_group *lgr) in smc_lgr_cleanup_early() argument
642 if (!lgr) in smc_lgr_cleanup_early()
645 smc_lgr_list_head(lgr, &lgr_lock); in smc_lgr_cleanup_early()
648 if (!list_empty(&lgr->list)) in smc_lgr_cleanup_early()
649 list_del_init(&lgr->list); in smc_lgr_cleanup_early()
651 __smc_lgr_terminate(lgr, true); in smc_lgr_cleanup_early()
654 static void smcr_lgr_link_deactivate_all(struct smc_link_group *lgr) in smcr_lgr_link_deactivate_all() argument
659 struct smc_link *lnk = &lgr->lnk[i]; in smcr_lgr_link_deactivate_all()
664 wake_up_all(&lgr->llc_msg_waiter); in smcr_lgr_link_deactivate_all()
665 wake_up_all(&lgr->llc_flow_waiter); in smcr_lgr_link_deactivate_all()
668 static void smc_lgr_free(struct smc_link_group *lgr);
672 struct smc_link_group *lgr = container_of(to_delayed_work(work), in smc_lgr_free_work() local
678 smc_lgr_list_head(lgr, &lgr_lock); in smc_lgr_free_work()
680 if (lgr->freeing) { in smc_lgr_free_work()
684 read_lock_bh(&lgr->conns_lock); in smc_lgr_free_work()
685 conns = RB_EMPTY_ROOT(&lgr->conns_all); in smc_lgr_free_work()
686 read_unlock_bh(&lgr->conns_lock); in smc_lgr_free_work()
691 list_del_init(&lgr->list); /* remove from smc_lgr_list */ in smc_lgr_free_work()
692 lgr->freeing = 1; /* this instance does the freeing, no new schedule */ in smc_lgr_free_work()
694 cancel_delayed_work(&lgr->free_work); in smc_lgr_free_work()
696 if (!lgr->is_smcd && !lgr->terminating) in smc_lgr_free_work()
697 smc_llc_send_link_delete_all(lgr, true, in smc_lgr_free_work()
699 if (lgr->is_smcd && !lgr->terminating) in smc_lgr_free_work()
700 smc_ism_signal_shutdown(lgr); in smc_lgr_free_work()
701 if (!lgr->is_smcd) in smc_lgr_free_work()
702 smcr_lgr_link_deactivate_all(lgr); in smc_lgr_free_work()
703 smc_lgr_free(lgr); in smc_lgr_free_work()
708 struct smc_link_group *lgr = container_of(work, struct smc_link_group, in smc_lgr_terminate_work() local
711 __smc_lgr_terminate(lgr, true); in smc_lgr_terminate_work()
715 static u8 smcr_next_link_id(struct smc_link_group *lgr) in smcr_next_link_id() argument
722 link_id = ++lgr->next_link_id; in smcr_next_link_id()
724 link_id = ++lgr->next_link_id; in smcr_next_link_id()
726 if (smc_link_usable(&lgr->lnk[i]) && in smcr_next_link_id()
727 lgr->lnk[i].link_id == link_id) in smcr_next_link_id()
744 int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk, in smcr_link_init() argument
751 if (lgr->smc_version == SMC_V2) { in smcr_link_init()
763 lnk->link_id = smcr_next_link_id(lgr); in smcr_link_init()
764 lnk->lgr = lgr; in smcr_link_init()
765 smc_lgr_hold(lgr); /* lgr_put in smcr_link_clear() */ in smcr_link_init()
783 lgr->smc_version == SMC_V2 ? in smcr_link_init()
821 smc_lgr_put(lgr); /* lgr_hold above */ in smcr_link_init()
828 struct smc_link_group *lgr; in smc_lgr_create() local
845 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL); in smc_lgr_create()
846 if (!lgr) { in smc_lgr_create()
850 lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", 0, 0, in smc_lgr_create()
851 SMC_LGR_ID_SIZE, &lgr->id); in smc_lgr_create()
852 if (!lgr->tx_wq) { in smc_lgr_create()
856 lgr->is_smcd = ini->is_smcd; in smc_lgr_create()
857 lgr->sync_err = 0; in smc_lgr_create()
858 lgr->terminating = 0; in smc_lgr_create()
859 lgr->freeing = 0; in smc_lgr_create()
860 lgr->vlan_id = ini->vlan_id; in smc_lgr_create()
861 refcount_set(&lgr->refcnt, 1); /* set lgr refcnt to 1 */ in smc_lgr_create()
862 init_rwsem(&lgr->sndbufs_lock); in smc_lgr_create()
863 init_rwsem(&lgr->rmbs_lock); in smc_lgr_create()
864 rwlock_init(&lgr->conns_lock); in smc_lgr_create()
866 INIT_LIST_HEAD(&lgr->sndbufs[i]); in smc_lgr_create()
867 INIT_LIST_HEAD(&lgr->rmbs[i]); in smc_lgr_create()
869 lgr->next_link_id = 0; in smc_lgr_create()
871 memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE); in smc_lgr_create()
872 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); in smc_lgr_create()
873 INIT_WORK(&lgr->terminate_work, smc_lgr_terminate_work); in smc_lgr_create()
874 lgr->conns_all = RB_ROOT; in smc_lgr_create()
879 lgr->peer_gid = ini->ism_peer_gid[ini->ism_selected]; in smc_lgr_create()
880 lgr->smcd = ini->ism_dev[ini->ism_selected]; in smc_lgr_create()
882 lgr_lock = &lgr->smcd->lgr_lock; in smc_lgr_create()
883 lgr->smc_version = ini->smcd_version; in smc_lgr_create()
884 lgr->peer_shutdown = 0; in smc_lgr_create()
891 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; in smc_lgr_create()
892 lgr->smc_version = ini->smcr_version; in smc_lgr_create()
893 memcpy(lgr->peer_systemid, ini->peer_systemid, in smc_lgr_create()
895 if (lgr->smc_version == SMC_V2) { in smc_lgr_create()
898 lgr->saddr = ini->smcrv2.saddr; in smc_lgr_create()
899 lgr->uses_gateway = ini->smcrv2.uses_gateway; in smc_lgr_create()
900 memcpy(lgr->nexthop_mac, ini->smcrv2.nexthop_mac, in smc_lgr_create()
902 lgr->max_conns = ini->max_conns; in smc_lgr_create()
903 lgr->max_links = ini->max_links; in smc_lgr_create()
907 lgr->max_conns = SMC_CONN_PER_LGR_MAX; in smc_lgr_create()
908 lgr->max_links = SMC_LINKS_ADD_LNK_MAX; in smc_lgr_create()
910 memcpy(lgr->pnet_id, ibdev->pnetid[ibport - 1], in smc_lgr_create()
912 rc = smc_wr_alloc_lgr_mem(lgr); in smc_lgr_create()
915 smc_llc_lgr_init(lgr, smc); in smc_lgr_create()
918 lnk = &lgr->lnk[link_idx]; in smc_lgr_create()
919 rc = smcr_link_init(lgr, lnk, link_idx, ini); in smc_lgr_create()
921 smc_wr_free_lgr_mem(lgr); in smc_lgr_create()
924 lgr->net = smc_ib_net(lnk->smcibdev); in smc_lgr_create()
927 lgr->buf_type = lgr->net->smc.sysctl_smcr_buf_type; in smc_lgr_create()
930 smc->conn.lgr = lgr; in smc_lgr_create()
932 list_add_tail(&lgr->list, lgr_list); in smc_lgr_create()
937 destroy_workqueue(lgr->tx_wq); in smc_lgr_create()
939 kfree(lgr); in smc_lgr_create()
1007 queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work, 0); in smc_switch_cursor()
1029 struct smc_link *smc_switch_conns(struct smc_link_group *lgr, in smc_switch_conns() argument
1044 if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx) in smc_switch_conns()
1046 if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev && in smc_switch_conns()
1047 from_lnk->ibport == lgr->lnk[i].ibport) { in smc_switch_conns()
1050 to_lnk = &lgr->lnk[i]; in smc_switch_conns()
1054 smc_lgr_terminate_sched(lgr); in smc_switch_conns()
1058 read_lock_bh(&lgr->conns_lock); in smc_switch_conns()
1059 for (node = rb_first(&lgr->conns_all); node; node = rb_next(node)) { in smc_switch_conns()
1082 read_unlock_bh(&lgr->conns_lock); in smc_switch_conns()
1097 read_unlock_bh(&lgr->conns_lock); in smc_switch_conns()
1108 struct smc_link_group *lgr) in smcr_buf_unuse() argument
1113 if (is_rmb && buf_desc->is_conf_rkey && !list_empty(&lgr->list)) { in smcr_buf_unuse()
1115 rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); in smcr_buf_unuse()
1118 down_read(&lgr->llc_conf_mutex); in smcr_buf_unuse()
1119 smc_llc_do_delete_rkey(lgr, buf_desc); in smcr_buf_unuse()
1121 up_read(&lgr->llc_conf_mutex); in smcr_buf_unuse()
1122 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); in smcr_buf_unuse()
1128 lock = is_rmb ? &lgr->rmbs_lock : in smcr_buf_unuse()
1129 &lgr->sndbufs_lock; in smcr_buf_unuse()
1134 smc_buf_free(lgr, is_rmb, buf_desc); in smcr_buf_unuse()
1143 struct smc_link_group *lgr) in smc_buf_unuse() argument
1146 if (!lgr->is_smcd && conn->sndbuf_desc->is_vm) { in smc_buf_unuse()
1147 smcr_buf_unuse(conn->sndbuf_desc, false, lgr); in smc_buf_unuse()
1154 if (!lgr->is_smcd) { in smc_buf_unuse()
1155 smcr_buf_unuse(conn->rmb_desc, true, lgr); in smc_buf_unuse()
1167 struct smc_link_group *lgr = conn->lgr; in smc_conn_free() local
1169 if (!lgr || conn->freed) in smc_conn_free()
1182 if (lgr->is_smcd) { in smc_conn_free()
1183 if (!list_empty(&lgr->list)) in smc_conn_free()
1191 if (!list_empty(&lgr->list)) { in smc_conn_free()
1192 smc_buf_unuse(conn, lgr); /* allow buffer reuse */ in smc_conn_free()
1196 if (!lgr->conns_num) in smc_conn_free()
1197 smc_lgr_schedule_free_work(lgr); in smc_conn_free()
1199 if (!lgr->is_smcd) in smc_conn_free()
1201 smc_lgr_put(lgr); /* lgr_hold in smc_conn_create() */ in smc_conn_free()
1230 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_unmap_lgr() local
1235 down_write(&lgr->rmbs_lock); in smcr_buf_unmap_lgr()
1236 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) in smcr_buf_unmap_lgr()
1238 up_write(&lgr->rmbs_lock); in smcr_buf_unmap_lgr()
1240 down_write(&lgr->sndbufs_lock); in smcr_buf_unmap_lgr()
1241 list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], in smcr_buf_unmap_lgr()
1244 up_write(&lgr->sndbufs_lock); in smcr_buf_unmap_lgr()
1250 struct smc_link_group *lgr = lnk->lgr; in smcr_rtoken_clear_link() local
1254 lgr->rtokens[i][lnk->link_idx].rkey = 0; in smcr_rtoken_clear_link()
1255 lgr->rtokens[i][lnk->link_idx].dma_addr = 0; in smcr_rtoken_clear_link()
1261 struct smc_link_group *lgr = lnk->lgr; in __smcr_link_clear() local
1272 smc_lgr_put(lgr); /* lgr_hold in smcr_link_init() */ in __smcr_link_clear()
1278 if (!lnk->lgr || lnk->clearing || in smcr_link_clear()
1304 static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb, in smcr_buf_free() argument
1310 smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]); in smcr_buf_free()
1319 static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb, in smcd_buf_free() argument
1325 smc_ism_unregister_dmb(lgr->smcd, buf_desc); in smcd_buf_free()
1332 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb, in smc_buf_free() argument
1335 if (lgr->is_smcd) in smc_buf_free()
1336 smcd_buf_free(lgr, is_rmb, buf_desc); in smc_buf_free()
1338 smcr_buf_free(lgr, is_rmb, buf_desc); in smc_buf_free()
1341 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb) in __smc_lgr_free_bufs() argument
1349 buf_list = &lgr->rmbs[i]; in __smc_lgr_free_bufs()
1351 buf_list = &lgr->sndbufs[i]; in __smc_lgr_free_bufs()
1355 smc_buf_free(lgr, is_rmb, buf_desc); in __smc_lgr_free_bufs()
1360 static void smc_lgr_free_bufs(struct smc_link_group *lgr) in smc_lgr_free_bufs() argument
1363 __smc_lgr_free_bufs(lgr, false); in smc_lgr_free_bufs()
1365 __smc_lgr_free_bufs(lgr, true); in smc_lgr_free_bufs()
1369 static void __smc_lgr_free(struct smc_link_group *lgr) in __smc_lgr_free() argument
1371 smc_lgr_free_bufs(lgr); in __smc_lgr_free()
1372 if (lgr->is_smcd) { in __smc_lgr_free()
1373 if (!atomic_dec_return(&lgr->smcd->lgr_cnt)) in __smc_lgr_free()
1374 wake_up(&lgr->smcd->lgrs_deleted); in __smc_lgr_free()
1376 smc_wr_free_lgr_mem(lgr); in __smc_lgr_free()
1380 kfree(lgr); in __smc_lgr_free()
1384 static void smc_lgr_free(struct smc_link_group *lgr) in smc_lgr_free() argument
1388 if (!lgr->is_smcd) { in smc_lgr_free()
1389 down_write(&lgr->llc_conf_mutex); in smc_lgr_free()
1391 if (lgr->lnk[i].state != SMC_LNK_UNUSED) in smc_lgr_free()
1392 smcr_link_clear(&lgr->lnk[i], false); in smc_lgr_free()
1394 up_write(&lgr->llc_conf_mutex); in smc_lgr_free()
1395 smc_llc_lgr_clear(lgr); in smc_lgr_free()
1398 destroy_workqueue(lgr->tx_wq); in smc_lgr_free()
1399 if (lgr->is_smcd) { in smc_lgr_free()
1400 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id); in smc_lgr_free()
1401 put_device(lgr->smcd->ops->get_dev(lgr->smcd)); in smc_lgr_free()
1403 smc_lgr_put(lgr); /* theoretically last lgr_put */ in smc_lgr_free()
1406 void smc_lgr_hold(struct smc_link_group *lgr) in smc_lgr_hold() argument
1408 refcount_inc(&lgr->refcnt); in smc_lgr_hold()
1411 void smc_lgr_put(struct smc_link_group *lgr) in smc_lgr_put() argument
1413 if (refcount_dec_and_test(&lgr->refcnt)) in smc_lgr_put()
1414 __smc_lgr_free(lgr); in smc_lgr_put()
1429 if (conn->lgr->is_smcd && conn->lgr->peer_shutdown) in smc_conn_kill()
1436 if (conn->lgr->is_smcd) { in smc_conn_kill()
1449 static void smc_lgr_cleanup(struct smc_link_group *lgr) in smc_lgr_cleanup() argument
1451 if (lgr->is_smcd) { in smc_lgr_cleanup()
1452 smc_ism_signal_shutdown(lgr); in smc_lgr_cleanup()
1454 u32 rsn = lgr->llc_termination_rsn; in smc_lgr_cleanup()
1458 smc_llc_send_link_delete_all(lgr, false, rsn); in smc_lgr_cleanup()
1459 smcr_lgr_link_deactivate_all(lgr); in smc_lgr_cleanup()
1467 static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft) in __smc_lgr_terminate() argument
1473 if (lgr->terminating) in __smc_lgr_terminate()
1476 cancel_delayed_work(&lgr->free_work); in __smc_lgr_terminate()
1477 lgr->terminating = 1; in __smc_lgr_terminate()
1480 read_lock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1481 node = rb_first(&lgr->conns_all); in __smc_lgr_terminate()
1483 read_unlock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1491 read_lock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1492 node = rb_first(&lgr->conns_all); in __smc_lgr_terminate()
1494 read_unlock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1495 smc_lgr_cleanup(lgr); in __smc_lgr_terminate()
1496 smc_lgr_free(lgr); in __smc_lgr_terminate()
1500 void smc_lgr_terminate_sched(struct smc_link_group *lgr) in smc_lgr_terminate_sched() argument
1504 smc_lgr_list_head(lgr, &lgr_lock); in smc_lgr_terminate_sched()
1506 if (list_empty(&lgr->list) || lgr->terminating || lgr->freeing) { in smc_lgr_terminate_sched()
1510 list_del_init(&lgr->list); in smc_lgr_terminate_sched()
1511 lgr->freeing = 1; in smc_lgr_terminate_sched()
1513 schedule_work(&lgr->terminate_work); in smc_lgr_terminate_sched()
1519 struct smc_link_group *lgr, *l; in smc_smcd_terminate() local
1524 list_for_each_entry_safe(lgr, l, &dev->lgr_list, list) { in smc_smcd_terminate()
1525 if ((!peer_gid || lgr->peer_gid == peer_gid) && in smc_smcd_terminate()
1526 (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) { in smc_smcd_terminate()
1528 lgr->peer_shutdown = 1; in smc_smcd_terminate()
1529 list_move(&lgr->list, &lgr_free_list); in smc_smcd_terminate()
1530 lgr->freeing = 1; in smc_smcd_terminate()
1536 list_for_each_entry_safe(lgr, l, &lgr_free_list, list) { in smc_smcd_terminate()
1537 list_del_init(&lgr->list); in smc_smcd_terminate()
1538 schedule_work(&lgr->terminate_work); in smc_smcd_terminate()
1545 struct smc_link_group *lgr, *lg; in smc_smcd_terminate_all() local
1550 list_for_each_entry(lgr, &lgr_free_list, list) in smc_smcd_terminate_all()
1551 lgr->freeing = 1; in smc_smcd_terminate_all()
1554 list_for_each_entry_safe(lgr, lg, &lgr_free_list, list) { in smc_smcd_terminate_all()
1555 list_del_init(&lgr->list); in smc_smcd_terminate_all()
1556 __smc_lgr_terminate(lgr, false); in smc_smcd_terminate_all()
1569 struct smc_link_group *lgr, *lg; in smc_smcr_terminate_all() local
1576 list_for_each_entry(lgr, &lgr_free_list, list) in smc_smcr_terminate_all()
1577 lgr->freeing = 1; in smc_smcr_terminate_all()
1579 list_for_each_entry_safe(lgr, lg, &smc_lgr_list.list, list) { in smc_smcr_terminate_all()
1581 if (lgr->lnk[i].smcibdev == smcibdev) in smc_smcr_terminate_all()
1582 smcr_link_down_cond_sched(&lgr->lnk[i]); in smc_smcr_terminate_all()
1588 list_for_each_entry_safe(lgr, lg, &lgr_free_list, list) { in smc_smcr_terminate_all()
1589 list_del_init(&lgr->list); in smc_smcr_terminate_all()
1590 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_OP_INIT_TERM); in smc_smcr_terminate_all()
1591 __smc_lgr_terminate(lgr, false); in smc_smcr_terminate_all()
1605 void smcr_lgr_set_type(struct smc_link_group *lgr, enum smc_lgr_type new_type) in smcr_lgr_set_type() argument
1611 if (smc_link_usable(&lgr->lnk[i])) in smcr_lgr_set_type()
1612 lgr->lnk[i].link_is_asym = false; in smcr_lgr_set_type()
1613 if (lgr->type == new_type) in smcr_lgr_set_type()
1615 lgr->type = new_type; in smcr_lgr_set_type()
1617 switch (lgr->type) { in smcr_lgr_set_type()
1635 "%s, pnetid %.16s\n", SMC_LGR_ID_SIZE, &lgr->id, in smcr_lgr_set_type()
1636 lgr->net->net_cookie, lgr_type, lgr->pnet_id); in smcr_lgr_set_type()
1640 void smcr_lgr_set_type_asym(struct smc_link_group *lgr, in smcr_lgr_set_type_asym() argument
1643 smcr_lgr_set_type(lgr, new_type); in smcr_lgr_set_type_asym()
1644 lgr->lnk[asym_lnk_idx].link_is_asym = true; in smcr_lgr_set_type_asym()
1663 struct smc_link_group *lgr, *n; in smcr_port_add() local
1666 list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) { in smcr_port_add()
1669 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, in smcr_port_add()
1671 lgr->type == SMC_LGR_SYMMETRIC || in smcr_port_add()
1672 lgr->type == SMC_LGR_ASYMMETRIC_PEER || in smcr_port_add()
1673 !rdma_dev_access_netns(smcibdev->ibdev, lgr->net)) in smcr_port_add()
1676 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) in smcr_port_add()
1680 link = smc_llc_usable_link(lgr); in smcr_port_add()
1692 struct smc_link_group *lgr = lnk->lgr; in smcr_link_down() local
1696 if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list)) in smcr_link_down()
1699 to_lnk = smc_switch_conns(lgr, lnk, true); in smcr_link_down()
1704 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); in smcr_link_down()
1707 if (lgr->role == SMC_SERV) { in smcr_link_down()
1711 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { in smcr_link_down()
1713 up_write(&lgr->llc_conf_mutex); in smcr_link_down()
1714 wait_event_timeout(lgr->llc_flow_waiter, in smcr_link_down()
1715 (list_empty(&lgr->list) || in smcr_link_down()
1716 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE), in smcr_link_down()
1718 down_write(&lgr->llc_conf_mutex); in smcr_link_down()
1720 if (!list_empty(&lgr->list)) { in smcr_link_down()
1726 wake_up(&lgr->llc_flow_waiter); /* wake up next waiter */ in smcr_link_down()
1750 struct smc_link_group *lgr, *n; in smcr_port_err() local
1753 list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) { in smcr_port_err()
1754 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, in smcr_port_err()
1757 if (list_empty(&lgr->list)) in smcr_port_err()
1760 struct smc_link *lnk = &lgr->lnk[i]; in smcr_port_err()
1773 struct smc_link_group *lgr = link->lgr; in smc_link_down_work() local
1775 if (list_empty(&lgr->list)) in smc_link_down_work()
1777 wake_up_all(&lgr->llc_msg_waiter); in smc_link_down_work()
1778 down_write(&lgr->llc_conf_mutex); in smc_link_down_work()
1780 up_write(&lgr->llc_conf_mutex); in smc_link_down_work()
1831 static bool smcr_lgr_match(struct smc_link_group *lgr, u8 smcr_version, in smcr_lgr_match() argument
1841 if (memcmp(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN) || in smcr_lgr_match()
1842 lgr->role != role) in smcr_lgr_match()
1846 lnk = &lgr->lnk[i]; in smcr_lgr_match()
1853 if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) && in smcr_lgr_match()
1862 static bool smcd_lgr_match(struct smc_link_group *lgr, in smcd_lgr_match() argument
1865 return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev; in smcd_lgr_match()
1874 struct smc_link_group *lgr; in smc_conn_create() local
1891 list_for_each_entry(lgr, lgr_list, list) { in smc_conn_create()
1892 write_lock_bh(&lgr->conns_lock); in smc_conn_create()
1894 smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected], in smc_conn_create()
1896 smcr_lgr_match(lgr, ini->smcr_version, in smc_conn_create()
1900 !lgr->sync_err && in smc_conn_create()
1902 lgr->vlan_id == ini->vlan_id) && in smc_conn_create()
1904 (lgr->conns_num < lgr->max_conns && in smc_conn_create()
1905 !bitmap_full(lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX)))) { in smc_conn_create()
1908 conn->lgr = lgr; in smc_conn_create()
1910 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1911 if (!rc && delayed_work_pending(&lgr->free_work)) in smc_conn_create()
1912 cancel_delayed_work(&lgr->free_work); in smc_conn_create()
1915 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1935 lgr = conn->lgr; in smc_conn_create()
1936 write_lock_bh(&lgr->conns_lock); in smc_conn_create()
1938 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1940 smc_lgr_cleanup_early(lgr); in smc_conn_create()
1944 smc_lgr_hold(conn->lgr); /* lgr_put in smc_conn_free() */ in smc_conn_create()
1945 if (!conn->lgr->is_smcd) in smc_conn_create()
2112 if (list_empty(&link->lgr->list)) in smcr_link_reg_buf()
2150 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_map_lgr() local
2154 rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock, in smcr_buf_map_lgr()
2155 &lgr->rmbs[i], true); in smcr_buf_map_lgr()
2158 rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock, in smcr_buf_map_lgr()
2159 &lgr->sndbufs[i], false); in smcr_buf_map_lgr()
2171 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_reg_lgr() local
2176 down_write(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
2178 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) { in smcr_buf_reg_lgr()
2183 up_write(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
2188 up_write(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
2190 if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) in smcr_buf_reg_lgr()
2194 down_write(&lgr->sndbufs_lock); in smcr_buf_reg_lgr()
2196 list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], list) { in smcr_buf_reg_lgr()
2201 up_write(&lgr->sndbufs_lock); in smcr_buf_reg_lgr()
2206 up_write(&lgr->sndbufs_lock); in smcr_buf_reg_lgr()
2210 static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr, in smcr_new_buf_create() argument
2220 switch (lgr->buf_type) { in smcr_new_buf_create()
2235 if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) in smcr_new_buf_create()
2258 static int smcr_buf_map_usable_links(struct smc_link_group *lgr, in smcr_buf_map_usable_links() argument
2264 down_read(&lgr->llc_conf_mutex); in smcr_buf_map_usable_links()
2266 struct smc_link *lnk = &lgr->lnk[i]; in smcr_buf_map_usable_links()
2277 up_read(&lgr->llc_conf_mutex); in smcr_buf_map_usable_links()
2283 static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr, in smcd_new_buf_create() argument
2294 rc = smc_ism_register_dmb(lgr, bufsize, buf_desc); in smcd_new_buf_create()
2323 struct smc_link_group *lgr = conn->lgr; in __smc_buf_create() local
2339 lock = &lgr->rmbs_lock; in __smc_buf_create()
2340 buf_list = &lgr->rmbs[bufsize_comp]; in __smc_buf_create()
2342 lock = &lgr->sndbufs_lock; in __smc_buf_create()
2343 buf_list = &lgr->sndbufs[bufsize_comp]; in __smc_buf_create()
2357 buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize); in __smc_buf_create()
2359 buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize); in __smc_buf_create()
2384 if (smcr_buf_map_usable_links(lgr, buf_desc, is_rmb)) { in __smc_buf_create()
2385 smcr_buf_unuse(buf_desc, is_rmb, lgr); in __smc_buf_create()
2411 if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd || in smc_sndbuf_sync_sg_for_device()
2423 if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd) in smc_rmb_sync_sg_for_cpu()
2426 if (!smc_link_active(&conn->lgr->lnk[i])) in smc_rmb_sync_sg_for_cpu()
2428 smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc, in smc_rmb_sync_sg_for_cpu()
2450 down_write(&smc->conn.lgr->sndbufs_lock); in smc_buf_create()
2452 up_write(&smc->conn.lgr->sndbufs_lock); in smc_buf_create()
2453 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc); in smc_buf_create()
2459 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr) in smc_rmb_reserve_rtoken_idx() argument
2463 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { in smc_rmb_reserve_rtoken_idx()
2464 if (!test_and_set_bit(i, lgr->rtokens_used_mask)) in smc_rmb_reserve_rtoken_idx()
2470 static int smc_rtoken_find_by_link(struct smc_link_group *lgr, int lnk_idx, in smc_rtoken_find_by_link() argument
2476 if (test_bit(i, lgr->rtokens_used_mask) && in smc_rtoken_find_by_link()
2477 lgr->rtokens[i][lnk_idx].rkey == rkey) in smc_rtoken_find_by_link()
2484 void smc_rtoken_set(struct smc_link_group *lgr, int link_idx, int link_idx_new, in smc_rtoken_set() argument
2489 rtok_idx = smc_rtoken_find_by_link(lgr, link_idx, ntohl(nw_rkey_known)); in smc_rtoken_set()
2492 lgr->rtokens[rtok_idx][link_idx_new].rkey = ntohl(nw_rkey); in smc_rtoken_set()
2493 lgr->rtokens[rtok_idx][link_idx_new].dma_addr = be64_to_cpu(nw_vaddr); in smc_rtoken_set()
2497 void smc_rtoken_set2(struct smc_link_group *lgr, int rtok_idx, int link_id, in smc_rtoken_set2() argument
2506 if (lgr->lnk[link_idx].link_id == link_id) { in smc_rtoken_set2()
2513 lgr->rtokens[rtok_idx][link_idx].rkey = rkey; in smc_rtoken_set2()
2514 lgr->rtokens[rtok_idx][link_idx].dma_addr = dma_addr; in smc_rtoken_set2()
2520 struct smc_link_group *lgr = smc_get_lgr(lnk); in smc_rtoken_add() local
2526 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_add()
2527 lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr && in smc_rtoken_add()
2528 test_bit(i, lgr->rtokens_used_mask)) { in smc_rtoken_add()
2533 i = smc_rmb_reserve_rtoken_idx(lgr); in smc_rtoken_add()
2536 lgr->rtokens[i][lnk->link_idx].rkey = rkey; in smc_rtoken_add()
2537 lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr; in smc_rtoken_add()
2544 struct smc_link_group *lgr = smc_get_lgr(lnk); in smc_rtoken_delete() local
2549 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_delete()
2550 test_bit(i, lgr->rtokens_used_mask)) { in smc_rtoken_delete()
2552 lgr->rtokens[i][j].rkey = 0; in smc_rtoken_delete()
2553 lgr->rtokens[i][j].dma_addr = 0; in smc_rtoken_delete()
2555 clear_bit(i, lgr->rtokens_used_mask); in smc_rtoken_delete()