42 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
45 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)  in smc_lgr_schedule_free_work()  argument
51 	mod_delayed_work(system_wq, &lgr->free_work,  in smc_lgr_schedule_free_work()
52 			 (!lgr->is_smcd && lgr->role == SMC_CLNT) ?  in smc_lgr_schedule_free_work()
56 void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr)  in smc_lgr_schedule_free_work_fast()  argument
58 	mod_delayed_work(system_wq, &lgr->free_work, SMC_LGR_FREE_DELAY_FAST);  in smc_lgr_schedule_free_work_fast()
72 	link = &conn->lgr->conns_all.rb_node;  in smc_lgr_add_alert_token()
85 	rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);  in smc_lgr_add_alert_token()
104 		if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))  in smc_lgr_register_conn()
108 	conn->lgr->conns_num++;  in smc_lgr_register_conn()
116 	struct smc_link_group *lgr = conn->lgr;  in __smc_lgr_unregister_conn()  local
118 	rb_erase(&conn->alert_node, &lgr->conns_all);  in __smc_lgr_unregister_conn()
119 	lgr->conns_num--;  in __smc_lgr_unregister_conn()
128 	struct smc_link_group *lgr = conn->lgr;  in smc_lgr_unregister_conn()  local
130 	if (!lgr)  in smc_lgr_unregister_conn()
132 	write_lock_bh(&lgr->conns_lock);  in smc_lgr_unregister_conn()
136 	write_unlock_bh(&lgr->conns_lock);  in smc_lgr_unregister_conn()
153 static void smc_lgr_free(struct smc_link_group *lgr);
157 	struct smc_link_group *lgr = container_of(to_delayed_work(work),  in smc_lgr_free_work()  local
163 	read_lock_bh(&lgr->conns_lock);  in smc_lgr_free_work()
164 	conns = RB_EMPTY_ROOT(&lgr->conns_all);  in smc_lgr_free_work()
165 	read_unlock_bh(&lgr->conns_lock);  in smc_lgr_free_work()
170 	if (!list_empty(&lgr->list))  in smc_lgr_free_work()
171 		list_del_init(&lgr->list); /* remove from smc_lgr_list */  in smc_lgr_free_work()
174 	if (!lgr->is_smcd && !lgr->terminating)	{  in smc_lgr_free_work()
175 		struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];  in smc_lgr_free_work()
181 			smc_lgr_schedule_free_work(lgr);  in smc_lgr_free_work()
186 	if (!delayed_work_pending(&lgr->free_work)) {  in smc_lgr_free_work()
187 		struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];  in smc_lgr_free_work()
189 		if (!lgr->is_smcd && lnk->state != SMC_LNK_INACTIVE)  in smc_lgr_free_work()
191 		if (lgr->is_smcd)  in smc_lgr_free_work()
192 			smc_ism_signal_shutdown(lgr);  in smc_lgr_free_work()
193 		smc_lgr_free(lgr);  in smc_lgr_free_work()
200 	struct smc_link_group *lgr;  in smc_lgr_create()  local
213 	lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);  in smc_lgr_create()
214 	if (!lgr) {  in smc_lgr_create()
218 	lgr->is_smcd = ini->is_smcd;  in smc_lgr_create()
219 	lgr->sync_err = 0;  in smc_lgr_create()
220 	lgr->vlan_id = ini->vlan_id;  in smc_lgr_create()
221 	rwlock_init(&lgr->sndbufs_lock);  in smc_lgr_create()
222 	rwlock_init(&lgr->rmbs_lock);  in smc_lgr_create()
223 	rwlock_init(&lgr->conns_lock);  in smc_lgr_create()
225 		INIT_LIST_HEAD(&lgr->sndbufs[i]);  in smc_lgr_create()
226 		INIT_LIST_HEAD(&lgr->rmbs[i]);  in smc_lgr_create()
229 	memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE);  in smc_lgr_create()
230 	INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);  in smc_lgr_create()
231 	lgr->conns_all = RB_ROOT;  in smc_lgr_create()
234 		lgr->peer_gid = ini->ism_gid;  in smc_lgr_create()
235 		lgr->smcd = ini->ism_dev;  in smc_lgr_create()
238 		lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;  in smc_lgr_create()
239 		memcpy(lgr->peer_systemid, ini->ib_lcl->id_for_peer,  in smc_lgr_create()
242 		lnk = &lgr->lnk[SMC_SINGLE_LINK];  in smc_lgr_create()
276 	smc->conn.lgr = lgr;  in smc_lgr_create()
278 	list_add(&lgr->list, &smc_lgr_list.list);  in smc_lgr_create()
291 	kfree(lgr);  in smc_lgr_create()
306 			  struct smc_link_group *lgr)  in smc_buf_unuse()  argument
312 			if (!lgr->is_smcd) {  in smc_buf_unuse()
315 						&lgr->lnk[SMC_SINGLE_LINK],  in smc_buf_unuse()
321 			write_lock_bh(&lgr->rmbs_lock);  in smc_buf_unuse()
323 			write_unlock_bh(&lgr->rmbs_lock);  in smc_buf_unuse()
325 			smc_buf_free(lgr, true, conn->rmb_desc);  in smc_buf_unuse()
333 	struct smc_link_group *lgr = conn->lgr;  in smc_conn_free()  local
335 	if (!lgr)  in smc_conn_free()
337 	if (lgr->is_smcd) {  in smc_conn_free()
344 	smc_buf_unuse(conn, lgr);		/* allow buffer reuse */  in smc_conn_free()
345 	conn->lgr = NULL;  in smc_conn_free()
347 	if (!lgr->conns_num)  in smc_conn_free()
348 		smc_lgr_schedule_free_work(lgr);  in smc_conn_free()
362 static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb,  in smcr_buf_free()  argument
365 	struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];  in smcr_buf_free()
383 static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb,  in smcd_buf_free()  argument
389 		smc_ism_unregister_dmb(lgr->smcd, buf_desc);  in smcd_buf_free()
396 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,  in smc_buf_free()  argument
399 	if (lgr->is_smcd)  in smc_buf_free()
400 		smcd_buf_free(lgr, is_rmb, buf_desc);  in smc_buf_free()
402 		smcr_buf_free(lgr, is_rmb, buf_desc);  in smc_buf_free()
405 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)  in __smc_lgr_free_bufs()  argument
413 			buf_list = &lgr->rmbs[i];  in __smc_lgr_free_bufs()
415 			buf_list = &lgr->sndbufs[i];  in __smc_lgr_free_bufs()
419 			smc_buf_free(lgr, is_rmb, buf_desc);  in __smc_lgr_free_bufs()
424 static void smc_lgr_free_bufs(struct smc_link_group *lgr)  in smc_lgr_free_bufs()  argument
427 	__smc_lgr_free_bufs(lgr, false);  in smc_lgr_free_bufs()
429 	__smc_lgr_free_bufs(lgr, true);  in smc_lgr_free_bufs()
433 static void smc_lgr_free(struct smc_link_group *lgr)  in smc_lgr_free()  argument
435 	smc_lgr_free_bufs(lgr);  in smc_lgr_free()
436 	if (lgr->is_smcd)  in smc_lgr_free()
437 		smc_ism_put_vlan(lgr->smcd, lgr->vlan_id);  in smc_lgr_free()
439 		smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);  in smc_lgr_free()
440 	kfree(lgr);  in smc_lgr_free()
443 void smc_lgr_forget(struct smc_link_group *lgr)  in smc_lgr_forget()  argument
447 	if (!list_empty(&lgr->list))  in smc_lgr_forget()
448 		list_del_init(&lgr->list);  in smc_lgr_forget()
453 static void __smc_lgr_terminate(struct smc_link_group *lgr)  in __smc_lgr_terminate()  argument
459 	if (lgr->terminating)  in __smc_lgr_terminate()
461 	lgr->terminating = 1;  in __smc_lgr_terminate()
462 	if (!list_empty(&lgr->list)) /* forget lgr */  in __smc_lgr_terminate()
463 		list_del_init(&lgr->list);  in __smc_lgr_terminate()
464 	if (!lgr->is_smcd)  in __smc_lgr_terminate()
465 		smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]);  in __smc_lgr_terminate()
467 	write_lock_bh(&lgr->conns_lock);  in __smc_lgr_terminate()
468 	node = rb_first(&lgr->conns_all);  in __smc_lgr_terminate()
475 		conn->lgr = NULL;  in __smc_lgr_terminate()
476 		write_unlock_bh(&lgr->conns_lock);  in __smc_lgr_terminate()
479 		write_lock_bh(&lgr->conns_lock);  in __smc_lgr_terminate()
480 		node = rb_first(&lgr->conns_all);  in __smc_lgr_terminate()
482 	write_unlock_bh(&lgr->conns_lock);  in __smc_lgr_terminate()
483 	if (!lgr->is_smcd)  in __smc_lgr_terminate()
484 		wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait);  in __smc_lgr_terminate()
485 	smc_lgr_schedule_free_work(lgr);  in __smc_lgr_terminate()
488 void smc_lgr_terminate(struct smc_link_group *lgr)  in smc_lgr_terminate()  argument
491 	__smc_lgr_terminate(lgr);  in smc_lgr_terminate()
498 	struct smc_link_group *lgr, *l;  in smc_port_terminate()  local
501 	list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {  in smc_port_terminate()
502 		if (!lgr->is_smcd &&  in smc_port_terminate()
503 		    lgr->lnk[SMC_SINGLE_LINK].smcibdev == smcibdev &&  in smc_port_terminate()
504 		    lgr->lnk[SMC_SINGLE_LINK].ibport == ibport)  in smc_port_terminate()
505 			__smc_lgr_terminate(lgr);  in smc_port_terminate()
513 	struct smc_link_group *lgr, *l;  in smc_smcd_terminate()  local
518 	list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {  in smc_smcd_terminate()
519 		if (lgr->is_smcd && lgr->smcd == dev &&  in smc_smcd_terminate()
520 		    (!peer_gid || lgr->peer_gid == peer_gid) &&  in smc_smcd_terminate()
521 		    (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) {  in smc_smcd_terminate()
522 			__smc_lgr_terminate(lgr);  in smc_smcd_terminate()
523 			list_move(&lgr->list, &lgr_free_list);  in smc_smcd_terminate()
529 	list_for_each_entry_safe(lgr, l, &lgr_free_list, list) {  in smc_smcd_terminate()
530 		list_del_init(&lgr->list);  in smc_smcd_terminate()
531 		cancel_delayed_work_sync(&lgr->free_work);  in smc_smcd_terminate()
533 			smc_ism_signal_shutdown(lgr);  in smc_smcd_terminate()
534 		smc_lgr_free(lgr);  in smc_smcd_terminate()
585 static bool smcr_lgr_match(struct smc_link_group *lgr,  in smcr_lgr_match()  argument
589 	return !memcmp(lgr->peer_systemid, lcl->id_for_peer,  in smcr_lgr_match()
591 		!memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,  in smcr_lgr_match()
593 		!memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,  in smcr_lgr_match()
595 		lgr->role == role &&  in smcr_lgr_match()
596 		(lgr->role == SMC_SERV ||  in smcr_lgr_match()
597 		 lgr->lnk[SMC_SINGLE_LINK].peer_qpn == clcqpn);  in smcr_lgr_match()
600 static bool smcd_lgr_match(struct smc_link_group *lgr,  in smcd_lgr_match()  argument
603 	return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev;  in smcd_lgr_match()
610 	struct smc_link_group *lgr;  in smc_conn_create()  local
622 	list_for_each_entry(lgr, &smc_lgr_list.list, list) {  in smc_conn_create()
623 		write_lock_bh(&lgr->conns_lock);  in smc_conn_create()
625 		     smcd_lgr_match(lgr, ini->ism_dev, ini->ism_gid) :  in smc_conn_create()
626 		     smcr_lgr_match(lgr, ini->ib_lcl, role, ini->ib_clcqpn)) &&  in smc_conn_create()
627 		    !lgr->sync_err &&  in smc_conn_create()
628 		    lgr->vlan_id == ini->vlan_id &&  in smc_conn_create()
630 		     lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) {  in smc_conn_create()
633 			conn->lgr = lgr;  in smc_conn_create()
635 			if (delayed_work_pending(&lgr->free_work))  in smc_conn_create()
636 				cancel_delayed_work(&lgr->free_work);  in smc_conn_create()
637 			write_unlock_bh(&lgr->conns_lock);  in smc_conn_create()
640 		write_unlock_bh(&lgr->conns_lock);  in smc_conn_create()
658 		lgr = conn->lgr;  in smc_conn_create()
659 		write_lock_bh(&lgr->conns_lock);  in smc_conn_create()
661 		write_unlock_bh(&lgr->conns_lock);  in smc_conn_create()
734 static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr,  in smcr_new_buf_create()  argument
758 	lnk = &lgr->lnk[SMC_SINGLE_LINK];  in smcr_new_buf_create()
762 		smc_buf_free(lgr, is_rmb, buf_desc);  in smcr_new_buf_create()
773 		smc_buf_free(lgr, is_rmb, buf_desc);  in smcr_new_buf_create()
784 			smc_buf_free(lgr, is_rmb, buf_desc);  in smcr_new_buf_create()
795 static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr,  in smcd_new_buf_create()  argument
809 		rc = smc_ism_register_dmb(lgr, bufsize, buf_desc);  in smcd_new_buf_create()
834 	struct smc_link_group *lgr = conn->lgr;  in __smc_buf_create()  local
851 			lock = &lgr->rmbs_lock;  in __smc_buf_create()
852 			buf_list = &lgr->rmbs[bufsize_short];  in __smc_buf_create()
854 			lock = &lgr->sndbufs_lock;  in __smc_buf_create()
855 			buf_list = &lgr->sndbufs[bufsize_short];  in __smc_buf_create()
869 			buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize);  in __smc_buf_create()
871 			buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize);  in __smc_buf_create()
907 	struct smc_link_group *lgr = conn->lgr;  in smc_sndbuf_sync_sg_for_cpu()  local
909 	if (!conn->lgr || conn->lgr->is_smcd)  in smc_sndbuf_sync_sg_for_cpu()
911 	smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,  in smc_sndbuf_sync_sg_for_cpu()
917 	struct smc_link_group *lgr = conn->lgr;  in smc_sndbuf_sync_sg_for_device()  local
919 	if (!conn->lgr || conn->lgr->is_smcd)  in smc_sndbuf_sync_sg_for_device()
921 	smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,  in smc_sndbuf_sync_sg_for_device()
927 	struct smc_link_group *lgr = conn->lgr;  in smc_rmb_sync_sg_for_cpu()  local
929 	if (!conn->lgr || conn->lgr->is_smcd)  in smc_rmb_sync_sg_for_cpu()
931 	smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,  in smc_rmb_sync_sg_for_cpu()
937 	struct smc_link_group *lgr = conn->lgr;  in smc_rmb_sync_sg_for_device()  local
939 	if (!conn->lgr || conn->lgr->is_smcd)  in smc_rmb_sync_sg_for_device()
941 	smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,  in smc_rmb_sync_sg_for_device()
962 		smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc);  in smc_buf_create()
966 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)  in smc_rmb_reserve_rtoken_idx()  argument
970 	for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {  in smc_rmb_reserve_rtoken_idx()
971 		if (!test_and_set_bit(i, lgr->rtokens_used_mask))  in smc_rmb_reserve_rtoken_idx()
978 int smc_rtoken_add(struct smc_link_group *lgr, __be64 nw_vaddr, __be32 nw_rkey)  in smc_rtoken_add()  argument
985 		if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&  in smc_rtoken_add()
986 		    (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&  in smc_rtoken_add()
987 		    test_bit(i, lgr->rtokens_used_mask)) {  in smc_rtoken_add()
992 	i = smc_rmb_reserve_rtoken_idx(lgr);  in smc_rtoken_add()
995 	lgr->rtokens[i][SMC_SINGLE_LINK].rkey = rkey;  in smc_rtoken_add()
996 	lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = dma_addr;  in smc_rtoken_add()
1001 int smc_rtoken_delete(struct smc_link_group *lgr, __be32 nw_rkey)  in smc_rtoken_delete()  argument
1007 		if (lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey &&  in smc_rtoken_delete()
1008 		    test_bit(i, lgr->rtokens_used_mask)) {  in smc_rtoken_delete()
1009 			lgr->rtokens[i][SMC_SINGLE_LINK].rkey = 0;  in smc_rtoken_delete()
1010 			lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = 0;  in smc_rtoken_delete()
1012 			clear_bit(i, lgr->rtokens_used_mask);  in smc_rtoken_delete()
1023 	conn->rtoken_idx = smc_rtoken_add(conn->lgr, clc->rmb_dma_addr,  in smc_rmb_rtoken_handling()
1033 	struct smc_link_group *lgr, *lg;  in smc_core_exit()  local
1040 	list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) {  in smc_core_exit()
1041 		list_del_init(&lgr->list);  in smc_core_exit()
1042 		if (!lgr->is_smcd) {  in smc_core_exit()
1043 			struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];  in smc_core_exit()
1050 		cancel_delayed_work_sync(&lgr->free_work);  in smc_core_exit()
1051 		if (lgr->is_smcd)  in smc_core_exit()
1052 			smc_ism_signal_shutdown(lgr);  in smc_core_exit()
1053 		smc_lgr_free(lgr); /* free link group */  in smc_core_exit()