136 	rc = ib_req_notify_cq(lnk->smcibdev->roce_cq_recv,  in smc_ib_ready_link()
155 static int smc_ib_fill_mac(struct smc_ib_device *smcibdev, u8 ibport)  in smc_ib_fill_mac()  argument
160 	attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, 0);  in smc_ib_fill_mac()
164 	rc = rdma_read_gid_l2_fields(attr, NULL, smcibdev->mac[ibport - 1]);  in smc_ib_fill_mac()
174 static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev,  in smc_ib_define_local_systemid()  argument
177 	memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1],  in smc_ib_define_local_systemid()
178 	       sizeof(smcibdev->mac[ibport - 1]));  in smc_ib_define_local_systemid()
191 bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)  in smc_ib_port_active()  argument
193 	return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;  in smc_ib_port_active()
268 int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport,  in smc_ib_determine_gid()  argument
276 	for (i = 0; i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) {  in smc_ib_determine_gid()
277 		attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, i);  in smc_ib_determine_gid()
302 				  struct smc_ib_device *smcibdev, u8 ibport)  in smc_ib_check_link_gid()  argument
308 	for (i = 0; !rc && i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) {  in smc_ib_check_link_gid()
309 		attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, i);  in smc_ib_check_link_gid()
327 static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport)  in smc_ib_gid_check()  argument
334 		if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id,  in smc_ib_gid_check()
341 			    lgr->lnk[i].smcibdev != smcibdev)  in smc_ib_gid_check()
345 						   smcibdev, ibport))  in smc_ib_gid_check()
346 				smcr_port_err(smcibdev, ibport);  in smc_ib_gid_check()
352 static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)  in smc_ib_remember_port_attr()  argument
356 	memset(&smcibdev->pattr[ibport - 1], 0,  in smc_ib_remember_port_attr()
357 	       sizeof(smcibdev->pattr[ibport - 1]));  in smc_ib_remember_port_attr()
358 	rc = ib_query_port(smcibdev->ibdev, ibport,  in smc_ib_remember_port_attr()
359 			   &smcibdev->pattr[ibport - 1]);  in smc_ib_remember_port_attr()
363 	rc = smc_ib_fill_mac(smcibdev, ibport);  in smc_ib_remember_port_attr()
367 	    smc_ib_port_active(smcibdev, ibport))  in smc_ib_remember_port_attr()
369 		smc_ib_define_local_systemid(smcibdev, ibport);  in smc_ib_remember_port_attr()
377 	struct smc_ib_device *smcibdev = container_of(  in smc_ib_port_event_work()  local
381 	for_each_set_bit(port_idx, &smcibdev->port_event_mask, SMC_MAX_PORTS) {  in smc_ib_port_event_work()
382 		smc_ib_remember_port_attr(smcibdev, port_idx + 1);  in smc_ib_port_event_work()
383 		clear_bit(port_idx, &smcibdev->port_event_mask);  in smc_ib_port_event_work()
384 		if (!smc_ib_port_active(smcibdev, port_idx + 1)) {  in smc_ib_port_event_work()
385 			set_bit(port_idx, smcibdev->ports_going_away);  in smc_ib_port_event_work()
386 			smcr_port_err(smcibdev, port_idx + 1);  in smc_ib_port_event_work()
388 			clear_bit(port_idx, smcibdev->ports_going_away);  in smc_ib_port_event_work()
389 			smcr_port_add(smcibdev, port_idx + 1);  in smc_ib_port_event_work()
390 			smc_ib_gid_check(smcibdev, port_idx + 1);  in smc_ib_port_event_work()
399 	struct smc_ib_device *smcibdev;  in smc_ib_global_event_handler()  local
403 	smcibdev = container_of(handler, struct smc_ib_device, event_handler);  in smc_ib_global_event_handler()
409 			set_bit(port_idx, &smcibdev->port_event_mask);  in smc_ib_global_event_handler()
411 					      smcibdev->ports_going_away))  in smc_ib_global_event_handler()
415 			schedule_work(&smcibdev->port_event_work);  in smc_ib_global_event_handler()
421 		set_bit(port_idx, &smcibdev->port_event_mask);  in smc_ib_global_event_handler()
422 		if (test_and_clear_bit(port_idx, smcibdev->ports_going_away))  in smc_ib_global_event_handler()
423 			schedule_work(&smcibdev->port_event_work);  in smc_ib_global_event_handler()
429 		set_bit(port_idx, &smcibdev->port_event_mask);  in smc_ib_global_event_handler()
430 		if (!test_and_set_bit(port_idx, smcibdev->ports_going_away))  in smc_ib_global_event_handler()
431 			schedule_work(&smcibdev->port_event_work);  in smc_ib_global_event_handler()
437 		set_bit(port_idx, &smcibdev->port_event_mask);  in smc_ib_global_event_handler()
438 		schedule_work(&smcibdev->port_event_work);  in smc_ib_global_event_handler()
456 	lnk->roce_pd = ib_alloc_pd(lnk->smcibdev->ibdev, 0);  in smc_ib_create_protection_domain()
464 				      struct smc_ib_device *smcibdev)  in smcr_diag_is_dev_critical()  argument
476 			    lgr->lnk[i].smcibdev != smcibdev)  in smcr_diag_is_dev_critical()
492 				  struct smc_ib_device *smcibdev,  in smc_nl_handle_dev_port()  argument
505 		       smcibdev->pnetid_by_user[port]))  in smc_nl_handle_dev_port()
507 	memcpy(smc_pnet, &smcibdev->pnetid[port], SMC_MAX_PNETID_LEN);  in smc_nl_handle_dev_port()
512 			smcibdev->ndev_ifidx[port]))  in smc_nl_handle_dev_port()
516 	port_state = smc_ib_port_active(smcibdev, port + 1);  in smc_nl_handle_dev_port()
519 	lnk_count = atomic_read(&smcibdev->lnk_cnt_by_port[port]);  in smc_nl_handle_dev_port()
546 static int smc_nl_handle_smcr_dev(struct smc_ib_device *smcibdev,  in smc_nl_handle_smcr_dev()  argument
566 	is_crit = smcr_diag_is_dev_critical(&smc_lgr_list, smcibdev);  in smc_nl_handle_smcr_dev()
569 	if (smcibdev->ibdev->dev.parent) {  in smc_nl_handle_smcr_dev()
571 		pci_dev = to_pci_dev(smcibdev->ibdev->dev.parent);  in smc_nl_handle_smcr_dev()
576 	snprintf(smc_ibname, sizeof(smc_ibname), "%s", smcibdev->ibdev->name);  in smc_nl_handle_smcr_dev()
580 		if (!rdma_is_port_valid(smcibdev->ibdev, i))  in smc_nl_handle_smcr_dev()
582 		if (smc_nl_handle_dev_port(skb, smcibdev->ibdev,  in smc_nl_handle_smcr_dev()
583 					   smcibdev, i - 1))  in smc_nl_handle_smcr_dev()
604 	struct smc_ib_device *smcibdev;  in smc_nl_prep_smcr_dev()  local
609 	list_for_each_entry(smcibdev, &dev_list->list, list) {  in smc_nl_prep_smcr_dev()
612 		if (smc_nl_handle_smcr_dev(smcibdev, skb, cb))  in smc_nl_prep_smcr_dev()
631 	struct smc_ib_device *smcibdev = lnk->smcibdev;  in smc_ib_qp_event_handler()  local
640 		set_bit(port_idx, &smcibdev->port_event_mask);  in smc_ib_qp_event_handler()
641 		if (!test_and_set_bit(port_idx, smcibdev->ports_going_away))  in smc_ib_qp_event_handler()
642 			schedule_work(&smcibdev->port_event_work);  in smc_ib_qp_event_handler()
663 		.send_cq = lnk->smcibdev->roce_cq_send,  in smc_ib_create_queue_pair()
664 		.recv_cq = lnk->smcibdev->roce_cq_recv,  in smc_ib_create_queue_pair()
745 		if (dma_need_sync(lnk->smcibdev->ibdev->dma_device,  in smc_ib_is_sg_need_sync()
772 		ib_dma_sync_single_for_cpu(lnk->smcibdev->ibdev,  in smc_ib_sync_sg_for_cpu()
795 		ib_dma_sync_single_for_device(lnk->smcibdev->ibdev,  in smc_ib_sync_sg_for_device()
809 	mapped_nents = ib_dma_map_sg(lnk->smcibdev->ibdev,  in smc_ib_buf_map_sg()
826 	ib_dma_unmap_sg(lnk->smcibdev->ibdev,  in smc_ib_buf_unmap_sg()
833 long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev)  in smc_ib_setup_per_ibdev()  argument
840 	mutex_lock(&smcibdev->mutex);  in smc_ib_setup_per_ibdev()
842 	if (smcibdev->initialized)  in smc_ib_setup_per_ibdev()
849 	smcibdev->roce_cq_send = ib_create_cq(smcibdev->ibdev,  in smc_ib_setup_per_ibdev()
851 					      smcibdev, &cqattr);  in smc_ib_setup_per_ibdev()
852 	rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_send);  in smc_ib_setup_per_ibdev()
853 	if (IS_ERR(smcibdev->roce_cq_send)) {  in smc_ib_setup_per_ibdev()
854 		smcibdev->roce_cq_send = NULL;  in smc_ib_setup_per_ibdev()
857 	smcibdev->roce_cq_recv = ib_create_cq(smcibdev->ibdev,  in smc_ib_setup_per_ibdev()
859 					      smcibdev, &cqattr);  in smc_ib_setup_per_ibdev()
860 	rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_recv);  in smc_ib_setup_per_ibdev()
861 	if (IS_ERR(smcibdev->roce_cq_recv)) {  in smc_ib_setup_per_ibdev()
862 		smcibdev->roce_cq_recv = NULL;  in smc_ib_setup_per_ibdev()
865 	smc_wr_add_dev(smcibdev);  in smc_ib_setup_per_ibdev()
866 	smcibdev->initialized = 1;  in smc_ib_setup_per_ibdev()
870 	ib_destroy_cq(smcibdev->roce_cq_send);  in smc_ib_setup_per_ibdev()
872 	mutex_unlock(&smcibdev->mutex);  in smc_ib_setup_per_ibdev()
876 static void smc_ib_cleanup_per_ibdev(struct smc_ib_device *smcibdev)  in smc_ib_cleanup_per_ibdev()  argument
878 	mutex_lock(&smcibdev->mutex);  in smc_ib_cleanup_per_ibdev()
879 	if (!smcibdev->initialized)  in smc_ib_cleanup_per_ibdev()
881 	smcibdev->initialized = 0;  in smc_ib_cleanup_per_ibdev()
882 	ib_destroy_cq(smcibdev->roce_cq_recv);  in smc_ib_cleanup_per_ibdev()
883 	ib_destroy_cq(smcibdev->roce_cq_send);  in smc_ib_cleanup_per_ibdev()
884 	smc_wr_remove_dev(smcibdev);  in smc_ib_cleanup_per_ibdev()
886 	mutex_unlock(&smcibdev->mutex);  in smc_ib_cleanup_per_ibdev()
891 static void smc_copy_netdev_ifindex(struct smc_ib_device *smcibdev, int port)  in smc_copy_netdev_ifindex()  argument
893 	struct ib_device *ibdev = smcibdev->ibdev;  in smc_copy_netdev_ifindex()
900 		smcibdev->ndev_ifidx[port] = ndev->ifindex;  in smc_copy_netdev_ifindex()
907 	struct smc_ib_device *smcibdev;  in smc_ib_ndev_change()  local
914 	list_for_each_entry(smcibdev, &smc_ib_devices.list, list) {  in smc_ib_ndev_change()
915 		port_cnt = smcibdev->ibdev->phys_port_cnt;  in smc_ib_ndev_change()
917 			libdev = smcibdev->ibdev;  in smc_ib_ndev_change()
925 				smcibdev->ndev_ifidx[i] = ndev->ifindex;  in smc_ib_ndev_change()
927 				smcibdev->ndev_ifidx[i] = 0;  in smc_ib_ndev_change()
936 	struct smc_ib_device *smcibdev;  in smc_ib_add_dev()  local
943 	smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL);  in smc_ib_add_dev()
944 	if (!smcibdev)  in smc_ib_add_dev()
947 	smcibdev->ibdev = ibdev;  in smc_ib_add_dev()
948 	INIT_WORK(&smcibdev->port_event_work, smc_ib_port_event_work);  in smc_ib_add_dev()
949 	atomic_set(&smcibdev->lnk_cnt, 0);  in smc_ib_add_dev()
950 	init_waitqueue_head(&smcibdev->lnks_deleted);  in smc_ib_add_dev()
951 	mutex_init(&smcibdev->mutex);  in smc_ib_add_dev()
953 	list_add_tail(&smcibdev->list, &smc_ib_devices.list);  in smc_ib_add_dev()
955 	ib_set_client_data(ibdev, &smc_ib_client, smcibdev);  in smc_ib_add_dev()
956 	INIT_IB_EVENT_HANDLER(&smcibdev->event_handler, smcibdev->ibdev,  in smc_ib_add_dev()
958 	ib_register_event_handler(&smcibdev->event_handler);  in smc_ib_add_dev()
961 	port_cnt = smcibdev->ibdev->phys_port_cnt;  in smc_ib_add_dev()
963 			    smcibdev->ibdev->name, port_cnt);  in smc_ib_add_dev()
967 		set_bit(i, &smcibdev->port_event_mask);  in smc_ib_add_dev()
970 					   smcibdev->pnetid[i]))  in smc_ib_add_dev()
971 			smc_pnetid_by_table_ib(smcibdev, i + 1);  in smc_ib_add_dev()
972 		smc_copy_netdev_ifindex(smcibdev, i);  in smc_ib_add_dev()
975 				    smcibdev->ibdev->name, i + 1,  in smc_ib_add_dev()
976 				    smcibdev->pnetid[i],  in smc_ib_add_dev()
977 				    smcibdev->pnetid_by_user[i] ?  in smc_ib_add_dev()
981 	schedule_work(&smcibdev->port_event_work);  in smc_ib_add_dev()
988 	struct smc_ib_device *smcibdev = client_data;  in smc_ib_remove_dev()  local
991 	list_del_init(&smcibdev->list); /* remove from smc_ib_devices */  in smc_ib_remove_dev()
994 			    smcibdev->ibdev->name);  in smc_ib_remove_dev()
995 	smc_smcr_terminate_all(smcibdev);  in smc_ib_remove_dev()
996 	smc_ib_cleanup_per_ibdev(smcibdev);  in smc_ib_remove_dev()
997 	ib_unregister_event_handler(&smcibdev->event_handler);  in smc_ib_remove_dev()
998 	cancel_work_sync(&smcibdev->port_event_work);  in smc_ib_remove_dev()
999 	kfree(smcibdev);  in smc_ib_remove_dev()