203 	struct ntb_transport_ctx *nt;  member
253 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)  argument
309 static int ntb_bus_init(struct ntb_transport_ctx *nt)  in ntb_bus_init()  argument
311 	list_add_tail(&nt->entry, &ntb_transport_list);  in ntb_bus_init()
315 static void ntb_bus_remove(struct ntb_transport_ctx *nt)  in ntb_bus_remove()  argument
319 	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {  in ntb_bus_remove()
326 	list_del(&nt->entry);  in ntb_bus_remove()
346 	struct ntb_transport_ctx *nt;  in ntb_transport_unregister_client_dev()  local
348 	list_for_each_entry(nt, &ntb_transport_list, entry)  in ntb_transport_unregister_client_dev()
349 		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)  in ntb_transport_unregister_client_dev()
367 	struct ntb_transport_ctx *nt;  in ntb_transport_register_client_dev()  local
374 	list_for_each_entry(nt, &ntb_transport_list, entry) {  in ntb_transport_register_client_dev()
377 		node = dev_to_node(&nt->ndev->dev);  in ntb_transport_register_client_dev()
392 		dev->parent = &nt->ndev->dev;  in ntb_transport_register_client_dev()
400 		list_add_tail(&client_dev->entry, &nt->client_devs);  in ntb_transport_register_client_dev()
596 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,  in ntb_transport_setup_qp_mw()  argument
599 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_mw()
601 	struct ntb_dev *ndev = nt->ndev;  in ntb_transport_setup_qp_mw()
608 	mw_count = nt->mw_count;  in ntb_transport_setup_qp_mw()
609 	qp_count = nt->qp_count;  in ntb_transport_setup_qp_mw()
611 	mw_num = QP_TO_MW(nt, qp_num);  in ntb_transport_setup_qp_mw()
612 	mw = &nt->mw_vec[mw_num];  in ntb_transport_setup_qp_mw()
666 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)  in ntb_free_mw()  argument
668 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_free_mw()
669 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_free_mw()
674 	ntb_mw_clear_trans(nt->ndev, PIDX, num_mw);  in ntb_free_mw()
682 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,  in ntb_set_mw()  argument
685 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_set_mw()
686 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_set_mw()
695 	rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align,  in ntb_set_mw()
708 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
733 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
738 	rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr,  in ntb_set_mw()
742 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
774 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup()  local
775 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_qp_link_cleanup()
791 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup_work()  local
795 	if (nt->link_is_up)  in ntb_qp_link_cleanup_work()
805 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)  in ntb_transport_link_cleanup()  argument
811 	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;  in ntb_transport_link_cleanup()
814 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_link_cleanup()
816 			qp = &nt->qp_vec[i];  in ntb_transport_link_cleanup()
822 	if (!nt->link_is_up)  in ntb_transport_link_cleanup()
823 		cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_link_cleanup()
829 	count = ntb_spad_count(nt->ndev);  in ntb_transport_link_cleanup()
831 		ntb_spad_write(nt->ndev, i, 0);  in ntb_transport_link_cleanup()
836 	struct ntb_transport_ctx *nt =  in ntb_transport_link_cleanup_work()  local
839 	ntb_transport_link_cleanup(nt);  in ntb_transport_link_cleanup_work()
844 	struct ntb_transport_ctx *nt = data;  in ntb_transport_event_callback()  local
846 	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)  in ntb_transport_event_callback()
847 		schedule_delayed_work(&nt->link_work, 0);  in ntb_transport_event_callback()
849 		schedule_work(&nt->link_cleanup);  in ntb_transport_event_callback()
854 	struct ntb_transport_ctx *nt =  in ntb_transport_link_work()  local
856 	struct ntb_dev *ndev = nt->ndev;  in ntb_transport_link_work()
863 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
864 		size = nt->mw_vec[i].phys_size;  in ntb_transport_link_work()
876 	ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count);  in ntb_transport_link_work()
878 	ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count);  in ntb_transport_link_work()
890 	if (val != nt->qp_count)  in ntb_transport_link_work()
895 	if (val != nt->mw_count)  in ntb_transport_link_work()
898 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
909 		rc = ntb_set_mw(nt, i, val64);  in ntb_transport_link_work()
914 	nt->link_is_up = true;  in ntb_transport_link_work()
916 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_link_work()
917 		struct ntb_transport_qp *qp = &nt->qp_vec[i];  in ntb_transport_link_work()
919 		ntb_transport_setup_qp_mw(nt, i);  in ntb_transport_link_work()
928 	for (i = 0; i < nt->mw_count; i++)  in ntb_transport_link_work()
929 		ntb_free_mw(nt, i);  in ntb_transport_link_work()
937 		schedule_delayed_work(&nt->link_work,  in ntb_transport_link_work()
947 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_work()  local
950 	WARN_ON(!nt->link_is_up);  in ntb_qp_link_work()
952 	val = ntb_spad_read(nt->ndev, QP_LINKS);  in ntb_qp_link_work()
954 	ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num));  in ntb_qp_link_work()
970 	} else if (nt->link_is_up)  in ntb_qp_link_work()
975 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,  in ntb_transport_init_queue()  argument
985 	mw_count = nt->mw_count;  in ntb_transport_init_queue()
986 	qp_count = nt->qp_count;  in ntb_transport_init_queue()
988 	mw_num = QP_TO_MW(nt, qp_num);  in ntb_transport_init_queue()
990 	qp = &nt->qp_vec[qp_num];  in ntb_transport_init_queue()
992 	qp->transport = nt;  in ntb_transport_init_queue()
993 	qp->ndev = nt->ndev;  in ntb_transport_init_queue()
1003 	mw_base = nt->mw_vec[mw_num].phys_addr;  in ntb_transport_init_queue()
1004 	mw_size = nt->mw_vec[mw_num].phys_size;  in ntb_transport_init_queue()
1012 	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;  in ntb_transport_init_queue()
1027 	if (nt->debugfs_node_dir) {  in ntb_transport_init_queue()
1032 						     nt->debugfs_node_dir);  in ntb_transport_init_queue()
1061 	struct ntb_transport_ctx *nt;  in ntb_transport_probe()  local
1087 	nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);  in ntb_transport_probe()
1088 	if (!nt)  in ntb_transport_probe()
1091 	nt->ndev = ndev;  in ntb_transport_probe()
1097 		nt->mw_count = 0;  in ntb_transport_probe()
1103 	nt->mw_count = min(mw_count, max_mw_count_for_spads);  in ntb_transport_probe()
1105 	nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec),  in ntb_transport_probe()
1107 	if (!nt->mw_vec) {  in ntb_transport_probe()
1113 		mw = &nt->mw_vec[i];  in ntb_transport_probe()
1137 	else if (nt->mw_count < qp_count)  in ntb_transport_probe()
1138 		qp_count = nt->mw_count;  in ntb_transport_probe()
1142 	nt->qp_count = qp_count;  in ntb_transport_probe()
1143 	nt->qp_bitmap = qp_bitmap;  in ntb_transport_probe()
1144 	nt->qp_bitmap_free = qp_bitmap;  in ntb_transport_probe()
1146 	nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec),  in ntb_transport_probe()
1148 	if (!nt->qp_vec) {  in ntb_transport_probe()
1154 		nt->debugfs_node_dir =  in ntb_transport_probe()
1160 		rc = ntb_transport_init_queue(nt, i);  in ntb_transport_probe()
1165 	INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);  in ntb_transport_probe()
1166 	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);  in ntb_transport_probe()
1168 	rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops);  in ntb_transport_probe()
1172 	INIT_LIST_HEAD(&nt->client_devs);  in ntb_transport_probe()
1173 	rc = ntb_bus_init(nt);  in ntb_transport_probe()
1177 	nt->link_is_up = false;  in ntb_transport_probe()
1186 	kfree(nt->qp_vec);  in ntb_transport_probe()
1189 		mw = &nt->mw_vec[i];  in ntb_transport_probe()
1192 	kfree(nt->mw_vec);  in ntb_transport_probe()
1194 	kfree(nt);  in ntb_transport_probe()
1200 	struct ntb_transport_ctx *nt = ndev->ctx;  in ntb_transport_free()  local
1205 	ntb_transport_link_cleanup(nt);  in ntb_transport_free()
1206 	cancel_work_sync(&nt->link_cleanup);  in ntb_transport_free()
1207 	cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_free()
1209 	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;  in ntb_transport_free()
1212 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_free()
1213 		qp = &nt->qp_vec[i];  in ntb_transport_free()
1222 	ntb_bus_remove(nt);  in ntb_transport_free()
1224 	for (i = nt->mw_count; i--; ) {  in ntb_transport_free()
1225 		ntb_free_mw(nt, i);  in ntb_transport_free()
1226 		iounmap(nt->mw_vec[i].vbase);  in ntb_transport_free()
1229 	kfree(nt->qp_vec);  in ntb_transport_free()
1230 	kfree(nt->mw_vec);  in ntb_transport_free()
1231 	kfree(nt);  in ntb_transport_free()
1772 	struct ntb_transport_ctx *nt;  in ntb_transport_create_queue()  local
1783 	nt = ndev->ctx;  in ntb_transport_create_queue()
1787 	free_queue = ffs(nt->qp_bitmap_free);  in ntb_transport_create_queue()
1794 	qp = &nt->qp_vec[free_queue];  in ntb_transport_create_queue()
1797 	nt->qp_bitmap_free &= ~qp_bit;  in ntb_transport_create_queue()
1869 	nt->qp_bitmap_free |= qp_bit;  in ntb_transport_create_queue()
2197 	struct ntb_transport_ctx *nt = data;  in ntb_transport_doorbell_callback()  local
2202 	db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &  in ntb_transport_doorbell_callback()
2203 		   ntb_db_vector_mask(nt->ndev, vector));  in ntb_transport_doorbell_callback()
2207 		qp = &nt->qp_vec[qp_num];  in ntb_transport_doorbell_callback()