119 	struct ntb_dev ntb;  member
149 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
166 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)  in epf_ntb_link_up()  argument
169 		ntb->reg->link_status |= LINK_STATUS_UP;  in epf_ntb_link_up()
171 		ntb->reg->link_status &= ~LINK_STATUS_UP;  in epf_ntb_link_up()
173 	ntb_link_event(&ntb->ntb);  in epf_ntb_link_up()
199 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)  in epf_ntb_configure_mw()  argument
206 	phys_addr = ntb->vpci_mw_phy[mw];  in epf_ntb_configure_mw()
207 	addr = ntb->reg->addr;  in epf_ntb_configure_mw()
208 	size = ntb->reg->size;  in epf_ntb_configure_mw()
210 	func_no = ntb->epf->func_no;  in epf_ntb_configure_mw()
211 	vfunc_no = ntb->epf->vfunc_no;  in epf_ntb_configure_mw()
213 	ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);  in epf_ntb_configure_mw()
215 		dev_err(&ntb->epf->epc->dev,  in epf_ntb_configure_mw()
228 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)  in epf_ntb_teardown_mw()  argument
230 	pci_epc_unmap_addr(ntb->epf->epc,  in epf_ntb_teardown_mw()
231 			   ntb->epf->func_no,  in epf_ntb_teardown_mw()
232 			   ntb->epf->vfunc_no,  in epf_ntb_teardown_mw()
233 			   ntb->vpci_mw_phy[mw]);  in epf_ntb_teardown_mw()
249 	struct epf_ntb *ntb;  in epf_ntb_cmd_handler()  local
254 	ntb = container_of(work, struct epf_ntb, cmd_handler.work);  in epf_ntb_cmd_handler()
256 	for (i = 1; i < ntb->db_count; i++) {  in epf_ntb_cmd_handler()
257 		if (readl(ntb->epf_db + i * 4)) {  in epf_ntb_cmd_handler()
258 			if (readl(ntb->epf_db + i * 4))  in epf_ntb_cmd_handler()
259 				ntb->db |= 1 << (i - 1);  in epf_ntb_cmd_handler()
261 			ntb_db_event(&ntb->ntb, i);  in epf_ntb_cmd_handler()
262 			writel(0, ntb->epf_db + i * 4);  in epf_ntb_cmd_handler()
266 	ctrl = ntb->reg;  in epf_ntb_cmd_handler()
275 	ctrl = ntb->reg;  in epf_ntb_cmd_handler()
276 	dev = &ntb->epf->dev;  in epf_ntb_cmd_handler()
286 		ret = epf_ntb_configure_mw(ntb, argument);  in epf_ntb_cmd_handler()
293 		epf_ntb_teardown_mw(ntb, argument);  in epf_ntb_cmd_handler()
297 		ntb->linkup = true;  in epf_ntb_cmd_handler()
298 		ret = epf_ntb_link_up(ntb, true);  in epf_ntb_cmd_handler()
305 		ntb->linkup = false;  in epf_ntb_cmd_handler()
306 		ret = epf_ntb_link_up(ntb, false);  in epf_ntb_cmd_handler()
318 	queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,  in epf_ntb_cmd_handler()
337 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)  in epf_ntb_config_sspad_bar_clear()  argument
342 	barno = ntb->epf_ntb_bar[BAR_CONFIG];  in epf_ntb_config_sspad_bar_clear()
343 	epf_bar = &ntb->epf->bar[barno];  in epf_ntb_config_sspad_bar_clear()
345 	pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);  in epf_ntb_config_sspad_bar_clear()
358 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)  in epf_ntb_config_sspad_bar_set()  argument
366 	dev = &ntb->epf->dev;  in epf_ntb_config_sspad_bar_set()
367 	func_no = ntb->epf->func_no;  in epf_ntb_config_sspad_bar_set()
368 	vfunc_no = ntb->epf->vfunc_no;  in epf_ntb_config_sspad_bar_set()
369 	barno = ntb->epf_ntb_bar[BAR_CONFIG];  in epf_ntb_config_sspad_bar_set()
370 	epf_bar = &ntb->epf->bar[barno];  in epf_ntb_config_sspad_bar_set()
372 	ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);  in epf_ntb_config_sspad_bar_set()
385 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)  in epf_ntb_config_spad_bar_free()  argument
389 	barno = ntb->epf_ntb_bar[BAR_CONFIG];  in epf_ntb_config_spad_bar_free()
390 	pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);  in epf_ntb_config_spad_bar_free()
402 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)  in epf_ntb_config_spad_bar_alloc()  argument
409 	struct pci_epf *epf = ntb->epf;  in epf_ntb_config_spad_bar_alloc()
417 	barno = ntb->epf_ntb_bar[BAR_CONFIG];  in epf_ntb_config_spad_bar_alloc()
424 	spad_count = ntb->spad_count;  in epf_ntb_config_spad_bar_alloc()
448 	ntb->reg = base;  in epf_ntb_config_spad_bar_alloc()
450 	ctrl = ntb->reg;  in epf_ntb_config_spad_bar_alloc()
454 	ctrl->num_mws = ntb->num_mws;  in epf_ntb_config_spad_bar_alloc()
455 	ntb->spad_size = spad_size;  in epf_ntb_config_spad_bar_alloc()
459 	for (i = 0; i < ntb->db_count; i++) {  in epf_ntb_config_spad_bar_alloc()
460 		ntb->reg->db_data[i] = 1 + i;  in epf_ntb_config_spad_bar_alloc()
461 		ntb->reg->db_offset[i] = 0;  in epf_ntb_config_spad_bar_alloc()
474 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)  in epf_ntb_configure_interrupt()  argument
481 	dev = &ntb->epf->dev;  in epf_ntb_configure_interrupt()
483 	epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);  in epf_ntb_configure_interrupt()
490 	db_count = ntb->db_count;  in epf_ntb_configure_interrupt()
496 	ntb->db_count = db_count;  in epf_ntb_configure_interrupt()
499 		ret = pci_epc_set_msi(ntb->epf->epc,  in epf_ntb_configure_interrupt()
500 				      ntb->epf->func_no,  in epf_ntb_configure_interrupt()
501 				      ntb->epf->vfunc_no,  in epf_ntb_configure_interrupt()
516 static int epf_ntb_db_bar_init(struct epf_ntb *ntb)  in epf_ntb_db_bar_init()  argument
520 	struct device *dev = &ntb->epf->dev;  in epf_ntb_db_bar_init()
525 	size_t size = 4 * ntb->db_count;  in epf_ntb_db_bar_init()
527 	epc_features = pci_epc_get_features(ntb->epf->epc,  in epf_ntb_db_bar_init()
528 					    ntb->epf->func_no,  in epf_ntb_db_bar_init()
529 					    ntb->epf->vfunc_no);  in epf_ntb_db_bar_init()
540 	barno = ntb->epf_ntb_bar[BAR_DB];  in epf_ntb_db_bar_init()
542 	mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0);  in epf_ntb_db_bar_init()
548 	ntb->epf_db = mw_addr;  in epf_ntb_db_bar_init()
550 	epf_bar = &ntb->epf->bar[barno];  in epf_ntb_db_bar_init()
552 	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);  in epf_ntb_db_bar_init()
560 	pci_epc_mem_free_addr(ntb->epf->epc, epf_bar->phys_addr, mw_addr, epf_bar->size);  in epf_ntb_db_bar_init()
564 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
571 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)  in epf_ntb_db_bar_clear()  argument
575 	barno = ntb->epf_ntb_bar[BAR_DB];  in epf_ntb_db_bar_clear()
576 	pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);  in epf_ntb_db_bar_clear()
577 	pci_epc_clear_bar(ntb->epf->epc,  in epf_ntb_db_bar_clear()
578 			  ntb->epf->func_no,  in epf_ntb_db_bar_clear()
579 			  ntb->epf->vfunc_no,  in epf_ntb_db_bar_clear()
580 			  &ntb->epf->bar[barno]);  in epf_ntb_db_bar_clear()
588 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)  in epf_ntb_mw_bar_init()  argument
594 	struct device *dev = &ntb->epf->dev;  in epf_ntb_mw_bar_init()
596 	for (i = 0; i < ntb->num_mws; i++) {  in epf_ntb_mw_bar_init()
597 		size = ntb->mws_size[i];  in epf_ntb_mw_bar_init()
598 		barno = ntb->epf_ntb_bar[BAR_MW0 + i];  in epf_ntb_mw_bar_init()
600 		ntb->epf->bar[barno].barno = barno;  in epf_ntb_mw_bar_init()
601 		ntb->epf->bar[barno].size = size;  in epf_ntb_mw_bar_init()
602 		ntb->epf->bar[barno].addr = NULL;  in epf_ntb_mw_bar_init()
603 		ntb->epf->bar[barno].phys_addr = 0;  in epf_ntb_mw_bar_init()
604 		ntb->epf->bar[barno].flags |= upper_32_bits(size) ?  in epf_ntb_mw_bar_init()
608 		ret = pci_epc_set_bar(ntb->epf->epc,  in epf_ntb_mw_bar_init()
609 				      ntb->epf->func_no,  in epf_ntb_mw_bar_init()
610 				      ntb->epf->vfunc_no,  in epf_ntb_mw_bar_init()
611 				      &ntb->epf->bar[barno]);  in epf_ntb_mw_bar_init()
618 		ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,  in epf_ntb_mw_bar_init()
619 							      &ntb->vpci_mw_phy[i],  in epf_ntb_mw_bar_init()
621 		if (!ntb->vpci_mw_addr[i]) {  in epf_ntb_mw_bar_init()
631 	pci_epc_clear_bar(ntb->epf->epc,  in epf_ntb_mw_bar_init()
632 			  ntb->epf->func_no,  in epf_ntb_mw_bar_init()
633 			  ntb->epf->vfunc_no,  in epf_ntb_mw_bar_init()
634 			  &ntb->epf->bar[barno]);  in epf_ntb_mw_bar_init()
636 	epf_ntb_mw_bar_clear(ntb, i);  in epf_ntb_mw_bar_init()
644 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)  in epf_ntb_mw_bar_clear()  argument
650 		barno = ntb->epf_ntb_bar[BAR_MW0 + i];  in epf_ntb_mw_bar_clear()
651 		pci_epc_clear_bar(ntb->epf->epc,  in epf_ntb_mw_bar_clear()
652 				  ntb->epf->func_no,  in epf_ntb_mw_bar_clear()
653 				  ntb->epf->vfunc_no,  in epf_ntb_mw_bar_clear()
654 				  &ntb->epf->bar[barno]);  in epf_ntb_mw_bar_clear()
656 		pci_epc_mem_free_addr(ntb->epf->epc,  in epf_ntb_mw_bar_clear()
657 				      ntb->vpci_mw_phy[i],  in epf_ntb_mw_bar_clear()
658 				      ntb->vpci_mw_addr[i],  in epf_ntb_mw_bar_clear()
659 				      ntb->mws_size[i]);  in epf_ntb_mw_bar_clear()
669 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)  in epf_ntb_epc_destroy()  argument
671 	pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);  in epf_ntb_epc_destroy()
672 	pci_epc_put(ntb->epf->epc);  in epf_ntb_epc_destroy()
680 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)  in epf_ntb_init_epc_bar()  argument
690 	num_mws = ntb->num_mws;  in epf_ntb_init_epc_bar()
691 	dev = &ntb->epf->dev;  in epf_ntb_init_epc_bar()
692 	epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);  in epf_ntb_init_epc_bar()
701 		ntb->epf_ntb_bar[bar] = barno;  in epf_ntb_init_epc_bar()
708 			ntb->num_mws = i;  in epf_ntb_init_epc_bar()
711 		ntb->epf_ntb_bar[bar] = barno;  in epf_ntb_init_epc_bar()
725 static int epf_ntb_epc_init(struct epf_ntb *ntb)  in epf_ntb_epc_init()  argument
733 	epf = ntb->epf;  in epf_ntb_epc_init()
736 	func_no = ntb->epf->func_no;  in epf_ntb_epc_init()
737 	vfunc_no = ntb->epf->vfunc_no;  in epf_ntb_epc_init()
739 	ret = epf_ntb_config_sspad_bar_set(ntb);  in epf_ntb_epc_init()
745 	ret = epf_ntb_configure_interrupt(ntb);  in epf_ntb_epc_init()
751 	ret = epf_ntb_db_bar_init(ntb);  in epf_ntb_epc_init()
757 	ret = epf_ntb_mw_bar_init(ntb);  in epf_ntb_epc_init()
771 	INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);  in epf_ntb_epc_init()
772 	queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);  in epf_ntb_epc_init()
777 	epf_ntb_mw_bar_clear(ntb, ntb->num_mws);  in epf_ntb_epc_init()
779 	epf_ntb_db_bar_clear(ntb);  in epf_ntb_epc_init()
782 	epf_ntb_config_sspad_bar_clear(ntb);  in epf_ntb_epc_init()
794 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)  in epf_ntb_epc_cleanup()  argument
796 	epf_ntb_db_bar_clear(ntb);  in epf_ntb_epc_cleanup()
797 	epf_ntb_mw_bar_clear(ntb, ntb->num_mws);  in epf_ntb_epc_cleanup()
805 	struct epf_ntb *ntb = to_epf_ntb(group);			\
807 	return sprintf(page, "%d\n", ntb->_name);			\
815 	struct epf_ntb *ntb = to_epf_ntb(group);			\
823 	ntb->_name = val;						\
833 	struct epf_ntb *ntb = to_epf_ntb(group);			\
834 	struct device *dev = &ntb->epf->dev;				\
840 	if (win_no <= 0 || win_no > ntb->num_mws) {			\
841 		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
845 	return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]);	\
853 	struct epf_ntb *ntb = to_epf_ntb(group);			\
854 	struct device *dev = &ntb->epf->dev;				\
866 	if (win_no <= 0 || win_no > ntb->num_mws) {			\
867 		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
871 	ntb->mws_size[win_no - 1] = val;				\
880 	struct epf_ntb *ntb = to_epf_ntb(group);  in epf_ntb_num_mws_store()  local
891 	ntb->num_mws = val;  in epf_ntb_num_mws_store()
958 	struct epf_ntb *ntb = epf_get_drvdata(epf);  in epf_ntb_add_cfs()  local
959 	struct config_group *ntb_group = &ntb->group;  in epf_ntb_add_cfs()
1023 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)  in vntb_epf_mw_count()  argument
1025 	struct epf_ntb *ndev = ntb_ndev(ntb);  in vntb_epf_mw_count()
1030 static int vntb_epf_spad_count(struct ntb_dev *ntb)  in vntb_epf_spad_count()  argument
1032 	return ntb_ndev(ntb)->spad_count;  in vntb_epf_spad_count()
1035 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)  in vntb_epf_peer_mw_count()  argument
1037 	return ntb_ndev(ntb)->num_mws;  in vntb_epf_peer_mw_count()
1040 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)  in vntb_epf_db_valid_mask()  argument
1042 	return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;  in vntb_epf_db_valid_mask()
1045 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)  in vntb_epf_db_set_mask()  argument
1053 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_mw_set_trans()  local
1059 	dev = &ntb->ntb.dev;  in vntb_epf_mw_set_trans()
1060 	barno = ntb->epf_ntb_bar[BAR_MW0 + idx];  in vntb_epf_mw_set_trans()
1061 	epf_bar = &ntb->epf->bar[barno];  in vntb_epf_mw_set_trans()
1066 	ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);  in vntb_epf_mw_set_trans()
1074 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)  in vntb_epf_mw_clear_trans()  argument
1083 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_peer_mw_get_addr()  local
1086 		*base = ntb->vpci_mw_phy[idx];  in vntb_epf_peer_mw_get_addr()
1089 		*size = ntb->mws_size[idx];  in vntb_epf_peer_mw_get_addr()
1094 static int vntb_epf_link_enable(struct ntb_dev *ntb,  in vntb_epf_link_enable()  argument
1103 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_spad_read()  local
1104 	int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * 4;  in vntb_epf_spad_read()
1106 	void __iomem *base = ntb->reg;  in vntb_epf_spad_read()
1114 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_spad_write()  local
1115 	struct epf_ntb_ctrl *ctrl = ntb->reg;  in vntb_epf_spad_write()
1117 	void __iomem *base = ntb->reg;  in vntb_epf_spad_write()
1125 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_peer_spad_read()  local
1126 	struct epf_ntb_ctrl *ctrl = ntb->reg;  in vntb_epf_peer_spad_read()
1128 	void __iomem *base = ntb->reg;  in vntb_epf_peer_spad_read()
1137 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_peer_spad_write()  local
1138 	struct epf_ntb_ctrl *ctrl = ntb->reg;  in vntb_epf_peer_spad_write()
1140 	void __iomem *base = ntb->reg;  in vntb_epf_peer_spad_write()
1149 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_peer_db_set()  local
1153 	func_no = ntb->epf->func_no;  in vntb_epf_peer_db_set()
1154 	vfunc_no = ntb->epf->vfunc_no;  in vntb_epf_peer_db_set()
1156 	ret = pci_epc_raise_irq(ntb->epf->epc,  in vntb_epf_peer_db_set()
1162 		dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");  in vntb_epf_peer_db_set()
1169 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_db_read()  local
1171 	return ntb->db;  in vntb_epf_db_read()
1179 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_mw_get_align()  local
1188 		*size_max = ntb->mws_size[idx];  in vntb_epf_mw_get_align()
1197 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_link_is_up()  local
1199 	return ntb->reg->link_status;  in vntb_epf_link_is_up()
1209 	struct epf_ntb *ntb = ntb_ndev(ndev);  in vntb_epf_db_clear()  local
1211 	ntb->db &= ~db_bits;  in vntb_epf_db_clear()
1215 static int vntb_epf_link_disable(struct ntb_dev *ntb)  in vntb_epf_link_disable()  argument
1249 	ndev->ntb.pdev = pdev;  in pci_vntb_probe()
1250 	ndev->ntb.topo = NTB_TOPO_NONE;  in pci_vntb_probe()
1251 	ndev->ntb.ops =  &vntb_epf_ops;  in pci_vntb_probe()
1259 	ret = ntb_register_device(&ndev->ntb);  in pci_vntb_probe()
1298 	struct epf_ntb *ntb = epf_get_drvdata(epf);  in epf_ntb_bind()  local
1307 	ret = epf_ntb_init_epc_bar(ntb);  in epf_ntb_bind()
1313 	ret = epf_ntb_config_spad_bar_alloc(ntb);  in epf_ntb_bind()
1319 	ret = epf_ntb_epc_init(ntb);  in epf_ntb_bind()
1325 	epf_set_drvdata(epf, ntb);  in epf_ntb_bind()
1327 	pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;  in epf_ntb_bind()
1328 	pci_vntb_table[0].vendor = ntb->vntb_vid;  in epf_ntb_bind()
1329 	pci_vntb_table[0].device = ntb->vntb_pid;  in epf_ntb_bind()
1337 	vpci_scan_bus(ntb);  in epf_ntb_bind()
1342 	epf_ntb_config_spad_bar_free(ntb);  in epf_ntb_bind()
1345 	epf_ntb_epc_destroy(ntb);  in epf_ntb_bind()
1358 	struct epf_ntb *ntb = epf_get_drvdata(epf);  in epf_ntb_unbind()  local
1360 	epf_ntb_epc_cleanup(ntb);  in epf_ntb_unbind()
1361 	epf_ntb_config_spad_bar_free(ntb);  in epf_ntb_unbind()
1362 	epf_ntb_epc_destroy(ntb);  in epf_ntb_unbind()
1383 	struct epf_ntb *ntb;  in epf_ntb_probe()  local
1388 	ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);  in epf_ntb_probe()
1389 	if (!ntb)  in epf_ntb_probe()
1393 	ntb->epf = epf;  in epf_ntb_probe()
1394 	ntb->vbus_number = 0xff;  in epf_ntb_probe()
1395 	epf_set_drvdata(epf, ntb);  in epf_ntb_probe()