97 static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command,  in ntb_epf_send_command()  argument
105 	mutex_lock(&ndev->cmd_lock);  in ntb_epf_send_command()
106 	writel(argument, ndev->ctrl_reg + NTB_EPF_ARGUMENT);  in ntb_epf_send_command()
107 	writel(command, ndev->ctrl_reg + NTB_EPF_COMMAND);  in ntb_epf_send_command()
112 		status = readw(ndev->ctrl_reg + NTB_EPF_CMD_STATUS);  in ntb_epf_send_command()
130 	writew(0, ndev->ctrl_reg + NTB_EPF_CMD_STATUS);  in ntb_epf_send_command()
131 	mutex_unlock(&ndev->cmd_lock);  in ntb_epf_send_command()
136 static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx)  in ntb_epf_mw_to_bar()  argument
138 	struct device *dev = ndev->dev;  in ntb_epf_mw_to_bar()
140 	if (idx < 0 || idx > ndev->mw_count) {  in ntb_epf_mw_to_bar()
150 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_mw_count()  local
151 	struct device *dev = ndev->dev;  in ntb_epf_mw_count()
158 	return ndev->mw_count;  in ntb_epf_mw_count()
166 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_mw_get_align()  local
167 	struct device *dev = ndev->dev;  in ntb_epf_mw_get_align()
175 	bar = ntb_epf_mw_to_bar(ndev, idx);  in ntb_epf_mw_get_align()
186 		*size_max = pci_resource_len(ndev->ntb.pdev, bar);  in ntb_epf_mw_get_align()
195 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_link_is_up()  local
198 	status = readw(ndev->ctrl_reg + NTB_EPF_LINK_STATUS);  in ntb_epf_link_is_up()
205 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_spad_read()  local
206 	struct device *dev = ndev->dev;  in ntb_epf_spad_read()
209 	if (idx < 0 || idx >= ndev->spad_count) {  in ntb_epf_spad_read()
214 	offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET);  in ntb_epf_spad_read()
217 	return readl(ndev->ctrl_reg + offset);  in ntb_epf_spad_read()
223 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_spad_write()  local
224 	struct device *dev = ndev->dev;  in ntb_epf_spad_write()
227 	if (idx < 0 || idx >= ndev->spad_count) {  in ntb_epf_spad_write()
232 	offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET);  in ntb_epf_spad_write()
234 	writel(val, ndev->ctrl_reg + offset);  in ntb_epf_spad_write()
241 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_peer_spad_read()  local
242 	struct device *dev = ndev->dev;  in ntb_epf_peer_spad_read()
250 	if (idx < 0 || idx >= ndev->spad_count) {  in ntb_epf_peer_spad_read()
256 	return readl(ndev->peer_spad_reg + offset);  in ntb_epf_peer_spad_read()
262 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_peer_spad_write()  local
263 	struct device *dev = ndev->dev;  in ntb_epf_peer_spad_write()
271 	if (idx < 0 || idx >= ndev->spad_count) {  in ntb_epf_peer_spad_write()
277 	writel(val, ndev->peer_spad_reg + offset);  in ntb_epf_peer_spad_write()
286 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_link_enable()  local
287 	struct device *dev = ndev->dev;  in ntb_epf_link_enable()
290 	ret = ntb_epf_send_command(ndev, CMD_LINK_UP, 0);  in ntb_epf_link_enable()
301 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_link_disable()  local
302 	struct device *dev = ndev->dev;  in ntb_epf_link_disable()
305 	ret = ntb_epf_send_command(ndev, CMD_LINK_DOWN, 0);  in ntb_epf_link_disable()
316 	struct ntb_epf_dev *ndev = dev;  in ntb_epf_vec_isr()  local
319 	irq_no = irq - pci_irq_vector(ndev->ntb.pdev, 0);  in ntb_epf_vec_isr()
320 	ndev->db_val = irq_no + 1;  in ntb_epf_vec_isr()
323 		ntb_link_event(&ndev->ntb);  in ntb_epf_vec_isr()
325 		ntb_db_event(&ndev->ntb, irq_no);  in ntb_epf_vec_isr()
330 static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max)  in ntb_epf_init_isr()  argument
332 	struct pci_dev *pdev = ndev->ntb.pdev;  in ntb_epf_init_isr()
333 	struct device *dev = ndev->dev;  in ntb_epf_init_isr()
353 				  0, "ntb_epf", ndev);  in ntb_epf_init_isr()
360 	ndev->db_count = irq - 1;  in ntb_epf_init_isr()
362 	ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL,  in ntb_epf_init_isr()
372 	for (i = 0; i < ndev->db_count + 1; i++)  in ntb_epf_init_isr()
373 		free_irq(pci_irq_vector(pdev, i), ndev);  in ntb_epf_init_isr()
404 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_mw_set_trans()  local
405 	struct device *dev = ndev->dev;  in ntb_epf_mw_set_trans()
424 	writel(lower_32_bits(addr), ndev->ctrl_reg + NTB_EPF_LOWER_ADDR);  in ntb_epf_mw_set_trans()
425 	writel(upper_32_bits(addr), ndev->ctrl_reg + NTB_EPF_UPPER_ADDR);  in ntb_epf_mw_set_trans()
426 	writel(lower_32_bits(size), ndev->ctrl_reg + NTB_EPF_LOWER_SIZE);  in ntb_epf_mw_set_trans()
427 	writel(upper_32_bits(size), ndev->ctrl_reg + NTB_EPF_UPPER_SIZE);  in ntb_epf_mw_set_trans()
428 	ntb_epf_send_command(ndev, CMD_CONFIGURE_MW, idx);  in ntb_epf_mw_set_trans()
435 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_mw_clear_trans()  local
436 	struct device *dev = ndev->dev;  in ntb_epf_mw_clear_trans()
439 	ntb_epf_send_command(ndev, CMD_TEARDOWN_MW, idx);  in ntb_epf_mw_clear_trans()
449 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_peer_mw_get_addr()  local
454 		offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET);  in ntb_epf_peer_mw_get_addr()
459 		*base = pci_resource_start(ndev->ntb.pdev, bar) + offset;  in ntb_epf_peer_mw_get_addr()
462 		*size = pci_resource_len(ndev->ntb.pdev, bar) - offset;  in ntb_epf_peer_mw_get_addr()
469 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_peer_db_set()  local
471 	struct device *dev = ndev->dev;  in ntb_epf_peer_db_set()
476 	if (interrupt_num > ndev->db_count) {  in ntb_epf_peer_db_set()
478 			interrupt_num, ndev->db_count);  in ntb_epf_peer_db_set()
482 	db_entry_size = readl(ndev->ctrl_reg + NTB_EPF_DB_ENTRY_SIZE);  in ntb_epf_peer_db_set()
484 	db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num));  in ntb_epf_peer_db_set()
485 	db_offset = readl(ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num));  in ntb_epf_peer_db_set()
486 	writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num) +  in ntb_epf_peer_db_set()
494 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_db_read()  local
496 	return ndev->db_val;  in ntb_epf_db_read()
506 	struct ntb_epf_dev *ndev = ntb_ndev(ntb);  in ntb_epf_db_clear()  local
508 	ndev->db_val = 0;  in ntb_epf_db_clear()
536 static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev,  in ntb_epf_init_struct()  argument
539 	ndev->ntb.pdev = pdev;  in ntb_epf_init_struct()
540 	ndev->ntb.topo = NTB_TOPO_NONE;  in ntb_epf_init_struct()
541 	ndev->ntb.ops = &ntb_epf_ops;  in ntb_epf_init_struct()
544 static int ntb_epf_init_dev(struct ntb_epf_dev *ndev)  in ntb_epf_init_dev()  argument
546 	struct device *dev = ndev->dev;  in ntb_epf_init_dev()
550 	ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1,  in ntb_epf_init_dev()
557 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;  in ntb_epf_init_dev()
558 	ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT);  in ntb_epf_init_dev()
559 	ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT);  in ntb_epf_init_dev()
564 static int ntb_epf_init_pci(struct ntb_epf_dev *ndev,  in ntb_epf_init_pci()  argument
567 	struct device *dev = ndev->dev;  in ntb_epf_init_pci()
570 	pci_set_drvdata(pdev, ndev);  in ntb_epf_init_pci()
596 	ndev->ctrl_reg = pci_iomap(pdev, ndev->ctrl_reg_bar, 0);  in ntb_epf_init_pci()
597 	if (!ndev->ctrl_reg) {  in ntb_epf_init_pci()
602 	ndev->peer_spad_reg = pci_iomap(pdev, ndev->peer_spad_reg_bar, 0);  in ntb_epf_init_pci()
603 	if (!ndev->peer_spad_reg) {  in ntb_epf_init_pci()
608 	ndev->db_reg = pci_iomap(pdev, ndev->db_reg_bar, 0);  in ntb_epf_init_pci()
609 	if (!ndev->db_reg) {  in ntb_epf_init_pci()
628 static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev)  in ntb_epf_deinit_pci()  argument
630 	struct pci_dev *pdev = ndev->ntb.pdev;  in ntb_epf_deinit_pci()
632 	pci_iounmap(pdev, ndev->ctrl_reg);  in ntb_epf_deinit_pci()
633 	pci_iounmap(pdev, ndev->peer_spad_reg);  in ntb_epf_deinit_pci()
634 	pci_iounmap(pdev, ndev->db_reg);  in ntb_epf_deinit_pci()
642 static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev)  in ntb_epf_cleanup_isr()  argument
644 	struct pci_dev *pdev = ndev->ntb.pdev;  in ntb_epf_cleanup_isr()
647 	ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, ndev->db_count + 1);  in ntb_epf_cleanup_isr()
649 	for (i = 0; i < ndev->db_count + 1; i++)  in ntb_epf_cleanup_isr()
650 		free_irq(pci_irq_vector(pdev, i), ndev);  in ntb_epf_cleanup_isr()
662 	struct ntb_epf_dev *ndev;  in ntb_epf_pci_probe()  local
668 	ndev = devm_kzalloc(dev, sizeof(*ndev), GFP_KERNEL);  in ntb_epf_pci_probe()
669 	if (!ndev)  in ntb_epf_pci_probe()
682 	ndev->peer_spad_reg_bar = peer_spad_reg_bar;  in ntb_epf_pci_probe()
683 	ndev->ctrl_reg_bar = ctrl_reg_bar;  in ntb_epf_pci_probe()
684 	ndev->db_reg_bar = db_reg_bar;  in ntb_epf_pci_probe()
685 	ndev->dev = dev;  in ntb_epf_pci_probe()
687 	ntb_epf_init_struct(ndev, pdev);  in ntb_epf_pci_probe()
688 	mutex_init(&ndev->cmd_lock);  in ntb_epf_pci_probe()
690 	ret = ntb_epf_init_pci(ndev, pdev);  in ntb_epf_pci_probe()
696 	ret = ntb_epf_init_dev(ndev);  in ntb_epf_pci_probe()
702 	ret = ntb_register_device(&ndev->ntb);  in ntb_epf_pci_probe()
711 	ntb_epf_cleanup_isr(ndev);  in ntb_epf_pci_probe()
714 	ntb_epf_deinit_pci(ndev);  in ntb_epf_pci_probe()
721 	struct ntb_epf_dev *ndev = pci_get_drvdata(pdev);  in ntb_epf_pci_remove()  local
723 	ntb_unregister_device(&ndev->ntb);  in ntb_epf_pci_remove()
724 	ntb_epf_cleanup_isr(ndev);  in ntb_epf_pci_remove()
725 	ntb_epf_deinit_pci(ndev);  in ntb_epf_pci_remove()