76 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)  in ndev_mw_to_bar()  argument
78 	if (idx < 0 || idx > ndev->mw_count)  in ndev_mw_to_bar()
81 	return ndev->dev_data->mw_idx << idx;  in ndev_mw_to_bar()
97 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_mw_get_align()  local
103 	bar = ndev_mw_to_bar(ndev, idx);  in amd_ntb_mw_get_align()
114 		*size_max = pci_resource_len(ndev->ntb.pdev, bar);  in amd_ntb_mw_get_align()
122 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_mw_set_trans()  local
132 	bar = ndev_mw_to_bar(ndev, idx);  in amd_ntb_mw_set_trans()
142 	mmio = ndev->self_mmio;  in amd_ntb_mw_set_trans()
143 	peer_mmio = ndev->peer_mmio;  in amd_ntb_mw_set_trans()
198 static int amd_link_is_up(struct amd_ntb_dev *ndev)  in amd_link_is_up()  argument
200 	if (!ndev->peer_sta)  in amd_link_is_up()
201 		return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);  in amd_link_is_up()
203 	if (ndev->peer_sta & AMD_LINK_UP_EVENT) {  in amd_link_is_up()
204 		ndev->peer_sta = 0;  in amd_link_is_up()
214 	if (ndev->peer_sta & AMD_PEER_RESET_EVENT)  in amd_link_is_up()
215 		ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;  in amd_link_is_up()
216 	else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))  in amd_link_is_up()
217 		ndev->peer_sta = 0;  in amd_link_is_up()
226 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_link_is_up()  local
229 	if (amd_link_is_up(ndev)) {  in amd_ntb_link_is_up()
231 			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);  in amd_ntb_link_is_up()
233 			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);  in amd_ntb_link_is_up()
254 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_link_enable()  local
255 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_link_enable()
259 	ndev->int_mask &= ~AMD_EVENT_INTMASK;  in amd_ntb_link_enable()
260 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);  in amd_ntb_link_enable()
262 	if (ndev->ntb.topo == NTB_TOPO_SEC)  in amd_ntb_link_enable()
275 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_link_disable()  local
276 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_link_disable()
280 	ndev->int_mask |= AMD_EVENT_INTMASK;  in amd_ntb_link_disable()
281 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);  in amd_ntb_link_disable()
283 	if (ndev->ntb.topo == NTB_TOPO_SEC)  in amd_ntb_link_disable()
303 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_peer_mw_get_addr()  local
306 	bar = ndev_mw_to_bar(ndev, idx);  in amd_ntb_peer_mw_get_addr()
311 		*base = pci_resource_start(ndev->ntb.pdev, bar);  in amd_ntb_peer_mw_get_addr()
314 		*size = pci_resource_len(ndev->ntb.pdev, bar);  in amd_ntb_peer_mw_get_addr()
331 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_db_vector_mask()  local
333 	if (db_vector < 0 || db_vector > ndev->db_count)  in amd_ntb_db_vector_mask()
341 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_db_read()  local
342 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_db_read()
349 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_db_clear()  local
350 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_db_clear()
359 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_db_set_mask()  local
360 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_db_set_mask()
363 	if (db_bits & ~ndev->db_valid_mask)  in amd_ntb_db_set_mask()
366 	spin_lock_irqsave(&ndev->db_mask_lock, flags);  in amd_ntb_db_set_mask()
367 	ndev->db_mask |= db_bits;  in amd_ntb_db_set_mask()
368 	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);  in amd_ntb_db_set_mask()
369 	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);  in amd_ntb_db_set_mask()
376 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_db_clear_mask()  local
377 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_db_clear_mask()
380 	if (db_bits & ~ndev->db_valid_mask)  in amd_ntb_db_clear_mask()
383 	spin_lock_irqsave(&ndev->db_mask_lock, flags);  in amd_ntb_db_clear_mask()
384 	ndev->db_mask &= ~db_bits;  in amd_ntb_db_clear_mask()
385 	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);  in amd_ntb_db_clear_mask()
386 	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);  in amd_ntb_db_clear_mask()
393 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_peer_db_set()  local
394 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_peer_db_set()
408 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_spad_read()  local
409 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_spad_read()
412 	if (idx < 0 || idx >= ndev->spad_count)  in amd_ntb_spad_read()
415 	offset = ndev->self_spad + (idx << 2);  in amd_ntb_spad_read()
422 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_spad_write()  local
423 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_spad_write()
426 	if (idx < 0 || idx >= ndev->spad_count)  in amd_ntb_spad_write()
429 	offset = ndev->self_spad + (idx << 2);  in amd_ntb_spad_write()
437 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_peer_spad_read()  local
438 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_peer_spad_read()
441 	if (sidx < 0 || sidx >= ndev->spad_count)  in amd_ntb_peer_spad_read()
444 	offset = ndev->peer_spad + (sidx << 2);  in amd_ntb_peer_spad_read()
451 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);  in amd_ntb_peer_spad_write()  local
452 	void __iomem *mmio = ndev->self_mmio;  in amd_ntb_peer_spad_write()
455 	if (sidx < 0 || sidx >= ndev->spad_count)  in amd_ntb_peer_spad_write()
458 	offset = ndev->peer_spad + (sidx << 2);  in amd_ntb_peer_spad_write()
488 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)  in amd_ack_smu()  argument
490 	void __iomem *mmio = ndev->self_mmio;  in amd_ack_smu()
497 	ndev->peer_sta |= bit;  in amd_ack_smu()
500 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)  in amd_handle_event()  argument
502 	void __iomem *mmio = ndev->self_mmio;  in amd_handle_event()
503 	struct device *dev = &ndev->ntb.pdev->dev;  in amd_handle_event()
518 		amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);  in amd_handle_event()
521 		ntb_link_event(&ndev->ntb);  in amd_handle_event()
523 		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);  in amd_handle_event()
530 		amd_ack_smu(ndev, status);  in amd_handle_event()
533 		ntb_link_event(&ndev->ntb);  in amd_handle_event()
537 		mmio = ndev->peer_mmio;  in amd_handle_event()
543 		amd_ack_smu(ndev, AMD_PEER_D0_EVENT);  in amd_handle_event()
546 		schedule_delayed_work(&ndev->hb_timer,  in amd_handle_event()
555 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)  in ndev_interrupt()  argument
557 	dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);  in ndev_interrupt()
559 	if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))  in ndev_interrupt()
560 		amd_handle_event(ndev, vec);  in ndev_interrupt()
563 		ntb_db_event(&ndev->ntb, vec);  in ndev_interrupt()
572 	return ndev_interrupt(nvec->ndev, nvec->num);  in ndev_vec_isr()
577 	struct amd_ntb_dev *ndev = dev;  in ndev_irq_isr()  local
579 	return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);  in ndev_irq_isr()
582 static int ndev_init_isr(struct amd_ntb_dev *ndev,  in ndev_init_isr()  argument
588 	pdev = ndev->ntb.pdev;  in ndev_init_isr()
592 	ndev->db_mask = ndev->db_valid_mask;  in ndev_init_isr()
595 	ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),  in ndev_init_isr()
597 	if (!ndev->vec)  in ndev_init_isr()
600 	ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),  in ndev_init_isr()
602 	if (!ndev->msix)  in ndev_init_isr()
606 		ndev->msix[i].entry = i;  in ndev_init_isr()
608 	msix_count = pci_enable_msix_range(pdev, ndev->msix,  in ndev_init_isr()
622 		ndev->vec[i].ndev = ndev;  in ndev_init_isr()
623 		ndev->vec[i].num = i;  in ndev_init_isr()
624 		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,  in ndev_init_isr()
625 				 "ndev_vec_isr", &ndev->vec[i]);  in ndev_init_isr()
631 	ndev->db_count = msix_min;  in ndev_init_isr()
632 	ndev->msix_vec_count = msix_max;  in ndev_init_isr()
637 		free_irq(ndev->msix[i].vector, &ndev->vec[i]);  in ndev_init_isr()
640 	kfree(ndev->msix);  in ndev_init_isr()
642 	kfree(ndev->vec);  in ndev_init_isr()
644 	ndev->msix = NULL;  in ndev_init_isr()
645 	ndev->vec = NULL;  in ndev_init_isr()
653 			 "ndev_irq_isr", ndev);  in ndev_init_isr()
658 	ndev->db_count = 1;  in ndev_init_isr()
659 	ndev->msix_vec_count = 1;  in ndev_init_isr()
670 			 "ndev_irq_isr", ndev);  in ndev_init_isr()
675 	ndev->db_count = 1;  in ndev_init_isr()
676 	ndev->msix_vec_count = 1;  in ndev_init_isr()
683 static void ndev_deinit_isr(struct amd_ntb_dev *ndev)  in ndev_deinit_isr()  argument
686 	void __iomem *mmio = ndev->self_mmio;  in ndev_deinit_isr()
689 	pdev = ndev->ntb.pdev;  in ndev_deinit_isr()
692 	ndev->db_mask = ndev->db_valid_mask;  in ndev_deinit_isr()
693 	writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);  in ndev_deinit_isr()
695 	if (ndev->msix) {  in ndev_deinit_isr()
696 		i = ndev->msix_vec_count;  in ndev_deinit_isr()
698 			free_irq(ndev->msix[i].vector, &ndev->vec[i]);  in ndev_deinit_isr()
700 		kfree(ndev->msix);  in ndev_deinit_isr()
701 		kfree(ndev->vec);  in ndev_deinit_isr()
703 		free_irq(pdev->irq, ndev);  in ndev_deinit_isr()
714 	struct amd_ntb_dev *ndev;  in ndev_debugfs_read()  local
721 	ndev = filp->private_data;  in ndev_debugfs_read()
722 	mmio = ndev->self_mmio;  in ndev_debugfs_read()
737 			 ntb_topo_string(ndev->ntb.topo));  in ndev_debugfs_read()
740 			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);  in ndev_debugfs_read()
742 	if (!amd_link_is_up(ndev)) {  in ndev_debugfs_read()
750 				 NTB_LNK_STA_SPEED(ndev->lnk_sta));  in ndev_debugfs_read()
753 				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));  in ndev_debugfs_read()
757 			 "Memory Window Count -\t%u\n", ndev->mw_count);  in ndev_debugfs_read()
759 			 "Scratchpad Count -\t%u\n", ndev->spad_count);  in ndev_debugfs_read()
761 			 "Doorbell Count -\t%u\n", ndev->db_count);  in ndev_debugfs_read()
763 			 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);  in ndev_debugfs_read()
766 			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);  in ndev_debugfs_read()
768 	u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);  in ndev_debugfs_read()
783 	u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);  in ndev_debugfs_read()
787 	u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);  in ndev_debugfs_read()
795 	u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);  in ndev_debugfs_read()
799 	u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);  in ndev_debugfs_read()
808 static void ndev_init_debugfs(struct amd_ntb_dev *ndev)  in ndev_init_debugfs()  argument
811 		ndev->debugfs_dir = NULL;  in ndev_init_debugfs()
812 		ndev->debugfs_info = NULL;  in ndev_init_debugfs()
814 		ndev->debugfs_dir =  in ndev_init_debugfs()
815 			debugfs_create_dir(pci_name(ndev->ntb.pdev),  in ndev_init_debugfs()
817 		if (!ndev->debugfs_dir)  in ndev_init_debugfs()
818 			ndev->debugfs_info = NULL;  in ndev_init_debugfs()
820 			ndev->debugfs_info =  in ndev_init_debugfs()
822 						    ndev->debugfs_dir, ndev,  in ndev_init_debugfs()
827 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)  in ndev_deinit_debugfs()  argument
829 	debugfs_remove_recursive(ndev->debugfs_dir);  in ndev_deinit_debugfs()
832 static inline void ndev_init_struct(struct amd_ntb_dev *ndev,  in ndev_init_struct()  argument
835 	ndev->ntb.pdev = pdev;  in ndev_init_struct()
836 	ndev->ntb.topo = NTB_TOPO_NONE;  in ndev_init_struct()
837 	ndev->ntb.ops = &amd_ntb_ops;  in ndev_init_struct()
838 	ndev->int_mask = AMD_EVENT_INTMASK;  in ndev_init_struct()
839 	spin_lock_init(&ndev->db_mask_lock);  in ndev_init_struct()
842 static int amd_poll_link(struct amd_ntb_dev *ndev)  in amd_poll_link()  argument
844 	void __iomem *mmio = ndev->peer_mmio;  in amd_poll_link()
851 	dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);  in amd_poll_link()
853 	if (reg == ndev->cntl_sta)  in amd_poll_link()
856 	ndev->cntl_sta = reg;  in amd_poll_link()
858 	rc = pci_read_config_dword(ndev->ntb.pdev,  in amd_poll_link()
862 	ndev->lnk_sta = stat;  in amd_poll_link()
869 	struct amd_ntb_dev *ndev = hb_ndev(work);  in amd_link_hb()  local
871 	if (amd_poll_link(ndev))  in amd_link_hb()
872 		ntb_link_event(&ndev->ntb);  in amd_link_hb()
874 	if (!amd_link_is_up(ndev))  in amd_link_hb()
875 		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);  in amd_link_hb()
878 static int amd_init_isr(struct amd_ntb_dev *ndev)  in amd_init_isr()  argument
880 	return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);  in amd_init_isr()
883 static void amd_init_side_info(struct amd_ntb_dev *ndev)  in amd_init_side_info()  argument
885 	void __iomem *mmio = ndev->self_mmio;  in amd_init_side_info()
895 static void amd_deinit_side_info(struct amd_ntb_dev *ndev)  in amd_deinit_side_info()  argument
897 	void __iomem *mmio = ndev->self_mmio;  in amd_deinit_side_info()
908 static int amd_init_ntb(struct amd_ntb_dev *ndev)  in amd_init_ntb()  argument
910 	void __iomem *mmio = ndev->self_mmio;  in amd_init_ntb()
912 	ndev->mw_count = ndev->dev_data->mw_count;  in amd_init_ntb()
913 	ndev->spad_count = AMD_SPADS_CNT;  in amd_init_ntb()
914 	ndev->db_count = AMD_DB_CNT;  in amd_init_ntb()
916 	switch (ndev->ntb.topo) {  in amd_init_ntb()
919 		ndev->spad_count >>= 1;  in amd_init_ntb()
920 		if (ndev->ntb.topo == NTB_TOPO_PRI) {  in amd_init_ntb()
921 			ndev->self_spad = 0;  in amd_init_ntb()
922 			ndev->peer_spad = 0x20;  in amd_init_ntb()
924 			ndev->self_spad = 0x20;  in amd_init_ntb()
925 			ndev->peer_spad = 0;  in amd_init_ntb()
928 		INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);  in amd_init_ntb()
929 		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);  in amd_init_ntb()
933 		dev_err(&ndev->ntb.pdev->dev,  in amd_init_ntb()
938 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;  in amd_init_ntb()
941 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);  in amd_init_ntb()
946 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)  in amd_get_topo()  argument
948 	void __iomem *mmio = ndev->self_mmio;  in amd_get_topo()
958 static int amd_init_dev(struct amd_ntb_dev *ndev)  in amd_init_dev()  argument
963 	pdev = ndev->ntb.pdev;  in amd_init_dev()
965 	ndev->ntb.topo = amd_get_topo(ndev);  in amd_init_dev()
967 		ntb_topo_string(ndev->ntb.topo));  in amd_init_dev()
969 	rc = amd_init_ntb(ndev);  in amd_init_dev()
973 	rc = amd_init_isr(ndev);  in amd_init_dev()
979 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;  in amd_init_dev()
984 static void amd_deinit_dev(struct amd_ntb_dev *ndev)  in amd_deinit_dev()  argument
986 	cancel_delayed_work_sync(&ndev->hb_timer);  in amd_deinit_dev()
988 	ndev_deinit_isr(ndev);  in amd_deinit_dev()
991 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,  in amd_ntb_init_pci()  argument
996 	pci_set_drvdata(pdev, ndev);  in amd_ntb_init_pci()
1023 	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,  in amd_ntb_init_pci()
1028 	ndev->self_mmio = pci_iomap(pdev, 0, 0);  in amd_ntb_init_pci()
1029 	if (!ndev->self_mmio) {  in amd_ntb_init_pci()
1033 	ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;  in amd_ntb_init_pci()
1046 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)  in amd_ntb_deinit_pci()  argument
1048 	struct pci_dev *pdev = ndev->ntb.pdev;  in amd_ntb_deinit_pci()
1050 	pci_iounmap(pdev, ndev->self_mmio);  in amd_ntb_deinit_pci()
1061 	struct amd_ntb_dev *ndev;  in amd_ntb_pci_probe()  local
1066 	ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);  in amd_ntb_pci_probe()
1067 	if (!ndev) {  in amd_ntb_pci_probe()
1072 	ndev->dev_data = (struct ntb_dev_data *)id->driver_data;  in amd_ntb_pci_probe()
1074 	ndev_init_struct(ndev, pdev);  in amd_ntb_pci_probe()
1076 	rc = amd_ntb_init_pci(ndev, pdev);  in amd_ntb_pci_probe()
1080 	rc = amd_init_dev(ndev);  in amd_ntb_pci_probe()
1085 	amd_init_side_info(ndev);  in amd_ntb_pci_probe()
1087 	amd_poll_link(ndev);  in amd_ntb_pci_probe()
1089 	ndev_init_debugfs(ndev);  in amd_ntb_pci_probe()
1091 	rc = ntb_register_device(&ndev->ntb);  in amd_ntb_pci_probe()
1100 	ndev_deinit_debugfs(ndev);  in amd_ntb_pci_probe()
1101 	amd_deinit_dev(ndev);  in amd_ntb_pci_probe()
1103 	amd_ntb_deinit_pci(ndev);  in amd_ntb_pci_probe()
1105 	kfree(ndev);  in amd_ntb_pci_probe()
1112 	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);  in amd_ntb_pci_remove()  local
1114 	ntb_unregister_device(&ndev->ntb);  in amd_ntb_pci_remove()
1115 	ndev_deinit_debugfs(ndev);  in amd_ntb_pci_remove()
1116 	amd_deinit_side_info(ndev);  in amd_ntb_pci_remove()
1117 	amd_deinit_dev(ndev);  in amd_ntb_pci_remove()
1118 	amd_ntb_deinit_pci(ndev);  in amd_ntb_pci_remove()
1119 	kfree(ndev);  in amd_ntb_pci_remove()