102 static void qtnf_deassert_intx(struct qtnf_pcie_topaz_state *ts)  in qtnf_deassert_intx()  argument
104 	void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;  in qtnf_deassert_intx()
112 static inline int qtnf_topaz_intx_asserted(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_intx_asserted()  argument
114 	void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;  in qtnf_topaz_intx_asserted()
120 static void qtnf_topaz_reset_ep(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_reset_ep()  argument
123 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_topaz_reset_ep()
125 	pci_restore_state(ts->base.pdev);  in qtnf_topaz_reset_ep()
128 static void setup_rx_irqs(struct qtnf_pcie_topaz_state *ts)  in setup_rx_irqs()  argument
130 	void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);  in setup_rx_irqs()
132 	ts->dma_msi_imwr = readl(reg);  in setup_rx_irqs()
135 static void enable_rx_irqs(struct qtnf_pcie_topaz_state *ts)  in enable_rx_irqs()  argument
137 	void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);  in enable_rx_irqs()
139 	qtnf_non_posted_write(ts->dma_msi_imwr, reg);  in enable_rx_irqs()
142 static void disable_rx_irqs(struct qtnf_pcie_topaz_state *ts)  in disable_rx_irqs()  argument
144 	void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);  in disable_rx_irqs()
146 	qtnf_non_posted_write(QTN_HOST_LO32(ts->dma_msi_dummy), reg);  in disable_rx_irqs()
151 	struct qtnf_pcie_topaz_state *ts = arg;  in qtnf_topaz_ipc_gen_ep_int()  local
154 	       TOPAZ_CTL_M2L_INT(ts->base.sysctl_bar));  in qtnf_topaz_ipc_gen_ep_int()
182 static int topaz_alloc_bd_table(struct qtnf_pcie_topaz_state *ts,  in topaz_alloc_bd_table()  argument
186 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in topaz_alloc_bd_table()
206 	ts->tx_bd_vbase = vaddr;  in topaz_alloc_bd_table()
210 		ts->tx_bd_vbase[i].info |= cpu_to_le32(QTN_BD_EMPTY);  in topaz_alloc_bd_table()
222 	ts->rx_bd_vbase = vaddr;  in topaz_alloc_bd_table()
234 	ts->ep_next_rx_pkt = &extra_params->param1;  in topaz_alloc_bd_table()
237 	ts->txqueue_wake = &extra_params->param2;  in topaz_alloc_bd_table()
238 	ts->ep_pmstate = &extra_params->param3;  in topaz_alloc_bd_table()
239 	ts->dma_msi_dummy = paddr + QTNF_BD_PARAM_OFFSET(4);  in topaz_alloc_bd_table()
245 topaz_skb2rbd_attach(struct qtnf_pcie_topaz_state *ts, u16 index, u32 wrap)  in topaz_skb2rbd_attach()  argument
247 	struct qtnf_topaz_rx_bd *rxbd = &ts->rx_bd_vbase[index];  in topaz_skb2rbd_attach()
253 		ts->base.rx_skb[index] = NULL;  in topaz_skb2rbd_attach()
257 	ts->base.rx_skb[index] = skb;  in topaz_skb2rbd_attach()
259 	paddr = pci_map_single(ts->base.pdev, skb->data,  in topaz_skb2rbd_attach()
261 	if (pci_dma_mapping_error(ts->base.pdev, paddr)) {  in topaz_skb2rbd_attach()
269 	ts->base.rx_bd_w_index = index;  in topaz_skb2rbd_attach()
274 static int topaz_alloc_rx_buffers(struct qtnf_pcie_topaz_state *ts)  in topaz_alloc_rx_buffers()  argument
279 	memset(ts->rx_bd_vbase, 0x0,  in topaz_alloc_rx_buffers()
280 	       ts->base.rx_bd_num * sizeof(struct qtnf_topaz_rx_bd));  in topaz_alloc_rx_buffers()
282 	for (i = 0; i < ts->base.rx_bd_num; i++) {  in topaz_alloc_rx_buffers()
283 		ret = topaz_skb2rbd_attach(ts, i, 0);  in topaz_alloc_rx_buffers()
288 	ts->rx_bd_vbase[ts->base.rx_bd_num - 1].info |=  in topaz_alloc_rx_buffers()
295 static void qtnf_topaz_free_xfer_buffers(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_free_xfer_buffers()  argument
297 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_topaz_free_xfer_buffers()
307 			rxbd = &ts->rx_bd_vbase[i];  in qtnf_topaz_free_xfer_buffers()
322 			txbd = &ts->tx_bd_vbase[i];  in qtnf_topaz_free_xfer_buffers()
335 static int qtnf_pcie_topaz_init_xfer(struct qtnf_pcie_topaz_state *ts,  in qtnf_pcie_topaz_init_xfer()  argument
338 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pcie_topaz_init_xfer()
339 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_pcie_topaz_init_xfer()
365 	ret = topaz_alloc_bd_table(ts, bda);  in qtnf_pcie_topaz_init_xfer()
371 	ret = topaz_alloc_rx_buffers(ts);  in qtnf_pcie_topaz_init_xfer()
380 static void qtnf_topaz_data_tx_reclaim(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_data_tx_reclaim()  argument
382 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_topaz_data_tx_reclaim()
393 	tx_done_index = readl(ts->ep_next_rx_pkt);  in qtnf_topaz_data_tx_reclaim()
404 			txbd = &ts->tx_bd_vbase[i];  in qtnf_topaz_data_tx_reclaim()
436 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_try_stop_xmit()  local
440 		ts->base.tx_stopped = 1;  in qtnf_try_stop_xmit()
443 	writel(0x0, ts->txqueue_wake);  in qtnf_try_stop_xmit()
450 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_try_stop_xmit()
453 	tasklet_hi_schedule(&ts->base.reclaim_tq);  in qtnf_try_stop_xmit()
458 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_try_wake_xmit()  local
461 	ready = readl(ts->txqueue_wake);  in qtnf_try_wake_xmit()
467 		       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_try_wake_xmit()
471 static int qtnf_tx_queue_ready(struct qtnf_pcie_topaz_state *ts)  in qtnf_tx_queue_ready()  argument
473 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_tx_queue_ready()
477 		qtnf_topaz_data_tx_reclaim(ts);  in qtnf_tx_queue_ready()
491 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_pcie_data_tx()  local
492 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_pcie_data_tx()
493 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pcie_data_tx()
510 	if (!qtnf_tx_queue_ready(ts)) {  in qtnf_pcie_data_tx()
527 	txbd = &ts->tx_bd_vbase[i];  in qtnf_pcie_data_tx()
555 	qtnf_topaz_data_tx_reclaim(ts);  in qtnf_pcie_data_tx()
563 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_pcie_topaz_interrupt()  local
564 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_pcie_topaz_interrupt()
566 	if (!priv->msi_enabled && !qtnf_topaz_intx_asserted(ts))  in qtnf_pcie_topaz_interrupt()
570 		qtnf_deassert_intx(ts);  in qtnf_pcie_topaz_interrupt()
578 		disable_rx_irqs(ts);  in qtnf_pcie_topaz_interrupt()
587 static int qtnf_rx_data_ready(struct qtnf_pcie_topaz_state *ts)  in qtnf_rx_data_ready()  argument
589 	u16 index = ts->base.rx_bd_r_index;  in qtnf_rx_data_ready()
593 	rxbd = &ts->rx_bd_vbase[index];  in qtnf_rx_data_ready()
605 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_topaz_rx_poll()  local
606 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_topaz_rx_poll()
621 		if (!qtnf_rx_data_ready(ts))  in qtnf_topaz_rx_poll()
625 		rxbd = &ts->rx_bd_vbase[r_idx];  in qtnf_topaz_rx_poll()
676 		if (((++ts->rx_pkt_count) & RX_DONE_INTR_MSK) == 0)  in qtnf_topaz_rx_poll()
693 			ret = topaz_skb2rbd_attach(ts, w_idx,  in qtnf_topaz_rx_poll()
708 		enable_rx_irqs(ts);  in qtnf_topaz_rx_poll()
717 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_data_tx_timeout()  local
720 	tasklet_hi_schedule(&ts->base.reclaim_tq);  in qtnf_pcie_data_tx_timeout()
725 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_data_rx_start()  local
728 	enable_rx_irqs(ts);  in qtnf_pcie_data_rx_start()
733 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_data_rx_stop()  local
735 	disable_rx_irqs(ts);  in qtnf_pcie_data_rx_stop()
753 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_dbg_irq_stats()  local
755 	seq_printf(s, "pcie_irq_count(%u)\n", ts->base.pcie_irq_count);  in qtnf_dbg_irq_stats()
763 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_dbg_pkt_stats()  local
764 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_dbg_pkt_stats()
765 	u32 tx_done_index = readl(ts->ep_next_rx_pkt);  in qtnf_dbg_pkt_stats()
796 static void qtnf_reset_dma_offset(struct qtnf_pcie_topaz_state *ts)  in qtnf_reset_dma_offset()  argument
798 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_reset_dma_offset()
807 static int qtnf_pcie_endian_detect(struct qtnf_pcie_topaz_state *ts)  in qtnf_pcie_endian_detect()  argument
809 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pcie_endian_detect()
848 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_pre_init_ep()  local
849 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pre_init_ep()
853 	ret = qtnf_pcie_endian_detect(ts);  in qtnf_pre_init_ep()
859 	writeb(ts->base.msi_enabled, &ts->bda->bda_rc_msi_enabled);  in qtnf_pre_init_ep()
860 	qtnf_reset_dma_offset(ts);  in qtnf_pre_init_ep()
865 	if (ts->base.flashboot)  in qtnf_pre_init_ep()
872 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_RDY);  in qtnf_pre_init_ep()
873 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_RDY,  in qtnf_pre_init_ep()
882 static int qtnf_post_init_ep(struct qtnf_pcie_topaz_state *ts)  in qtnf_post_init_ep()  argument
884 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_post_init_ep()
886 	setup_rx_irqs(ts);  in qtnf_post_init_ep()
887 	disable_rx_irqs(ts);  in qtnf_post_init_ep()
889 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_QLINK_DONE,  in qtnf_post_init_ep()
898 qtnf_ep_fw_load(struct qtnf_pcie_topaz_state *ts, const u8 *fw, u32 fw_size)  in qtnf_ep_fw_load()  argument
900 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_ep_fw_load()
901 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_ep_fw_load()
915 	blksize = ts->base.fw_blksize;  in qtnf_ep_fw_load()
938 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_LOAD);  in qtnf_ep_fw_load()
939 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_EP_RDY,  in qtnf_ep_fw_load()
956 		qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);  in qtnf_ep_fw_load()
957 		if (qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_ep_fw_load()
973 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);  in qtnf_ep_fw_load()
974 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_DONE,  in qtnf_ep_fw_load()
982 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_END);  in qtnf_ep_fw_load()
983 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_LOAD_DONE,  in qtnf_ep_fw_load()
999 static int qtnf_topaz_fw_upload(struct qtnf_pcie_topaz_state *ts,  in qtnf_topaz_fw_upload()  argument
1003 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_topaz_fw_upload()
1006 	if (qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_upload()
1021 	ret = qtnf_ep_fw_load(ts, fw->data, fw->size);  in qtnf_topaz_fw_upload()
1033 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_topaz_fw_work_handler()  local
1034 	int bootloader_needed = readl(&ts->bda->bda_flags) & QTN_BDA_XMIT_UBOOT;  in qtnf_topaz_fw_work_handler()
1035 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_topaz_fw_work_handler()
1038 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_BOOT);  in qtnf_topaz_fw_work_handler()
1041 		ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_BOOTLD_NAME);  in qtnf_topaz_fw_work_handler()
1049 		qtnf_set_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1053 	if (ts->base.flashboot) {  in qtnf_topaz_fw_work_handler()
1056 		ret = qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1062 		ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_FW_NAME);  in qtnf_topaz_fw_work_handler()
1066 		qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_START);  in qtnf_topaz_fw_work_handler()
1067 		ret = qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1075 		qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_RUN);  in qtnf_topaz_fw_work_handler()
1076 		ret = qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1085 	ret = qtnf_post_init_ep(ts);  in qtnf_topaz_fw_work_handler()
1106 	struct qtnf_pcie_topaz_state *ts = (void *)data;  in qtnf_reclaim_tasklet_fn()  local
1108 	qtnf_topaz_data_tx_reclaim(ts);  in qtnf_reclaim_tasklet_fn()
1118 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_probe()  local
1119 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_pcie_topaz_probe()
1126 	ts->bda = ts->base.epmem_bar;  in qtnf_pcie_topaz_probe()
1129 	if (ts->base.msi_enabled)  in qtnf_pcie_topaz_probe()
1150 	ret = qtnf_pcie_topaz_init_xfer(ts, tx_bd_num);  in qtnf_pcie_topaz_probe()
1156 	tasklet_init(&ts->base.reclaim_tq, qtnf_reclaim_tasklet_fn,  in qtnf_pcie_topaz_probe()
1157 		     (unsigned long)ts);  in qtnf_pcie_topaz_probe()
1162 	ipc_int.arg = ts;  in qtnf_pcie_topaz_probe()
1163 	qtnf_pcie_init_shm_ipc(&ts->base, &ts->bda->bda_shm_reg1,  in qtnf_pcie_topaz_probe()
1164 			       &ts->bda->bda_shm_reg2, &ipc_int);  in qtnf_pcie_topaz_probe()
1171 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_remove()  local
1173 	qtnf_topaz_reset_ep(ts);  in qtnf_pcie_topaz_remove()
1174 	qtnf_topaz_free_xfer_buffers(ts);  in qtnf_pcie_topaz_remove()
1180 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_suspend()  local
1181 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_pcie_topaz_suspend()
1183 	writel((u32 __force)PCI_D3hot, ts->ep_pmstate);  in qtnf_pcie_topaz_suspend()
1186 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_pcie_topaz_suspend()
1197 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_resume()  local
1198 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_pcie_topaz_resume()
1204 	writel((u32 __force)PCI_D0, ts->ep_pmstate);  in qtnf_pcie_topaz_resume()
1207 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_pcie_topaz_resume()
1216 	struct qtnf_pcie_topaz_state *ts;  in qtnf_pcie_topaz_alloc()  local
1218 	bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ts), GFP_KERNEL);  in qtnf_pcie_topaz_alloc()
1222 	ts = get_bus_priv(bus);  in qtnf_pcie_topaz_alloc()
1223 	ts->base.probe_cb = qtnf_pcie_topaz_probe;  in qtnf_pcie_topaz_alloc()
1224 	ts->base.remove_cb = qtnf_pcie_topaz_remove;  in qtnf_pcie_topaz_alloc()
1225 	ts->base.dma_mask_get_cb = qtnf_topaz_dma_mask_get;  in qtnf_pcie_topaz_alloc()
1227 	ts->base.resume_cb = qtnf_pcie_topaz_resume;  in qtnf_pcie_topaz_alloc()
1228 	ts->base.suspend_cb = qtnf_pcie_topaz_suspend;  in qtnf_pcie_topaz_alloc()