49 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr)  in rtw_pci_read8()  argument
51 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read8()
56 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr)  in rtw_pci_read16()  argument
58 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read16()
63 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr)  in rtw_pci_read32()  argument
65 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read32()
70 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val)  in rtw_pci_write8()  argument
72 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write8()
77 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val)  in rtw_pci_write16()  argument
79 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write16()
84 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)  in rtw_pci_write32()  argument
86 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write32()
98 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev,  in rtw_pci_free_tx_ring_skbs()  argument
101 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_free_tx_ring_skbs()
117 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev,  in rtw_pci_free_tx_ring()  argument
120 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_free_tx_ring()
125 	rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);  in rtw_pci_free_tx_ring()
132 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev,  in rtw_pci_free_rx_ring_skbs()  argument
135 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_free_rx_ring_skbs()
153 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev,  in rtw_pci_free_rx_ring()  argument
156 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_free_rx_ring()
160 	rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring);  in rtw_pci_free_rx_ring()
165 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev)  in rtw_pci_free_trx_ring()  argument
167 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_free_trx_ring()
174 		rtw_pci_free_tx_ring(rtwdev, tx_ring);  in rtw_pci_free_trx_ring()
179 		rtw_pci_free_rx_ring(rtwdev, rx_ring);  in rtw_pci_free_trx_ring()
183 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev,  in rtw_pci_init_tx_ring()  argument
187 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_init_tx_ring()
193 		rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len);  in rtw_pci_init_tx_ring()
199 		rtw_err(rtwdev, "failed to allocate tx ring\n");  in rtw_pci_init_tx_ring()
214 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb,  in rtw_pci_reset_rx_desc()  argument
218 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_reset_rx_desc()
240 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma,  in rtw_pci_sync_rx_desc_device()  argument
244 	struct device *dev = rtwdev->dev;  in rtw_pci_sync_rx_desc_device()
257 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev,  in rtw_pci_init_rx_ring()  argument
261 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);  in rtw_pci_init_rx_ring()
272 		rtw_err(rtwdev, "failed to allocate rx ring\n");  in rtw_pci_init_rx_ring()
287 		ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size);  in rtw_pci_init_rx_ring()
315 	rtw_err(rtwdev, "failed to init rx buffer\n");  in rtw_pci_init_rx_ring()
320 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev)  in rtw_pci_init_trx_ring()  argument
322 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init_trx_ring()
325 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_init_trx_ring()
336 		ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len);  in rtw_pci_init_trx_ring()
345 		ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size,  in rtw_pci_init_trx_ring()
357 		rtw_pci_free_tx_ring(rtwdev, tx_ring);  in rtw_pci_init_trx_ring()
363 		rtw_pci_free_rx_ring(rtwdev, rx_ring);  in rtw_pci_init_trx_ring()
369 static void rtw_pci_deinit(struct rtw_dev *rtwdev)  in rtw_pci_deinit()  argument
371 	rtw_pci_free_trx_ring(rtwdev);  in rtw_pci_deinit()
374 static int rtw_pci_init(struct rtw_dev *rtwdev)  in rtw_pci_init()  argument
376 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init()
395 	ret = rtw_pci_init_trx_ring(rtwdev);  in rtw_pci_init()
400 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)  in rtw_pci_reset_buf_desc()  argument
402 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_reset_buf_desc()
407 	tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3);  in rtw_pci_reset_buf_desc()
408 	rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7);  in rtw_pci_reset_buf_desc()
411 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma);  in rtw_pci_reset_buf_desc()
413 	if (!rtw_chip_wcpu_11n(rtwdev)) {  in rtw_pci_reset_buf_desc()
418 		rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
419 		rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma);  in rtw_pci_reset_buf_desc()
426 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
427 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma);  in rtw_pci_reset_buf_desc()
433 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
434 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma);  in rtw_pci_reset_buf_desc()
440 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
441 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma);  in rtw_pci_reset_buf_desc()
447 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
448 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma);  in rtw_pci_reset_buf_desc()
454 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
455 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma);  in rtw_pci_reset_buf_desc()
461 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
462 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma);  in rtw_pci_reset_buf_desc()
468 	rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK);  in rtw_pci_reset_buf_desc()
469 	rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma);  in rtw_pci_reset_buf_desc()
472 	rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff);  in rtw_pci_reset_buf_desc()
475 	if (rtw_chip_wcpu_11ac(rtwdev))  in rtw_pci_reset_buf_desc()
476 		rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR,  in rtw_pci_reset_buf_desc()
480 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev)  in rtw_pci_reset_trx_ring()  argument
482 	rtw_pci_reset_buf_desc(rtwdev);  in rtw_pci_reset_trx_ring()
485 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev,  in rtw_pci_enable_interrupt()  argument
493 	rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);  in rtw_pci_enable_interrupt()
494 	rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);  in rtw_pci_enable_interrupt()
495 	if (rtw_chip_wcpu_11ac(rtwdev))  in rtw_pci_enable_interrupt()
496 		rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);  in rtw_pci_enable_interrupt()
503 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev,  in rtw_pci_disable_interrupt()  argument
513 	rtw_write32(rtwdev, RTK_PCI_HIMR0, 0);  in rtw_pci_disable_interrupt()
514 	rtw_write32(rtwdev, RTK_PCI_HIMR1, 0);  in rtw_pci_disable_interrupt()
515 	if (rtw_chip_wcpu_11ac(rtwdev))  in rtw_pci_disable_interrupt()
516 		rtw_write32(rtwdev, RTK_PCI_HIMR3, 0);  in rtw_pci_disable_interrupt()
524 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_reset()  argument
527 	rtw_write32_set(rtwdev, RTK_PCI_CTRL,  in rtw_pci_dma_reset()
532 static int rtw_pci_setup(struct rtw_dev *rtwdev)  in rtw_pci_setup()  argument
534 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup()
536 	rtw_pci_reset_trx_ring(rtwdev);  in rtw_pci_setup()
537 	rtw_pci_dma_reset(rtwdev, rtwpci);  in rtw_pci_setup()
542 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_release()  argument
547 	rtw_pci_reset_trx_ring(rtwdev);  in rtw_pci_dma_release()
550 		rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);  in rtw_pci_dma_release()
554 static void rtw_pci_napi_start(struct rtw_dev *rtwdev)  in rtw_pci_napi_start()  argument
556 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_start()
564 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev)  in rtw_pci_napi_stop()  argument
566 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_stop()
575 static int rtw_pci_start(struct rtw_dev *rtwdev)  in rtw_pci_start()  argument
577 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_start()
579 	rtw_pci_napi_start(rtwdev);  in rtw_pci_start()
583 	rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_start()
589 static void rtw_pci_stop(struct rtw_dev *rtwdev)  in rtw_pci_stop()  argument
591 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_stop()
596 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_stop()
600 	rtw_pci_napi_stop(rtwdev);  in rtw_pci_stop()
603 	rtw_pci_dma_release(rtwdev, rtwpci);  in rtw_pci_stop()
607 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev)  in rtw_pci_deep_ps_enter()  argument
609 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_enter()
614 	if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))  in rtw_pci_deep_ps_enter()
638 		rtw_dbg(rtwdev, RTW_DBG_PS,  in rtw_pci_deep_ps_enter()
643 	set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags);  in rtw_pci_deep_ps_enter()
644 	rtw_power_mode_change(rtwdev, true);  in rtw_pci_deep_ps_enter()
647 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev)  in rtw_pci_deep_ps_leave()  argument
649 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_leave()
653 	if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))  in rtw_pci_deep_ps_leave()
654 		rtw_power_mode_change(rtwdev, false);  in rtw_pci_deep_ps_leave()
657 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter)  in rtw_pci_deep_ps()  argument
659 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps()
663 	if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))  in rtw_pci_deep_ps()
664 		rtw_pci_deep_ps_enter(rtwdev);  in rtw_pci_deep_ps()
666 	if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))  in rtw_pci_deep_ps()
667 		rtw_pci_deep_ps_leave(rtwdev);  in rtw_pci_deep_ps()
719 static void rtw_pci_dma_check(struct rtw_dev *rtwdev,  in rtw_pci_dma_check()  argument
723 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_dma_check()
724 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_dma_check()
735 		rtw_warn(rtwdev, "pci bus timeout, check dma status\n");  in rtw_pci_dma_check()
740 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q)  in __pci_get_hw_tx_ring_rp()  argument
743 	u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2);  in __pci_get_hw_tx_ring_rp()
748 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop)  in __pci_flush_queue()  argument
750 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in __pci_flush_queue()
761 		cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q);  in __pci_flush_queue()
769 		rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q);  in __pci_flush_queue()
772 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues,  in __rtw_pci_flush_queues()  argument
783 			__pci_flush_queue(rtwdev, q, drop);  in __rtw_pci_flush_queues()
787 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop)  in rtw_pci_flush_queues()  argument
795 	if (queues == BIT(rtwdev->hw->queues) - 1) {  in rtw_pci_flush_queues()
798 		for (i = 0; i < rtwdev->hw->queues; i++)  in rtw_pci_flush_queues()
803 	__rtw_pci_flush_queues(rtwdev, pci_queues, drop);  in rtw_pci_flush_queues()
806 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, u8 queue)  in rtw_pci_tx_kick_off_queue()  argument
808 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off_queue()
816 	if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))  in rtw_pci_tx_kick_off_queue()
817 		rtw_pci_deep_ps_leave(rtwdev);  in rtw_pci_tx_kick_off_queue()
818 	rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK);  in rtw_pci_tx_kick_off_queue()
822 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev)  in rtw_pci_tx_kick_off()  argument
824 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off()
829 			rtw_pci_tx_kick_off_queue(rtwdev, queue);  in rtw_pci_tx_kick_off()
832 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev,  in rtw_pci_tx_write_data()  argument
836 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write_data()
837 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_tx_write_data()
901 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf,  in rtw_pci_write_data_rsvd_page()  argument
909 	skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size);  in rtw_pci_write_data_rsvd_page()
913 	ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN);  in rtw_pci_write_data_rsvd_page()
915 		rtw_err(rtwdev, "failed to write rsvd page data\n");  in rtw_pci_write_data_rsvd_page()
920 	reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK);  in rtw_pci_write_data_rsvd_page()
922 	rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work);  in rtw_pci_write_data_rsvd_page()
927 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size)  in rtw_pci_write_data_h2c()  argument
933 	skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size);  in rtw_pci_write_data_h2c()
937 	ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C);  in rtw_pci_write_data_h2c()
939 		rtw_err(rtwdev, "failed to write h2c data\n");  in rtw_pci_write_data_h2c()
943 	rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C);  in rtw_pci_write_data_h2c()
948 static int rtw_pci_tx_write(struct rtw_dev *rtwdev,  in rtw_pci_tx_write()  argument
952 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write()
957 	ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue);  in rtw_pci_tx_write()
964 		ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb));  in rtw_pci_tx_write()
972 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_tx_isr()  argument
975 	struct ieee80211_hw *hw = rtwdev->hw;  in rtw_pci_tx_isr()
988 	bd_idx = rtw_read32(rtwdev, bd_idx_addr);  in rtw_pci_tx_isr()
1000 			rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n",  in rtw_pci_tx_isr()
1024 		skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz);  in rtw_pci_tx_isr()
1030 			rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn);  in rtw_pci_tx_isr()
1047 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev)  in rtw_pci_rx_isr()  argument
1049 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_rx_isr()
1055 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev,  in rtw_pci_get_hw_rx_ring_nr()  argument
1063 	tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ);  in rtw_pci_get_hw_rx_ring_nr()
1073 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_rx_napi()  argument
1076 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_rx_napi()
1091 	count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);  in rtw_pci_rx_napi()
1095 		rtw_pci_dma_check(rtwdev, ring, cur_rp);  in rtw_pci_rx_napi()
1098 		dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE,  in rtw_pci_rx_napi()
1101 		chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status);  in rtw_pci_rx_napi()
1119 			rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new);  in rtw_pci_rx_napi()
1124 			rtw_rx_stats(rtwdev, pkt_stat.vif, new);  in rtw_pci_rx_napi()
1126 			ieee80211_rx_napi(rtwdev->hw, NULL, new, napi);  in rtw_pci_rx_napi()
1132 		rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp,  in rtw_pci_rx_napi()
1145 	rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp);  in rtw_pci_rx_napi()
1150 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,  in rtw_pci_irq_recognized()  argument
1157 	irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0);  in rtw_pci_irq_recognized()
1158 	irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1);  in rtw_pci_irq_recognized()
1159 	if (rtw_chip_wcpu_11ac(rtwdev))  in rtw_pci_irq_recognized()
1160 		irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3);  in rtw_pci_irq_recognized()
1166 	rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]);  in rtw_pci_irq_recognized()
1167 	rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]);  in rtw_pci_irq_recognized()
1168 	if (rtw_chip_wcpu_11ac(rtwdev))  in rtw_pci_irq_recognized()
1169 		rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]);  in rtw_pci_irq_recognized()
1176 	struct rtw_dev *rtwdev = dev;  in rtw_pci_interrupt_handler()  local
1177 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_handler()
1187 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_interrupt_handler()
1194 	struct rtw_dev *rtwdev = dev;  in rtw_pci_interrupt_threadfn()  local
1195 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_threadfn()
1200 	rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);  in rtw_pci_interrupt_threadfn()
1203 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);  in rtw_pci_interrupt_threadfn()
1205 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);  in rtw_pci_interrupt_threadfn()
1207 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);  in rtw_pci_interrupt_threadfn()
1209 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);  in rtw_pci_interrupt_threadfn()
1211 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);  in rtw_pci_interrupt_threadfn()
1213 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);  in rtw_pci_interrupt_threadfn()
1215 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);  in rtw_pci_interrupt_threadfn()
1217 		rtw_pci_rx_isr(rtwdev);  in rtw_pci_interrupt_threadfn()
1221 		rtw_fw_c2h_cmd_isr(rtwdev);  in rtw_pci_interrupt_threadfn()
1225 		rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);  in rtw_pci_interrupt_threadfn()
1231 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev,  in rtw_pci_io_mapping()  argument
1234 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_mapping()
1241 		rtw_err(rtwdev, "failed to request pci regions\n");  in rtw_pci_io_mapping()
1249 		rtw_err(rtwdev, "failed to map pci memory\n");  in rtw_pci_io_mapping()
1256 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev,  in rtw_pci_io_unmapping()  argument
1259 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_unmapping()
1267 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data)  in rtw_dbi_write8()  argument
1276 	rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data);  in rtw_dbi_write8()
1277 	rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr);  in rtw_dbi_write8()
1278 	rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16);  in rtw_dbi_write8()
1281 		flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);  in rtw_dbi_write8()
1291 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value)  in rtw_dbi_read8()  argument
1297 	rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr);  in rtw_dbi_read8()
1298 	rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16);  in rtw_dbi_read8()
1301 		flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);  in rtw_dbi_read8()
1304 			*value = rtw_read8(rtwdev, read_addr);  in rtw_dbi_read8()
1315 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1)  in rtw_mdio_write()  argument
1321 	rtw_write16(rtwdev, REG_MDIO_V1, data);  in rtw_mdio_write()
1325 	rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK);  in rtw_mdio_write()
1326 	rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page);  in rtw_mdio_write()
1327 	rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1);  in rtw_mdio_write()
1330 		wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG,  in rtw_mdio_write()
1341 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable)  in rtw_pci_clkreq_set()  argument
1349 	ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);  in rtw_pci_clkreq_set()
1351 		rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);  in rtw_pci_clkreq_set()
1360 	rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);  in rtw_pci_clkreq_set()
1363 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable)  in rtw_pci_clkreq_pad_low()  argument
1368 	ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);  in rtw_pci_clkreq_pad_low()
1370 		rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);  in rtw_pci_clkreq_pad_low()
1379 	rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);  in rtw_pci_clkreq_pad_low()
1382 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable)  in rtw_pci_aspm_set()  argument
1390 	ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);  in rtw_pci_aspm_set()
1392 		rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret);  in rtw_pci_aspm_set()
1401 	rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);  in rtw_pci_aspm_set()
1404 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)  in rtw_pci_link_ps()  argument
1406 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_ps()
1423 		rtw_pci_aspm_set(rtwdev, enter);  in rtw_pci_link_ps()
1426 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)  in rtw_pci_link_cfg()  argument
1428 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_link_cfg()
1429 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_cfg()
1438 		rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0);  in rtw_pci_link_cfg()
1458 		rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);  in rtw_pci_link_cfg()
1463 		rtw_pci_clkreq_set(rtwdev, true);  in rtw_pci_link_cfg()
1468 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)  in rtw_pci_interface_cfg()  argument
1470 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_interface_cfg()
1474 		if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D)  in rtw_pci_interface_cfg()
1475 			rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG,  in rtw_pci_interface_cfg()
1483 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev)  in rtw_pci_phy_cfg()  argument
1485 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_phy_cfg()
1486 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_phy_cfg()
1495 	cut = BIT(0) << rtwdev->hal.cut_version;  in rtw_pci_phy_cfg()
1506 			rtw_mdio_write(rtwdev, offset, value, true);  in rtw_pci_phy_cfg()
1508 			rtw_dbi_write8(rtwdev, offset, value);  in rtw_pci_phy_cfg()
1520 			rtw_mdio_write(rtwdev, offset, value, false);  in rtw_pci_phy_cfg()
1522 			rtw_dbi_write8(rtwdev, offset, value);  in rtw_pci_phy_cfg()
1525 	rtw_pci_link_cfg(rtwdev);  in rtw_pci_phy_cfg()
1532 			rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n",  in rtw_pci_phy_cfg()
1540 	struct rtw_dev *rtwdev = hw->priv;  in rtw_pci_suspend()  local
1541 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_suspend()
1542 	struct rtw_efuse *efuse = &rtwdev->efuse;  in rtw_pci_suspend()
1545 		rtw_pci_clkreq_pad_low(rtwdev, true);  in rtw_pci_suspend()
1552 	struct rtw_dev *rtwdev = hw->priv;  in rtw_pci_resume()  local
1553 	const struct rtw_chip_info *chip = rtwdev->chip;  in rtw_pci_resume()
1554 	struct rtw_efuse *efuse = &rtwdev->efuse;  in rtw_pci_resume()
1557 		rtw_pci_clkreq_pad_low(rtwdev, false);  in rtw_pci_resume()
1564 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev)  in rtw_pci_claim()  argument
1570 		rtw_err(rtwdev, "failed to enable pci device\n");  in rtw_pci_claim()
1575 	pci_set_drvdata(pdev, rtwdev->hw);  in rtw_pci_claim()
1576 	SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);  in rtw_pci_claim()
1581 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev)  in rtw_pci_declaim()  argument
1587 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev)  in rtw_pci_setup_resource()  argument
1592 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup_resource()
1596 	ret = rtw_pci_io_mapping(rtwdev, pdev);  in rtw_pci_setup_resource()
1598 		rtw_err(rtwdev, "failed to request pci io region\n");  in rtw_pci_setup_resource()
1602 	ret = rtw_pci_init(rtwdev);  in rtw_pci_setup_resource()
1604 		rtw_err(rtwdev, "failed to allocate pci resources\n");  in rtw_pci_setup_resource()
1611 	rtw_pci_io_unmapping(rtwdev, pdev);  in rtw_pci_setup_resource()
1617 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev)  in rtw_pci_destroy()  argument
1619 	rtw_pci_deinit(rtwdev);  in rtw_pci_destroy()
1620 	rtw_pci_io_unmapping(rtwdev, pdev);  in rtw_pci_destroy()
1644 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)  in rtw_pci_request_irq()  argument
1654 		rtw_err(rtwdev, "failed to alloc PCI irq vectors\n");  in rtw_pci_request_irq()
1658 	ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,  in rtw_pci_request_irq()
1661 					IRQF_SHARED, KBUILD_MODNAME, rtwdev);  in rtw_pci_request_irq()
1663 		rtw_err(rtwdev, "failed to request irq %d\n", ret);  in rtw_pci_request_irq()
1670 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)  in rtw_pci_free_irq()  argument
1672 	devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);  in rtw_pci_free_irq()
1679 	struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,  in rtw_pci_napi_poll()  local
1684 		rtw_pci_link_ps(rtwdev, false);  in rtw_pci_napi_poll()
1689 		work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,  in rtw_pci_napi_poll()
1699 			rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_napi_poll()
1706 		if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))  in rtw_pci_napi_poll()
1710 		rtw_pci_link_ps(rtwdev, true);  in rtw_pci_napi_poll()
1715 static void rtw_pci_napi_init(struct rtw_dev *rtwdev)  in rtw_pci_napi_init()  argument
1717 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_init()
1723 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)  in rtw_pci_napi_deinit()  argument
1725 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_deinit()
1727 	rtw_pci_napi_stop(rtwdev);  in rtw_pci_napi_deinit()
1736 	struct rtw_dev *rtwdev;  in rtw_pci_probe()  local
1748 	rtwdev = hw->priv;  in rtw_pci_probe()
1749 	rtwdev->hw = hw;  in rtw_pci_probe()
1750 	rtwdev->dev = &pdev->dev;  in rtw_pci_probe()
1751 	rtwdev->chip = (struct rtw_chip_info *)id->driver_data;  in rtw_pci_probe()
1752 	rtwdev->hci.ops = &rtw_pci_ops;  in rtw_pci_probe()
1753 	rtwdev->hci.type = RTW_HCI_TYPE_PCIE;  in rtw_pci_probe()
1755 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_probe()
1758 	ret = rtw_core_init(rtwdev);  in rtw_pci_probe()
1762 	rtw_dbg(rtwdev, RTW_DBG_PCI,  in rtw_pci_probe()
1766 	ret = rtw_pci_claim(rtwdev, pdev);  in rtw_pci_probe()
1768 		rtw_err(rtwdev, "failed to claim pci device\n");  in rtw_pci_probe()
1772 	ret = rtw_pci_setup_resource(rtwdev, pdev);  in rtw_pci_probe()
1774 		rtw_err(rtwdev, "failed to setup pci resources\n");  in rtw_pci_probe()
1778 	rtw_pci_napi_init(rtwdev);  in rtw_pci_probe()
1780 	ret = rtw_chip_info_setup(rtwdev);  in rtw_pci_probe()
1782 		rtw_err(rtwdev, "failed to setup chip information\n");  in rtw_pci_probe()
1787 	if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL)  in rtw_pci_probe()
1790 	rtw_pci_phy_cfg(rtwdev);  in rtw_pci_probe()
1792 	ret = rtw_register_hw(rtwdev, hw);  in rtw_pci_probe()
1794 		rtw_err(rtwdev, "failed to register hw\n");  in rtw_pci_probe()
1798 	ret = rtw_pci_request_irq(rtwdev, pdev);  in rtw_pci_probe()
1807 	rtw_pci_napi_deinit(rtwdev);  in rtw_pci_probe()
1808 	rtw_pci_destroy(rtwdev, pdev);  in rtw_pci_probe()
1811 	rtw_pci_declaim(rtwdev, pdev);  in rtw_pci_probe()
1814 	rtw_core_deinit(rtwdev);  in rtw_pci_probe()
1826 	struct rtw_dev *rtwdev;  in rtw_pci_remove()  local
1832 	rtwdev = hw->priv;  in rtw_pci_remove()
1833 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_remove()
1835 	rtw_unregister_hw(rtwdev, hw);  in rtw_pci_remove()
1836 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_remove()
1837 	rtw_pci_napi_deinit(rtwdev);  in rtw_pci_remove()
1838 	rtw_pci_destroy(rtwdev, pdev);  in rtw_pci_remove()
1839 	rtw_pci_declaim(rtwdev, pdev);  in rtw_pci_remove()
1840 	rtw_pci_free_irq(rtwdev, pdev);  in rtw_pci_remove()
1841 	rtw_core_deinit(rtwdev);  in rtw_pci_remove()
1849 	struct rtw_dev *rtwdev;  in rtw_pci_shutdown()  local
1855 	rtwdev = hw->priv;  in rtw_pci_shutdown()
1856 	chip = rtwdev->chip;  in rtw_pci_shutdown()
1859 		chip->ops->shutdown(rtwdev);  in rtw_pci_shutdown()