52 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read8()  local
54 	return readb(rtwpci->mmap + addr);  in rtw_pci_read8()
59 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read16()  local
61 	return readw(rtwpci->mmap + addr);  in rtw_pci_read16()
66 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read32()  local
68 	return readl(rtwpci->mmap + addr);  in rtw_pci_read32()
73 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write8()  local
75 	writeb(val, rtwpci->mmap + addr);  in rtw_pci_write8()
80 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write16()  local
82 	writew(val, rtwpci->mmap + addr);  in rtw_pci_write16()
87 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write32()  local
89 	writel(val, rtwpci->mmap + addr);  in rtw_pci_write32()
161 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_free_trx_ring()  local
167 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_free_trx_ring()
172 		rx_ring = &rtwpci->rx_rings[i];  in rtw_pci_free_trx_ring()
316 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init_trx_ring()  local
328 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_init_trx_ring()
338 		rx_ring = &rtwpci->rx_rings[j];  in rtw_pci_init_trx_ring()
350 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_init_trx_ring()
356 		rx_ring = &rtwpci->rx_rings[j];  in rtw_pci_init_trx_ring()
370 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init()  local
373 	rtwpci->irq_mask[0] = IMR_HIGHDOK |  in rtw_pci_init()
383 	rtwpci->irq_mask[1] = IMR_TXFOVW |  in rtw_pci_init()
385 	rtwpci->irq_mask[3] = IMR_H2CDOK |  in rtw_pci_init()
387 	spin_lock_init(&rtwpci->irq_lock);  in rtw_pci_init()
388 	spin_lock_init(&rtwpci->hwirq_lock);  in rtw_pci_init()
396 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_reset_buf_desc()  local
404 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;  in rtw_pci_reset_buf_desc()
408 		len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;  in rtw_pci_reset_buf_desc()
409 		dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;  in rtw_pci_reset_buf_desc()
410 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;  in rtw_pci_reset_buf_desc()
411 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;  in rtw_pci_reset_buf_desc()
416 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;  in rtw_pci_reset_buf_desc()
417 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;  in rtw_pci_reset_buf_desc()
418 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;  in rtw_pci_reset_buf_desc()
419 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;  in rtw_pci_reset_buf_desc()
423 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;  in rtw_pci_reset_buf_desc()
424 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;  in rtw_pci_reset_buf_desc()
425 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;  in rtw_pci_reset_buf_desc()
426 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;  in rtw_pci_reset_buf_desc()
430 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;  in rtw_pci_reset_buf_desc()
431 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;  in rtw_pci_reset_buf_desc()
432 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;  in rtw_pci_reset_buf_desc()
433 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;  in rtw_pci_reset_buf_desc()
437 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;  in rtw_pci_reset_buf_desc()
438 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;  in rtw_pci_reset_buf_desc()
439 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;  in rtw_pci_reset_buf_desc()
440 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;  in rtw_pci_reset_buf_desc()
444 	len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;  in rtw_pci_reset_buf_desc()
445 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;  in rtw_pci_reset_buf_desc()
446 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;  in rtw_pci_reset_buf_desc()
447 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;  in rtw_pci_reset_buf_desc()
451 	len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;  in rtw_pci_reset_buf_desc()
452 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;  in rtw_pci_reset_buf_desc()
453 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;  in rtw_pci_reset_buf_desc()
454 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;  in rtw_pci_reset_buf_desc()
458 	len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;  in rtw_pci_reset_buf_desc()
459 	dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;  in rtw_pci_reset_buf_desc()
460 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;  in rtw_pci_reset_buf_desc()
461 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;  in rtw_pci_reset_buf_desc()
480 				     struct rtw_pci *rtwpci, bool exclude_rx)  in rtw_pci_enable_interrupt()  argument
485 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_enable_interrupt()
487 	rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);  in rtw_pci_enable_interrupt()
488 	rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);  in rtw_pci_enable_interrupt()
490 		rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);  in rtw_pci_enable_interrupt()
492 	rtwpci->irq_enabled = true;  in rtw_pci_enable_interrupt()
494 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_enable_interrupt()
498 				      struct rtw_pci *rtwpci)  in rtw_pci_disable_interrupt()  argument
502 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_disable_interrupt()
504 	if (!rtwpci->irq_enabled)  in rtw_pci_disable_interrupt()
512 	rtwpci->irq_enabled = false;  in rtw_pci_disable_interrupt()
515 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_disable_interrupt()
518 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_reset()  argument
523 	rtwpci->rx_tag = 0;  in rtw_pci_dma_reset()
528 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup()  local
531 	rtw_pci_dma_reset(rtwdev, rtwpci);  in rtw_pci_setup()
536 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_release()  argument
543 		tx_ring = &rtwpci->tx_rings[queue];  in rtw_pci_dma_release()
550 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_start()  local
552 	if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))  in rtw_pci_napi_start()
555 	napi_enable(&rtwpci->napi);  in rtw_pci_napi_start()
560 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_stop()  local
562 	if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))  in rtw_pci_napi_stop()
565 	napi_synchronize(&rtwpci->napi);  in rtw_pci_napi_stop()
566 	napi_disable(&rtwpci->napi);  in rtw_pci_napi_stop()
571 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_start()  local
575 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_start()
576 	rtwpci->running = true;  in rtw_pci_start()
577 	rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_start()
578 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_start()
585 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_stop()  local
586 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_stop()
588 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
589 	rtwpci->running = false;  in rtw_pci_stop()
590 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_stop()
591 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
596 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
597 	rtw_pci_dma_release(rtwdev, rtwpci);  in rtw_pci_stop()
598 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
603 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_enter()  local
611 	lockdep_assert_held(&rtwpci->irq_lock);  in rtw_pci_deep_ps_enter()
622 		tx_ring = &rtwpci->tx_rings[queue];  in rtw_pci_deep_ps_enter()
643 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_leave()  local
645 	lockdep_assert_held(&rtwpci->irq_lock);  in rtw_pci_deep_ps_leave()
653 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps()  local
655 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_deep_ps()
663 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_deep_ps()
666 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,  in rtw_pci_release_rsvd_page()  argument
678 	dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);  in rtw_pci_release_rsvd_page()
686 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_dma_check()  local
697 	if (total_pkt_size != rtwpci->rx_tag)  in rtw_pci_dma_check()
700 	rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;  in rtw_pci_dma_check()
713 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in __pci_flush_queue()  local
714 	struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];  in __pci_flush_queue()
773 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off_queue()  local
777 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_kick_off_queue()
780 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_kick_off_queue()
784 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_kick_off_queue()
789 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off()  local
793 		if (test_and_clear_bit(queue, rtwpci->tx_queued))  in rtw_pci_tx_kick_off()
802 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write_data()  local
814 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_write_data()
819 		rtw_pci_release_rsvd_page(rtwpci, ring);  in rtw_pci_tx_write_data()
827 	dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,  in rtw_pci_tx_write_data()
829 	if (dma_mapping_error(&rtwpci->pdev->dev, dma))  in rtw_pci_tx_write_data()
849 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write_data()
857 	set_bit(queue, rtwpci->tx_queued);  in rtw_pci_tx_write_data()
862 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write_data()
919 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write()  local
927 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_write()
928 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write()
933 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write()
938 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_tx_isr()  argument
951 	ring = &rtwpci->tx_rings[hw_queue];  in rtw_pci_tx_isr()
971 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,  in rtw_pci_tx_isr()
1015 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_rx_isr()  local
1016 	struct napi_struct *napi = &rtwpci->napi;  in rtw_pci_rx_isr()
1022 				     struct rtw_pci *rtwpci)  in rtw_pci_get_hw_rx_ring_nr()  argument
1028 	ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];  in rtw_pci_get_hw_rx_ring_nr()
1039 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_rx_napi()  argument
1043 	struct napi_struct *napi = &rtwpci->napi;  in rtw_pci_rx_napi()
1044 	struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];  in rtw_pci_rx_napi()
1057 	count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);  in rtw_pci_rx_napi()
1117 				   struct rtw_pci *rtwpci, u32 *irq_status)  in rtw_pci_irq_recognized()  argument
1121 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_irq_recognized()
1129 	irq_status[0] &= rtwpci->irq_mask[0];  in rtw_pci_irq_recognized()
1130 	irq_status[1] &= rtwpci->irq_mask[1];  in rtw_pci_irq_recognized()
1131 	irq_status[3] &= rtwpci->irq_mask[3];  in rtw_pci_irq_recognized()
1137 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_irq_recognized()
1143 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_handler()  local
1153 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_interrupt_handler()
1161 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_threadfn()  local
1165 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_interrupt_threadfn()
1166 	rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);  in rtw_pci_interrupt_threadfn()
1169 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);  in rtw_pci_interrupt_threadfn()
1171 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);  in rtw_pci_interrupt_threadfn()
1173 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);  in rtw_pci_interrupt_threadfn()
1175 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);  in rtw_pci_interrupt_threadfn()
1177 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);  in rtw_pci_interrupt_threadfn()
1179 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);  in rtw_pci_interrupt_threadfn()
1181 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);  in rtw_pci_interrupt_threadfn()
1190 	if (rtwpci->running)  in rtw_pci_interrupt_threadfn()
1191 		rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);  in rtw_pci_interrupt_threadfn()
1192 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_interrupt_threadfn()
1200 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_mapping()  local
1212 	rtwpci->mmap = pci_iomap(pdev, bar_id, len);  in rtw_pci_io_mapping()
1213 	if (!rtwpci->mmap) {  in rtw_pci_io_mapping()
1225 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_unmapping()  local
1227 	if (rtwpci->mmap) {  in rtw_pci_io_unmapping()
1228 		pci_iounmap(pdev, rtwpci->mmap);  in rtw_pci_io_unmapping()
1372 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_ps()  local
1384 	if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))  in rtw_pci_link_ps()
1387 	if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||  in rtw_pci_link_ps()
1388 	    (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))  in rtw_pci_link_ps()
1395 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_cfg()  local
1396 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_link_cfg()
1431 	rtwpci->link_ctrl = link_ctrl;  in rtw_pci_link_cfg()
1451 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_phy_cfg()  local
1453 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_phy_cfg()
1554 	struct rtw_pci *rtwpci;  in rtw_pci_setup_resource()  local
1557 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup_resource()
1558 	rtwpci->pdev = pdev;  in rtw_pci_setup_resource()
1643 	struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);  in rtw_pci_napi_poll()  local
1644 	struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,  in rtw_pci_napi_poll()
1648 	if (rtwpci->rx_no_aspm)  in rtw_pci_napi_poll()
1654 		work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,  in rtw_pci_napi_poll()
1662 		spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_napi_poll()
1663 		if (rtwpci->running)  in rtw_pci_napi_poll()
1664 			rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_napi_poll()
1665 		spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_napi_poll()
1671 		if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))  in rtw_pci_napi_poll()
1674 	if (rtwpci->rx_no_aspm)  in rtw_pci_napi_poll()
1682 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_init()  local
1684 	init_dummy_netdev(&rtwpci->netdev);  in rtw_pci_napi_init()
1685 	netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);  in rtw_pci_napi_init()
1690 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_deinit()  local
1693 	netif_napi_del(&rtwpci->napi);  in rtw_pci_napi_deinit()
1702 	struct rtw_pci *rtwpci;  in rtw_pci_probe()  local
1720 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_probe()
1721 	atomic_set(&rtwpci->link_usage, 1);  in rtw_pci_probe()
1753 		rtwpci->rx_no_aspm = true;  in rtw_pci_probe()
1792 	struct rtw_pci *rtwpci;  in rtw_pci_remove()  local
1798 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_remove()
1801 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_remove()