51 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read8()  local
53 	return readb(rtwpci->mmap + addr);  in rtw_pci_read8()
58 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read16()  local
60 	return readw(rtwpci->mmap + addr);  in rtw_pci_read16()
65 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read32()  local
67 	return readl(rtwpci->mmap + addr);  in rtw_pci_read32()
72 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write8()  local
74 	writeb(val, rtwpci->mmap + addr);  in rtw_pci_write8()
79 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write16()  local
81 	writew(val, rtwpci->mmap + addr);  in rtw_pci_write16()
86 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write32()  local
88 	writel(val, rtwpci->mmap + addr);  in rtw_pci_write32()
167 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_free_trx_ring()  local
173 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_free_trx_ring()
178 		rx_ring = &rtwpci->rx_rings[i];  in rtw_pci_free_trx_ring()
322 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init_trx_ring()  local
334 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_init_trx_ring()
344 		rx_ring = &rtwpci->rx_rings[j];  in rtw_pci_init_trx_ring()
356 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_init_trx_ring()
362 		rx_ring = &rtwpci->rx_rings[j];  in rtw_pci_init_trx_ring()
376 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init()  local
379 	rtwpci->irq_mask[0] = IMR_HIGHDOK |  in rtw_pci_init()
389 	rtwpci->irq_mask[1] = IMR_TXFOVW |  in rtw_pci_init()
391 	rtwpci->irq_mask[3] = IMR_H2CDOK |  in rtw_pci_init()
393 	spin_lock_init(&rtwpci->irq_lock);  in rtw_pci_init()
394 	spin_lock_init(&rtwpci->hwirq_lock);  in rtw_pci_init()
402 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_reset_buf_desc()  local
410 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;  in rtw_pci_reset_buf_desc()
414 		len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;  in rtw_pci_reset_buf_desc()
415 		dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;  in rtw_pci_reset_buf_desc()
416 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;  in rtw_pci_reset_buf_desc()
417 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;  in rtw_pci_reset_buf_desc()
422 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;  in rtw_pci_reset_buf_desc()
423 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;  in rtw_pci_reset_buf_desc()
424 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;  in rtw_pci_reset_buf_desc()
425 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;  in rtw_pci_reset_buf_desc()
429 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;  in rtw_pci_reset_buf_desc()
430 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;  in rtw_pci_reset_buf_desc()
431 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;  in rtw_pci_reset_buf_desc()
432 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;  in rtw_pci_reset_buf_desc()
436 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;  in rtw_pci_reset_buf_desc()
437 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;  in rtw_pci_reset_buf_desc()
438 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;  in rtw_pci_reset_buf_desc()
439 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;  in rtw_pci_reset_buf_desc()
443 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;  in rtw_pci_reset_buf_desc()
444 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;  in rtw_pci_reset_buf_desc()
445 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;  in rtw_pci_reset_buf_desc()
446 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;  in rtw_pci_reset_buf_desc()
450 	len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;  in rtw_pci_reset_buf_desc()
451 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;  in rtw_pci_reset_buf_desc()
452 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;  in rtw_pci_reset_buf_desc()
453 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;  in rtw_pci_reset_buf_desc()
457 	len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;  in rtw_pci_reset_buf_desc()
458 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;  in rtw_pci_reset_buf_desc()
459 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;  in rtw_pci_reset_buf_desc()
460 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;  in rtw_pci_reset_buf_desc()
464 	len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;  in rtw_pci_reset_buf_desc()
465 	dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;  in rtw_pci_reset_buf_desc()
466 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;  in rtw_pci_reset_buf_desc()
467 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;  in rtw_pci_reset_buf_desc()
486 				     struct rtw_pci *rtwpci, bool exclude_rx)  in rtw_pci_enable_interrupt()  argument
491 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_enable_interrupt()
493 	rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);  in rtw_pci_enable_interrupt()
494 	rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);  in rtw_pci_enable_interrupt()
496 		rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);  in rtw_pci_enable_interrupt()
498 	rtwpci->irq_enabled = true;  in rtw_pci_enable_interrupt()
500 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_enable_interrupt()
504 				      struct rtw_pci *rtwpci)  in rtw_pci_disable_interrupt()  argument
508 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_disable_interrupt()
510 	if (!rtwpci->irq_enabled)  in rtw_pci_disable_interrupt()
518 	rtwpci->irq_enabled = false;  in rtw_pci_disable_interrupt()
521 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_disable_interrupt()
524 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_reset()  argument
529 	rtwpci->rx_tag = 0;  in rtw_pci_dma_reset()
534 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup()  local
537 	rtw_pci_dma_reset(rtwdev, rtwpci);  in rtw_pci_setup()
542 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_release()  argument
549 		tx_ring = &rtwpci->tx_rings[queue];  in rtw_pci_dma_release()
556 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_start()  local
558 	if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))  in rtw_pci_napi_start()
561 	napi_enable(&rtwpci->napi);  in rtw_pci_napi_start()
566 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_stop()  local
568 	if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))  in rtw_pci_napi_stop()
571 	napi_synchronize(&rtwpci->napi);  in rtw_pci_napi_stop()
572 	napi_disable(&rtwpci->napi);  in rtw_pci_napi_stop()
577 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_start()  local
581 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_start()
582 	rtwpci->running = true;  in rtw_pci_start()
583 	rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_start()
584 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_start()
591 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_stop()  local
592 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_stop()
594 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
595 	rtwpci->running = false;  in rtw_pci_stop()
596 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_stop()
597 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
602 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
603 	rtw_pci_dma_release(rtwdev, rtwpci);  in rtw_pci_stop()
604 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
609 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_enter()  local
617 	lockdep_assert_held(&rtwpci->irq_lock);  in rtw_pci_deep_ps_enter()
628 		tx_ring = &rtwpci->tx_rings[queue];  in rtw_pci_deep_ps_enter()
649 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_leave()  local
651 	lockdep_assert_held(&rtwpci->irq_lock);  in rtw_pci_deep_ps_leave()
659 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps()  local
661 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_deep_ps()
669 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_deep_ps()
703 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,  in rtw_pci_release_rsvd_page()  argument
715 	dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);  in rtw_pci_release_rsvd_page()
723 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_dma_check()  local
734 	if (total_pkt_size != rtwpci->rx_tag)  in rtw_pci_dma_check()
737 	rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;  in rtw_pci_dma_check()
750 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in __pci_flush_queue()  local
751 	struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];  in __pci_flush_queue()
808 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off_queue()  local
812 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_kick_off_queue()
815 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_kick_off_queue()
819 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_kick_off_queue()
824 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off()  local
828 		if (test_and_clear_bit(queue, rtwpci->tx_queued))  in rtw_pci_tx_kick_off()
836 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write_data()  local
848 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_write_data()
853 		rtw_pci_release_rsvd_page(rtwpci, ring);  in rtw_pci_tx_write_data()
861 	dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,  in rtw_pci_tx_write_data()
863 	if (dma_mapping_error(&rtwpci->pdev->dev, dma))  in rtw_pci_tx_write_data()
883 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write_data()
891 	set_bit(queue, rtwpci->tx_queued);  in rtw_pci_tx_write_data()
896 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write_data()
952 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write()  local
961 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_write()
962 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write()
967 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write()
972 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_tx_isr()  argument
985 	ring = &rtwpci->tx_rings[hw_queue];  in rtw_pci_tx_isr()
1005 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,  in rtw_pci_tx_isr()
1049 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_rx_isr()  local
1050 	struct napi_struct *napi = &rtwpci->napi;  in rtw_pci_rx_isr()
1056 				     struct rtw_pci *rtwpci)  in rtw_pci_get_hw_rx_ring_nr()  argument
1062 	ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];  in rtw_pci_get_hw_rx_ring_nr()
1073 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_rx_napi()  argument
1077 	struct napi_struct *napi = &rtwpci->napi;  in rtw_pci_rx_napi()
1078 	struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];  in rtw_pci_rx_napi()
1091 	count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);  in rtw_pci_rx_napi()
1151 				   struct rtw_pci *rtwpci, u32 *irq_status)  in rtw_pci_irq_recognized()  argument
1155 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_irq_recognized()
1163 	irq_status[0] &= rtwpci->irq_mask[0];  in rtw_pci_irq_recognized()
1164 	irq_status[1] &= rtwpci->irq_mask[1];  in rtw_pci_irq_recognized()
1165 	irq_status[3] &= rtwpci->irq_mask[3];  in rtw_pci_irq_recognized()
1171 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_irq_recognized()
1177 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_handler()  local
1187 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_interrupt_handler()
1195 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_threadfn()  local
1199 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_interrupt_threadfn()
1200 	rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);  in rtw_pci_interrupt_threadfn()
1203 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);  in rtw_pci_interrupt_threadfn()
1205 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);  in rtw_pci_interrupt_threadfn()
1207 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);  in rtw_pci_interrupt_threadfn()
1209 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);  in rtw_pci_interrupt_threadfn()
1211 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);  in rtw_pci_interrupt_threadfn()
1213 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);  in rtw_pci_interrupt_threadfn()
1215 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);  in rtw_pci_interrupt_threadfn()
1224 	if (rtwpci->running)  in rtw_pci_interrupt_threadfn()
1225 		rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);  in rtw_pci_interrupt_threadfn()
1226 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_interrupt_threadfn()
1234 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_mapping()  local
1246 	rtwpci->mmap = pci_iomap(pdev, bar_id, len);  in rtw_pci_io_mapping()
1247 	if (!rtwpci->mmap) {  in rtw_pci_io_mapping()
1259 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_unmapping()  local
1261 	if (rtwpci->mmap) {  in rtw_pci_io_unmapping()
1262 		pci_iounmap(pdev, rtwpci->mmap);  in rtw_pci_io_unmapping()
1406 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_ps()  local
1418 	if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))  in rtw_pci_link_ps()
1421 	if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||  in rtw_pci_link_ps()
1422 	    (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))  in rtw_pci_link_ps()
1429 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_cfg()  local
1430 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_link_cfg()
1465 	rtwpci->link_ctrl = link_ctrl;  in rtw_pci_link_cfg()
1485 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_phy_cfg()  local
1487 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_phy_cfg()
1589 	struct rtw_pci *rtwpci;  in rtw_pci_setup_resource()  local
1592 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup_resource()
1593 	rtwpci->pdev = pdev;  in rtw_pci_setup_resource()
1678 	struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);  in rtw_pci_napi_poll()  local
1679 	struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,  in rtw_pci_napi_poll()
1683 	if (rtwpci->rx_no_aspm)  in rtw_pci_napi_poll()
1689 		work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,  in rtw_pci_napi_poll()
1697 		spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_napi_poll()
1698 		if (rtwpci->running)  in rtw_pci_napi_poll()
1699 			rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_napi_poll()
1700 		spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_napi_poll()
1706 		if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))  in rtw_pci_napi_poll()
1709 	if (rtwpci->rx_no_aspm)  in rtw_pci_napi_poll()
1717 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_init()  local
1719 	init_dummy_netdev(&rtwpci->netdev);  in rtw_pci_napi_init()
1720 	netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);  in rtw_pci_napi_init()
1725 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_deinit()  local
1728 	netif_napi_del(&rtwpci->napi);  in rtw_pci_napi_deinit()
1737 	struct rtw_pci *rtwpci;  in rtw_pci_probe()  local
1755 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_probe()
1756 	atomic_set(&rtwpci->link_usage, 1);  in rtw_pci_probe()
1788 		rtwpci->rx_no_aspm = true;  in rtw_pci_probe()
1827 	struct rtw_pci *rtwpci;  in rtw_pci_remove()  local
1833 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_remove()
1836 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_remove()