431 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_is_awake()  local
432 	u32 val = ioread32(ar_pci->mem + PCIE_LOCAL_BASE_ADDRESS +  in ath10k_pci_is_awake()
440 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in __ath10k_pci_wake()  local
442 	lockdep_assert_held(&ar_pci->ps_lock);  in __ath10k_pci_wake()
445 		   ar_pci->ps_wake_refcount, ar_pci->ps_awake);  in __ath10k_pci_wake()
448 		  ar_pci->mem + PCIE_LOCAL_BASE_ADDRESS +  in __ath10k_pci_wake()
454 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in __ath10k_pci_sleep()  local
456 	lockdep_assert_held(&ar_pci->ps_lock);  in __ath10k_pci_sleep()
459 		   ar_pci->ps_wake_refcount, ar_pci->ps_awake);  in __ath10k_pci_sleep()
462 		  ar_pci->mem + PCIE_LOCAL_BASE_ADDRESS +  in __ath10k_pci_sleep()
464 	ar_pci->ps_awake = false;  in __ath10k_pci_sleep()
492 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_force_wake()  local
496 	if (ar_pci->pci_ps)  in ath10k_pci_force_wake()
499 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_force_wake()
501 	if (!ar_pci->ps_awake) {  in ath10k_pci_force_wake()
503 			  ar_pci->mem + PCIE_LOCAL_BASE_ADDRESS +  in ath10k_pci_force_wake()
508 			ar_pci->ps_awake = true;  in ath10k_pci_force_wake()
511 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_force_wake()
518 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_force_sleep()  local
521 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_force_sleep()
524 		  ar_pci->mem + PCIE_LOCAL_BASE_ADDRESS +  in ath10k_pci_force_sleep()
526 	ar_pci->ps_awake = false;  in ath10k_pci_force_sleep()
528 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_force_sleep()
533 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_wake()  local
537 	if (ar_pci->pci_ps == 0)  in ath10k_pci_wake()
540 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_wake()
543 		   ar_pci->ps_wake_refcount, ar_pci->ps_awake);  in ath10k_pci_wake()
548 	if (!ar_pci->ps_awake) {  in ath10k_pci_wake()
553 			ar_pci->ps_awake = true;  in ath10k_pci_wake()
557 		ar_pci->ps_wake_refcount++;  in ath10k_pci_wake()
558 		WARN_ON(ar_pci->ps_wake_refcount == 0);  in ath10k_pci_wake()
561 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_wake()
568 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_sleep()  local
571 	if (ar_pci->pci_ps == 0)  in ath10k_pci_sleep()
574 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_sleep()
577 		   ar_pci->ps_wake_refcount, ar_pci->ps_awake);  in ath10k_pci_sleep()
579 	if (WARN_ON(ar_pci->ps_wake_refcount == 0))  in ath10k_pci_sleep()
582 	ar_pci->ps_wake_refcount--;  in ath10k_pci_sleep()
584 	mod_timer(&ar_pci->ps_timer, jiffies +  in ath10k_pci_sleep()
588 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_sleep()
593 	struct ath10k_pci *ar_pci = from_timer(ar_pci, t, ps_timer);  in ath10k_pci_ps_timer()  local
594 	struct ath10k *ar = ar_pci->ar;  in ath10k_pci_ps_timer()
597 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_ps_timer()
600 		   ar_pci->ps_wake_refcount, ar_pci->ps_awake);  in ath10k_pci_ps_timer()
602 	if (ar_pci->ps_wake_refcount > 0)  in ath10k_pci_ps_timer()
608 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_ps_timer()
613 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_sleep_sync()  local
616 	if (ar_pci->pci_ps == 0) {  in ath10k_pci_sleep_sync()
621 	del_timer_sync(&ar_pci->ps_timer);  in ath10k_pci_sleep_sync()
623 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_sleep_sync()
624 	WARN_ON(ar_pci->ps_wake_refcount > 0);  in ath10k_pci_sleep_sync()
626 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_sleep_sync()
631 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_bus_pci_write32()  local
634 	if (unlikely(offset + sizeof(value) > ar_pci->mem_len)) {  in ath10k_bus_pci_write32()
636 			    offset, offset + sizeof(value), ar_pci->mem_len);  in ath10k_bus_pci_write32()
647 	iowrite32(value, ar_pci->mem + offset);  in ath10k_bus_pci_write32()
653 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_bus_pci_read32()  local
657 	if (unlikely(offset + sizeof(val) > ar_pci->mem_len)) {  in ath10k_bus_pci_read32()
659 			    offset, offset + sizeof(val), ar_pci->mem_len);  in ath10k_bus_pci_read32()
670 	val = ioread32(ar_pci->mem + offset);  in ath10k_bus_pci_read32()
756 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_get_irq_method()  local
758 	if (ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_MSI)  in ath10k_pci_get_irq_method()
806 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_rx_post_pipe()  local
827 			mod_timer(&ar_pci->rx_post_retry, jiffies +  in ath10k_pci_rx_post_pipe()
837 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_rx_post()  local
841 		ath10k_pci_rx_post_pipe(&ar_pci->pipe_info[i]);  in ath10k_pci_rx_post()
846 	struct ath10k_pci *ar_pci = from_timer(ar_pci, t, rx_post_retry);  in ath10k_pci_rx_replenish_retry()  local
847 	struct ath10k *ar = ar_pci->ar;  in ath10k_pci_rx_replenish_retry()
888 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_targ_cpu_to_ce_addr()  local
890 	if (WARN_ON_ONCE(!ar_pci->targ_cpu_to_ce_addr))  in ath10k_pci_targ_cpu_to_ce_addr()
893 	return ar_pci->targ_cpu_to_ce_addr(ar, addr);  in ath10k_pci_targ_cpu_to_ce_addr()
904 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_diag_read_mem()  local
915 	mutex_lock(&ar_pci->ce_diag_mutex);  in ath10k_pci_diag_read_mem()
916 	ce_diag = ar_pci->ce_diag;  in ath10k_pci_diag_read_mem()
1003 	mutex_unlock(&ar_pci->ce_diag_mutex);  in ath10k_pci_diag_read_mem()
1050 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_diag_write_mem()  local
1059 	mutex_lock(&ar_pci->ce_diag_mutex);  in ath10k_pci_diag_write_mem()
1060 	ce_diag = ar_pci->ce_diag;  in ath10k_pci_diag_write_mem()
1158 	mutex_unlock(&ar_pci->ce_diag_mutex);  in ath10k_pci_diag_write_mem()
1195 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_process_rx_cb()  local
1196 	struct ath10k_pci_pipe *pipe_info =  &ar_pci->pipe_info[ce_state->id];  in ath10k_pci_process_rx_cb()
1238 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_process_htt_rx_cb()  local
1239 	struct ath10k_pci_pipe *pipe_info =  &ar_pci->pipe_info[ce_state->id];  in ath10k_pci_process_htt_rx_cb()
1352 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_tx_sg()  local
1354 	struct ath10k_pci_pipe *pci_pipe = &ar_pci->pipe_info[pipe_id];  in ath10k_pci_hif_tx_sg()
1427 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_get_free_queue_number()  local
1431 	return ath10k_ce_num_free_src_entries(ar_pci->pipe_info[pipe].ce_hdl);  in ath10k_pci_hif_get_free_queue_number()
1586 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_dump_memory_sram()  local
1589 	base_addr = ioread32(ar_pci->mem + QCA99X0_PCIE_BAR0_START_REG);  in ath10k_pci_dump_memory_sram()
1593 		iowrite32(base_addr + i, ar_pci->mem + QCA99X0_CPU_MEM_ADDR_REG);  in ath10k_pci_dump_memory_sram()
1594 		*(u32 *)(buf + i) = ioread32(ar_pci->mem + QCA99X0_CPU_MEM_DATA_REG);  in ath10k_pci_dump_memory_sram()
1605 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_dump_memory_reg()  local
1617 		*(u32 *)(buf + i) = ioread32(ar_pci->mem + region->start + i);  in ath10k_pci_dump_memory_reg()
1750 	struct ath10k_pci *ar_pci = container_of(work, struct ath10k_pci,  in ath10k_pci_fw_dump_work()  local
1753 	struct ath10k *ar = ar_pci->ar;  in ath10k_pci_fw_dump_work()
1782 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_fw_crashed_dump()  local
1784 	queue_work(ar->workqueue, &ar_pci->dump_work);  in ath10k_pci_fw_crashed_dump()
1790 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_send_complete_check()  local
1809 		if (resources > (ar_pci->attr[pipe].src_nentries >> 1))  in ath10k_pci_hif_send_complete_check()
1817 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_rx_retry_sync()  local
1819 	del_timer_sync(&ar_pci->rx_post_retry);  in ath10k_pci_rx_retry_sync()
1825 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_map_service_to_pipe()  local
1833 		entry = &ar_pci->serv_to_pipe[i];  in ath10k_pci_hif_map_service_to_pipe()
1943 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_irq_sync()  local
1945 	synchronize_irq(ar_pci->pdev->irq);  in ath10k_pci_irq_sync()
1957 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_start()  local
1966 	pcie_capability_write_word(ar_pci->pdev, PCI_EXP_LNKCTL,  in ath10k_pci_hif_start()
1967 				   ar_pci->link_ctl);  in ath10k_pci_hif_start()
2043 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_buffer_cleanup()  local
2049 		pipe_info = &ar_pci->pipe_info[pipe_num];  in ath10k_pci_buffer_cleanup()
2071 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_stop()  local
2081 	cancel_work_sync(&ar_pci->dump_work);  in ath10k_pci_hif_stop()
2098 	spin_lock_irqsave(&ar_pci->ps_lock, flags);  in ath10k_pci_hif_stop()
2099 	WARN_ON(ar_pci->ps_wake_refcount > 0);  in ath10k_pci_hif_stop()
2100 	spin_unlock_irqrestore(&ar_pci->ps_lock, flags);  in ath10k_pci_hif_stop()
2107 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_exchange_bmi_msg()  local
2108 	struct ath10k_pci_pipe *pci_tx = &ar_pci->pipe_info[BMI_CE_NUM_TO_TARG];  in ath10k_pci_hif_exchange_bmi_msg()
2109 	struct ath10k_pci_pipe *pci_rx = &ar_pci->pipe_info[BMI_CE_NUM_TO_HOST];  in ath10k_pci_hif_exchange_bmi_msg()
2280 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_get_num_banks()  local
2282 	switch (ar_pci->pdev->device) {  in ath10k_pci_get_num_banks()
2323 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_init_config()  local
2369 					ar_pci->pipe_config,  in ath10k_pci_init_config()
2394 					ar_pci->serv_to_pipe,  in ath10k_pci_init_config()
2467 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_override_ce_config()  local
2474 	attr = &ar_pci->attr[5];  in ath10k_pci_override_ce_config()
2479 	config = &ar_pci->pipe_config[5];  in ath10k_pci_override_ce_config()
2484 	ar_pci->serv_to_pipe[15].pipenum = __cpu_to_le32(1);  in ath10k_pci_override_ce_config()
2489 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_alloc_pipes()  local
2495 		pipe = &ar_pci->pipe_info[i];  in ath10k_pci_alloc_pipes()
2500 		ret = ath10k_ce_alloc_pipe(ar, i, &ar_pci->attr[i]);  in ath10k_pci_alloc_pipes()
2509 			ar_pci->ce_diag = pipe->ce_hdl;  in ath10k_pci_alloc_pipes()
2513 		pipe->buf_sz = (size_t)(ar_pci->attr[i].src_sz_max);  in ath10k_pci_alloc_pipes()
2529 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_init_pipes()  local
2533 		ret = ath10k_ce_init_pipe(ar, i, &ar_pci->attr[i]);  in ath10k_pci_init_pipes()
2666 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_safe_chip_reset()  local
2668 	if (!ar_pci->pci_soft_reset)  in ath10k_pci_safe_chip_reset()
2671 	return ar_pci->pci_soft_reset(ar);  in ath10k_pci_safe_chip_reset()
2806 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_chip_reset()  local
2808 	if (WARN_ON(!ar_pci->pci_hard_reset))  in ath10k_pci_chip_reset()
2811 	return ar_pci->pci_hard_reset(ar);  in ath10k_pci_chip_reset()
2817 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_hif_power_up()  local
2822 	pcie_capability_read_word(ar_pci->pdev, PCI_EXP_LNKCTL,  in ath10k_pci_hif_power_up()
2823 				  &ar_pci->link_ctl);  in ath10k_pci_hif_power_up()
2824 	pcie_capability_write_word(ar_pci->pdev, PCI_EXP_LNKCTL,  in ath10k_pci_hif_power_up()
2825 				   ar_pci->link_ctl & ~PCI_EXP_LNKCTL_ASPMC);  in ath10k_pci_hif_power_up()
2911 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_resume()  local
2912 	struct pci_dev *pdev = ar_pci->pdev;  in ath10k_pci_resume()
3100 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_interrupt_handler()  local
3112 	if ((ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY) &&  in ath10k_pci_interrupt_handler()
3163 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_request_irq_msi()  local
3166 	ret = request_irq(ar_pci->pdev->irq,  in ath10k_pci_request_irq_msi()
3171 			    ar_pci->pdev->irq, ret);  in ath10k_pci_request_irq_msi()
3180 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_request_irq_legacy()  local
3183 	ret = request_irq(ar_pci->pdev->irq,  in ath10k_pci_request_irq_legacy()
3188 			    ar_pci->pdev->irq, ret);  in ath10k_pci_request_irq_legacy()
3197 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_request_irq()  local
3199 	switch (ar_pci->oper_irq_mode) {  in ath10k_pci_request_irq()
3211 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_free_irq()  local
3213 	free_irq(ar_pci->pdev->irq, ar);  in ath10k_pci_free_irq()
3224 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_init_irq()  local
3235 		ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_MSI;  in ath10k_pci_init_irq()
3236 		ret = pci_enable_msi(ar_pci->pdev);  in ath10k_pci_init_irq()
3252 	ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_LEGACY;  in ath10k_pci_init_irq()
3268 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_deinit_irq()  local
3270 	switch (ar_pci->oper_irq_mode) {  in ath10k_pci_deinit_irq()
3275 		pci_disable_msi(ar_pci->pdev);  in ath10k_pci_deinit_irq()
3284 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_wait_for_target_init()  local
3309 		if (ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY)  in ath10k_pci_wait_for_target_init()
3376 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_claim()  local
3377 	struct pci_dev *pdev = ar_pci->pdev;  in ath10k_pci_claim()
3405 	ar_pci->mem_len = pci_resource_len(pdev, BAR_NUM);  in ath10k_pci_claim()
3406 	ar_pci->mem = pci_iomap(pdev, BAR_NUM, 0);  in ath10k_pci_claim()
3407 	if (!ar_pci->mem) {  in ath10k_pci_claim()
3413 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot pci_mem 0x%pK\n", ar_pci->mem);  in ath10k_pci_claim()
3430 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_release()  local
3431 	struct pci_dev *pdev = ar_pci->pdev;  in ath10k_pci_release()
3433 	pci_iounmap(pdev, ar_pci->mem);  in ath10k_pci_release()
3458 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_setup_resource()  local
3463 	spin_lock_init(&ar_pci->ps_lock);  in ath10k_pci_setup_resource()
3464 	mutex_init(&ar_pci->ce_diag_mutex);  in ath10k_pci_setup_resource()
3466 	INIT_WORK(&ar_pci->dump_work, ath10k_pci_fw_dump_work);  in ath10k_pci_setup_resource()
3468 	timer_setup(&ar_pci->rx_post_retry, ath10k_pci_rx_replenish_retry, 0);  in ath10k_pci_setup_resource()
3470 	ar_pci->attr = kmemdup(pci_host_ce_config_wlan,  in ath10k_pci_setup_resource()
3473 	if (!ar_pci->attr)  in ath10k_pci_setup_resource()
3476 	ar_pci->pipe_config = kmemdup(pci_target_ce_config_wlan,  in ath10k_pci_setup_resource()
3479 	if (!ar_pci->pipe_config) {  in ath10k_pci_setup_resource()
3484 	ar_pci->serv_to_pipe = kmemdup(pci_target_service_to_ce_map_wlan,  in ath10k_pci_setup_resource()
3487 	if (!ar_pci->serv_to_pipe) {  in ath10k_pci_setup_resource()
3505 	kfree(ar_pci->serv_to_pipe);  in ath10k_pci_setup_resource()
3507 	kfree(ar_pci->pipe_config);  in ath10k_pci_setup_resource()
3509 	kfree(ar_pci->attr);  in ath10k_pci_setup_resource()
3515 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_release_resource()  local
3521 	kfree(ar_pci->attr);  in ath10k_pci_release_resource()
3522 	kfree(ar_pci->pipe_config);  in ath10k_pci_release_resource()
3523 	kfree(ar_pci->serv_to_pipe);  in ath10k_pci_release_resource()
3537 	struct ath10k_pci *ar_pci;  in ath10k_pci_probe()  local
3603 	ar = ath10k_core_create(sizeof(*ar_pci), &pdev->dev, ATH10K_BUS_PCI,  in ath10k_pci_probe()
3614 	ar_pci = ath10k_pci_priv(ar);  in ath10k_pci_probe()
3615 	ar_pci->pdev = pdev;  in ath10k_pci_probe()
3616 	ar_pci->dev = &pdev->dev;  in ath10k_pci_probe()
3617 	ar_pci->ar = ar;  in ath10k_pci_probe()
3619 	ar_pci->pci_ps = pci_ps;  in ath10k_pci_probe()
3620 	ar_pci->ce.bus_ops = &ath10k_pci_bus_ops;  in ath10k_pci_probe()
3621 	ar_pci->pci_soft_reset = pci_soft_reset;  in ath10k_pci_probe()
3622 	ar_pci->pci_hard_reset = pci_hard_reset;  in ath10k_pci_probe()
3623 	ar_pci->targ_cpu_to_ce_addr = targ_cpu_to_ce_addr;  in ath10k_pci_probe()
3624 	ar->ce_priv = &ar_pci->ce;  in ath10k_pci_probe()
3631 	timer_setup(&ar_pci->ps_timer, ath10k_pci_ps_timer, 0);  in ath10k_pci_probe()
3661 		    ath10k_pci_get_irq_method(ar), ar_pci->oper_irq_mode,  in ath10k_pci_probe()