91 static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,  in ipa_endpoint_data_valid_one()  argument
96 	struct device *dev = &ipa->pdev->dev;  in ipa_endpoint_data_valid_one()
165 		reg = ipa_reg(ipa, ENDP_INIT_AGGR);  in ipa_endpoint_data_valid_one()
179 	if (ipa->version >= IPA_VERSION_4_5) {  in ipa_endpoint_data_valid_one()
246 static bool ipa_endpoint_data_valid(struct ipa *ipa, u32 count,  in ipa_endpoint_data_valid()  argument
250 	struct device *dev = &ipa->pdev->dev;  in ipa_endpoint_data_valid()
278 		if (!ipa_endpoint_data_valid_one(ipa, count, data, dp))  in ipa_endpoint_data_valid()
288 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_trans_alloc()
304 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_ctrl()  local
313 		WARN_ON(ipa->version >= IPA_VERSION_4_2);  in ipa_endpoint_init_ctrl()
315 		WARN_ON(ipa->version >= IPA_VERSION_4_0);  in ipa_endpoint_init_ctrl()
317 	reg = ipa_reg(ipa, ENDP_INIT_CTRL);  in ipa_endpoint_init_ctrl()
319 	val = ioread32(ipa->reg_virt + offset);  in ipa_endpoint_init_ctrl()
329 		iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_ctrl()
340 	WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2);  in ipa_endpoint_program_delay()
349 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_aggr_active()  local
353 	WARN_ON(!(mask & ipa->available));  in ipa_endpoint_aggr_active()
355 	reg = ipa_reg(ipa, STATE_AGGR_ACTIVE);  in ipa_endpoint_aggr_active()
356 	val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));  in ipa_endpoint_aggr_active()
364 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_force_close()  local
367 	WARN_ON(!(mask & ipa->available));  in ipa_endpoint_force_close()
369 	reg = ipa_reg(ipa, AGGR_FORCE_CLOSE);  in ipa_endpoint_force_close()
370 	iowrite32(mask, ipa->reg_virt + ipa_reg_offset(reg));  in ipa_endpoint_force_close()
384 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_suspend_aggr()  local
396 	ipa_interrupt_simulate_suspend(ipa->interrupt);  in ipa_endpoint_suspend_aggr()
405 	if (endpoint->ipa->version >= IPA_VERSION_4_0)  in ipa_endpoint_program_suspend()
427 void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable)  in ipa_endpoint_modem_pause_all()  argument
432 		struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id];  in ipa_endpoint_modem_pause_all()
439 		else if (ipa->version < IPA_VERSION_4_2)  in ipa_endpoint_modem_pause_all()
442 			gsi_modem_channel_flow_control(&ipa->gsi,  in ipa_endpoint_modem_pause_all()
449 int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)  in ipa_endpoint_modem_exception_reset_all()  argument
451 	u32 initialized = ipa->initialized;  in ipa_endpoint_modem_exception_reset_all()
458 	count = ipa->modem_tx_count + ipa_cmd_pipeline_clear_count();  in ipa_endpoint_modem_exception_reset_all()
459 	trans = ipa_cmd_trans_alloc(ipa, count);  in ipa_endpoint_modem_exception_reset_all()
461 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_modem_exception_reset_all()
475 		endpoint = &ipa->endpoint[endpoint_id];  in ipa_endpoint_modem_exception_reset_all()
479 		reg = ipa_reg(ipa, ENDP_STATUS);  in ipa_endpoint_modem_exception_reset_all()
493 	ipa_cmd_pipeline_clear_wait(ipa);  in ipa_endpoint_modem_exception_reset_all()
501 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_cfg()  local
506 	reg = ipa_reg(ipa, ENDP_INIT_CFG);  in ipa_endpoint_init_cfg()
509 		enum ipa_version version = ipa->version;  in ipa_endpoint_init_cfg()
532 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_cfg()
538 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_nat()  local
545 	reg = ipa_reg(ipa, ENDP_INIT_NAT);  in ipa_endpoint_init_nat()
548 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_nat()
640 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hdr()  local
644 	reg = ipa_reg(ipa, ENDP_INIT_HDR);  in ipa_endpoint_init_hdr()
646 		enum ipa_version version = ipa->version;  in ipa_endpoint_init_hdr()
678 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_hdr()
685 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hdr_ext()  local
689 	reg = ipa_reg(ipa, ENDP_INIT_HDR_EXT);  in ipa_endpoint_init_hdr_ext()
716 	if (ipa->version >= IPA_VERSION_4_5) {  in ipa_endpoint_init_hdr_ext()
730 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_hdr_ext()
736 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hdr_metadata_mask()  local
744 	reg = ipa_reg(ipa,  ENDP_INIT_HDR_METADATA_MASK);  in ipa_endpoint_init_hdr_metadata_mask()
751 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_hdr_metadata_mask()
756 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_mode()  local
764 	reg = ipa_reg(ipa, ENDP_INIT_MODE);  in ipa_endpoint_init_mode()
767 		u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id;  in ipa_endpoint_init_mode()
777 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_mode()
805 static u32 aggr_time_limit_encode(struct ipa *ipa, const struct ipa_reg *reg,  in aggr_time_limit_encode()  argument
815 	if (ipa->version >= IPA_VERSION_4_5) {  in aggr_time_limit_encode()
843 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_aggr()  local
847 	reg = ipa_reg(ipa, ENDP_INIT_AGGR);  in ipa_endpoint_init_aggr()
864 			val |= aggr_time_limit_encode(ipa, reg, limit);  in ipa_endpoint_init_aggr()
882 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_aggr()
893 static u32 hol_block_timer_encode(struct ipa *ipa, const struct ipa_reg *reg,  in hol_block_timer_encode()  argument
906 	if (ipa->version >= IPA_VERSION_4_5) {  in hol_block_timer_encode()
925 	rate = ipa_core_clock_rate(ipa);  in hol_block_timer_encode()
932 	if (ipa->version < IPA_VERSION_4_2)  in hol_block_timer_encode()
965 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hol_block_timer()  local
970 	reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_TIMER);  in ipa_endpoint_init_hol_block_timer()
971 	val = hol_block_timer_encode(ipa, reg, microseconds);  in ipa_endpoint_init_hol_block_timer()
973 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_hol_block_timer()
980 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hol_block_en()  local
985 	reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_EN);  in ipa_endpoint_init_hol_block_en()
989 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_hol_block_en()
992 	if (enable && ipa->version >= IPA_VERSION_4_5)  in ipa_endpoint_init_hol_block_en()
993 		iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_hol_block_en()
1009 void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)  in ipa_endpoint_modem_hol_block_clear_all()  argument
1014 		struct ipa_endpoint *endpoint = &ipa->endpoint[i];  in ipa_endpoint_modem_hol_block_clear_all()
1027 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_deaggr()  local
1034 	reg = ipa_reg(ipa, ENDP_INIT_DEAGGR);  in ipa_endpoint_init_deaggr()
1040 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_deaggr()
1047 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_rsrc_grp()  local
1051 	reg = ipa_reg(ipa, ENDP_INIT_RSRC_GRP);  in ipa_endpoint_init_rsrc_grp()
1054 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_rsrc_grp()
1060 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_seq()  local
1067 	reg = ipa_reg(ipa, ENDP_INIT_SEQ);  in ipa_endpoint_init_seq()
1073 	if (ipa->version < IPA_VERSION_4_5)  in ipa_endpoint_init_seq()
1077 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_init_seq()
1126 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_status()  local
1130 	reg = ipa_reg(ipa, ENDP_STATUS);  in ipa_endpoint_status()
1138 			status_endpoint_id = ipa->name_map[name]->endpoint_id;  in ipa_endpoint_status()
1149 	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));  in ipa_endpoint_status()
1227 	if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id))  in ipa_endpoint_replenish()
1237 	if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id))  in ipa_endpoint_replenish_enable()
1337 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_status_tag()  local
1350 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];  in ipa_endpoint_status_tag()
1352 		complete(&ipa->completion);  in ipa_endpoint_status_tag()
1354 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_status_tag()
1396 			dev_err(&endpoint->ipa->pdev->dev,  in ipa_endpoint_status_parse()
1470 		struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_trans_release()  local
1473 		if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) {  in ipa_endpoint_trans_release()
1487 void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id)  in ipa_endpoint_default_route_set()  argument
1492 	reg = ipa_reg(ipa, ROUTE);  in ipa_endpoint_default_route_set()
1500 	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));  in ipa_endpoint_default_route_set()
1503 void ipa_endpoint_default_route_clear(struct ipa *ipa)  in ipa_endpoint_default_route_clear()  argument
1505 	ipa_endpoint_default_route_set(ipa, 0);  in ipa_endpoint_default_route_clear()
1520 	struct device *dev = &endpoint->ipa->pdev->dev;  in ipa_endpoint_reset_rx_aggr()
1521 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_reset_rx_aggr()  local
1522 	struct gsi *gsi = &ipa->gsi;  in ipa_endpoint_reset_rx_aggr()
1607 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_reset()  local
1615 	special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa &&  in ipa_endpoint_reset()
1620 		gsi_channel_reset(&ipa->gsi, channel_id, true);  in ipa_endpoint_reset()
1623 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_reset()
1637 		if (endpoint->ipa->version < IPA_VERSION_4_2)  in ipa_endpoint_program()
1664 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_enable_one()  local
1665 	struct gsi *gsi = &ipa->gsi;  in ipa_endpoint_enable_one()
1670 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_enable_one()
1678 		ipa_interrupt_suspend_enable(ipa->interrupt,  in ipa_endpoint_enable_one()
1683 	ipa->enabled |= BIT(endpoint->endpoint_id);  in ipa_endpoint_enable_one()
1691 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_disable_one()  local
1692 	struct gsi *gsi = &ipa->gsi;  in ipa_endpoint_disable_one()
1695 	if (!(ipa->enabled & mask))  in ipa_endpoint_disable_one()
1698 	ipa->enabled ^= mask;  in ipa_endpoint_disable_one()
1702 		ipa_interrupt_suspend_disable(ipa->interrupt,  in ipa_endpoint_disable_one()
1709 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_disable_one()
1716 	struct device *dev = &endpoint->ipa->pdev->dev;  in ipa_endpoint_suspend_one()
1717 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_suspend_one()
1720 	if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id)))  in ipa_endpoint_suspend_one()
1736 	struct device *dev = &endpoint->ipa->pdev->dev;  in ipa_endpoint_resume_one()
1737 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_resume_one()
1740 	if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id)))  in ipa_endpoint_resume_one()
1754 void ipa_endpoint_suspend(struct ipa *ipa)  in ipa_endpoint_suspend()  argument
1756 	if (!ipa->setup_complete)  in ipa_endpoint_suspend()
1759 	if (ipa->modem_netdev)  in ipa_endpoint_suspend()
1760 		ipa_modem_suspend(ipa->modem_netdev);  in ipa_endpoint_suspend()
1762 	ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);  in ipa_endpoint_suspend()
1763 	ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);  in ipa_endpoint_suspend()
1766 void ipa_endpoint_resume(struct ipa *ipa)  in ipa_endpoint_resume()  argument
1768 	if (!ipa->setup_complete)  in ipa_endpoint_resume()
1771 	ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);  in ipa_endpoint_resume()
1772 	ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);  in ipa_endpoint_resume()
1774 	if (ipa->modem_netdev)  in ipa_endpoint_resume()
1775 		ipa_modem_resume(ipa->modem_netdev);  in ipa_endpoint_resume()
1780 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_setup_one()
1800 	endpoint->ipa->set_up |= BIT(endpoint->endpoint_id);  in ipa_endpoint_setup_one()
1805 	endpoint->ipa->set_up &= ~BIT(endpoint->endpoint_id);  in ipa_endpoint_teardown_one()
1813 void ipa_endpoint_setup(struct ipa *ipa)  in ipa_endpoint_setup()  argument
1815 	u32 initialized = ipa->initialized;  in ipa_endpoint_setup()
1817 	ipa->set_up = 0;  in ipa_endpoint_setup()
1823 		ipa_endpoint_setup_one(&ipa->endpoint[endpoint_id]);  in ipa_endpoint_setup()
1827 void ipa_endpoint_teardown(struct ipa *ipa)  in ipa_endpoint_teardown()  argument
1829 	u32 set_up = ipa->set_up;  in ipa_endpoint_teardown()
1836 		ipa_endpoint_teardown_one(&ipa->endpoint[endpoint_id]);  in ipa_endpoint_teardown()
1838 	ipa->set_up = 0;  in ipa_endpoint_teardown()
1841 int ipa_endpoint_config(struct ipa *ipa)  in ipa_endpoint_config()  argument
1843 	struct device *dev = &ipa->pdev->dev;  in ipa_endpoint_config()
1863 	if (ipa->version < IPA_VERSION_3_5) {  in ipa_endpoint_config()
1864 		ipa->available = ~0;  in ipa_endpoint_config()
1871 	reg = ipa_reg(ipa, FLAVOR_0);  in ipa_endpoint_config()
1872 	val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));  in ipa_endpoint_config()
1888 	ipa->available = rx_mask | tx_mask;  in ipa_endpoint_config()
1891 	if (ipa->initialized & ~ipa->available) {  in ipa_endpoint_config()
1893 			ipa->initialized & ~ipa->available);  in ipa_endpoint_config()
1897 	initialized = ipa->initialized;  in ipa_endpoint_config()
1905 		endpoint = &ipa->endpoint[endpoint_id];  in ipa_endpoint_config()
1916 void ipa_endpoint_deconfig(struct ipa *ipa)  in ipa_endpoint_deconfig()  argument
1918 	ipa->available = 0;	/* Nothing more to do */  in ipa_endpoint_deconfig()
1921 static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name,  in ipa_endpoint_init_one()  argument
1926 	endpoint = &ipa->endpoint[data->endpoint_id];  in ipa_endpoint_init_one()
1929 		ipa->channel_map[data->channel_id] = endpoint;  in ipa_endpoint_init_one()
1930 	ipa->name_map[name] = endpoint;  in ipa_endpoint_init_one()
1932 	endpoint->ipa = ipa;  in ipa_endpoint_init_one()
1939 	ipa->initialized |= BIT(endpoint->endpoint_id);  in ipa_endpoint_init_one()
1944 	endpoint->ipa->initialized &= ~BIT(endpoint->endpoint_id);  in ipa_endpoint_exit_one()
1949 void ipa_endpoint_exit(struct ipa *ipa)  in ipa_endpoint_exit()  argument
1951 	u32 initialized = ipa->initialized;  in ipa_endpoint_exit()
1958 		ipa_endpoint_exit_one(&ipa->endpoint[endpoint_id]);  in ipa_endpoint_exit()
1960 	memset(ipa->name_map, 0, sizeof(ipa->name_map));  in ipa_endpoint_exit()
1961 	memset(ipa->channel_map, 0, sizeof(ipa->channel_map));  in ipa_endpoint_exit()
1965 u32 ipa_endpoint_init(struct ipa *ipa, u32 count,  in ipa_endpoint_init()  argument
1973 	if (!ipa_endpoint_data_valid(ipa, count, data))  in ipa_endpoint_init()
1976 	ipa->initialized = 0;  in ipa_endpoint_init()
1983 		ipa_endpoint_init_one(ipa, name, data);  in ipa_endpoint_init()
1988 			ipa->modem_tx_count++;  in ipa_endpoint_init()
1991 	if (!ipa_filter_map_valid(ipa, filter_map))  in ipa_endpoint_init()
1997 	ipa_endpoint_exit(ipa);  in ipa_endpoint_init()