78 static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,  in ipa_endpoint_data_valid_one()  argument
83 	struct device *dev = &ipa->pdev->dev;  in ipa_endpoint_data_valid_one()
167 static bool ipa_endpoint_data_valid(struct ipa *ipa, u32 count,  in ipa_endpoint_data_valid()  argument
171 	struct device *dev = &ipa->pdev->dev;  in ipa_endpoint_data_valid()
193 	limit = aggr_byte_limit_max(ipa->version) * SZ_1K;  in ipa_endpoint_data_valid()
220 		if (!ipa_endpoint_data_valid_one(ipa, count, data, dp))  in ipa_endpoint_data_valid()
230 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_trans_alloc()
246 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_ctrl()  local
255 		WARN_ON(ipa->version == IPA_VERSION_4_2);  in ipa_endpoint_init_ctrl()
257 		WARN_ON(ipa->version >= IPA_VERSION_4_0);  in ipa_endpoint_init_ctrl()
261 	val = ioread32(ipa->reg_virt + offset);  in ipa_endpoint_init_ctrl()
267 		iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_ctrl()
280 	if (endpoint->ipa->version != IPA_VERSION_4_2)  in ipa_endpoint_program_delay()
287 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_aggr_active()  local
291 	WARN_ON(!(mask & ipa->available));  in ipa_endpoint_aggr_active()
293 	offset = ipa_reg_state_aggr_active_offset(ipa->version);  in ipa_endpoint_aggr_active()
294 	val = ioread32(ipa->reg_virt + offset);  in ipa_endpoint_aggr_active()
302 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_force_close()  local
304 	WARN_ON(!(mask & ipa->available));  in ipa_endpoint_force_close()
306 	iowrite32(mask, ipa->reg_virt + IPA_REG_AGGR_FORCE_CLOSE_OFFSET);  in ipa_endpoint_force_close()
320 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_suspend_aggr()  local
332 	ipa_interrupt_simulate_suspend(ipa->interrupt);  in ipa_endpoint_suspend_aggr()
341 	if (endpoint->ipa->version >= IPA_VERSION_4_0)  in ipa_endpoint_program_suspend()
359 void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable)  in ipa_endpoint_modem_pause_all()  argument
364 	if (ipa->version == IPA_VERSION_4_2)  in ipa_endpoint_modem_pause_all()
368 		struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id];  in ipa_endpoint_modem_pause_all()
382 int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)  in ipa_endpoint_modem_exception_reset_all()  argument
384 	u32 initialized = ipa->initialized;  in ipa_endpoint_modem_exception_reset_all()
394 	trans = ipa_cmd_trans_alloc(ipa, count);  in ipa_endpoint_modem_exception_reset_all()
396 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_modem_exception_reset_all()
409 		endpoint = &ipa->endpoint[endpoint_id];  in ipa_endpoint_modem_exception_reset_all()
427 	ipa_cmd_pipeline_clear_wait(ipa);  in ipa_endpoint_modem_exception_reset_all()
440 		enum ipa_version version = endpoint->ipa->version;  in ipa_endpoint_init_cfg()
465 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_cfg()
479 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_nat()
527 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hdr()  local
531 		enum ipa_version version = ipa->version;  in ipa_endpoint_init_hdr()
563 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_hdr()
570 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hdr_ext()  local
595 	if (ipa->version >= IPA_VERSION_4_5) {  in ipa_endpoint_init_hdr_ext()
607 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_hdr_ext()
625 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_hdr_metadata_mask()
640 		dma_endpoint_id = endpoint->ipa->name_map[name]->endpoint_id;  in ipa_endpoint_init_mode()
649 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_mode()
720 	enum ipa_version version = endpoint->ipa->version;  in ipa_endpoint_init_aggr()
756 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_aggr()
763 static u32 hol_block_timer_qtime_val(struct ipa *ipa, u32 microseconds)  in hol_block_timer_qtime_val()  argument
797 static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds)  in hol_block_timer_val()  argument
809 	if (ipa->version >= IPA_VERSION_4_5)  in hol_block_timer_val()
810 		return hol_block_timer_qtime_val(ipa, microseconds);  in hol_block_timer_val()
813 	rate = ipa_core_clock_rate(ipa);  in hol_block_timer_val()
819 	if (ipa->version < IPA_VERSION_4_2)  in hol_block_timer_val()
852 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_hol_block_timer()  local
857 	val = hol_block_timer_val(ipa, microseconds);  in ipa_endpoint_init_hol_block_timer()
858 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_hol_block_timer()
870 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_hol_block_enable()
873 void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)  in ipa_endpoint_modem_hol_block_clear_all()  argument
878 		struct ipa_endpoint *endpoint = &ipa->endpoint[i];  in ipa_endpoint_modem_hol_block_clear_all()
901 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_deaggr()
907 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_init_rsrc_grp()  local
910 	val = rsrc_grp_encoded(ipa->version, endpoint->data->resource_group);  in ipa_endpoint_init_rsrc_grp()
911 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_init_rsrc_grp()
929 	iowrite32(val, endpoint->ipa->reg_virt + offset);  in ipa_endpoint_init_seq()
978 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_status()  local
991 			status_endpoint_id = ipa->name_map[name]->endpoint_id;  in ipa_endpoint_status()
1002 	iowrite32(val, ipa->reg_virt + offset);  in ipa_endpoint_status()
1093 	gsi = &endpoint->ipa->gsi;  in ipa_endpoint_replenish()
1101 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_replenish_enable()
1214 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_status_tag()  local
1227 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];  in ipa_endpoint_status_tag()
1229 		complete(&ipa->completion);  in ipa_endpoint_status_tag()
1231 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_status_tag()
1272 			dev_err(&endpoint->ipa->pdev->dev,  in ipa_endpoint_status_parse()
1359 		struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_trans_release()  local
1362 		if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) {  in ipa_endpoint_trans_release()
1376 void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id)  in ipa_endpoint_default_route_set()  argument
1387 	iowrite32(val, ipa->reg_virt + IPA_REG_ROUTE_OFFSET);  in ipa_endpoint_default_route_set()
1390 void ipa_endpoint_default_route_clear(struct ipa *ipa)  in ipa_endpoint_default_route_clear()  argument
1392 	ipa_endpoint_default_route_set(ipa, 0);  in ipa_endpoint_default_route_clear()
1407 	struct device *dev = &endpoint->ipa->pdev->dev;  in ipa_endpoint_reset_rx_aggr()
1408 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_reset_rx_aggr()  local
1409 	struct gsi *gsi = &ipa->gsi;  in ipa_endpoint_reset_rx_aggr()
1494 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_reset()  local
1502 	special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa &&  in ipa_endpoint_reset()
1507 		gsi_channel_reset(&ipa->gsi, channel_id, true);  in ipa_endpoint_reset()
1510 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_reset()
1536 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_enable_one()  local
1537 	struct gsi *gsi = &ipa->gsi;  in ipa_endpoint_enable_one()
1542 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_enable_one()
1550 		ipa_interrupt_suspend_enable(ipa->interrupt,  in ipa_endpoint_enable_one()
1555 	ipa->enabled |= BIT(endpoint->endpoint_id);  in ipa_endpoint_enable_one()
1563 	struct ipa *ipa = endpoint->ipa;  in ipa_endpoint_disable_one()  local
1564 	struct gsi *gsi = &ipa->gsi;  in ipa_endpoint_disable_one()
1567 	if (!(ipa->enabled & mask))  in ipa_endpoint_disable_one()
1570 	ipa->enabled ^= mask;  in ipa_endpoint_disable_one()
1574 		ipa_interrupt_suspend_disable(ipa->interrupt,  in ipa_endpoint_disable_one()
1581 		dev_err(&ipa->pdev->dev,  in ipa_endpoint_disable_one()
1588 	struct device *dev = &endpoint->ipa->pdev->dev;  in ipa_endpoint_suspend_one()
1589 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_suspend_one()
1592 	if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id)))  in ipa_endpoint_suspend_one()
1608 	struct device *dev = &endpoint->ipa->pdev->dev;  in ipa_endpoint_resume_one()
1609 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_resume_one()
1612 	if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id)))  in ipa_endpoint_resume_one()
1626 void ipa_endpoint_suspend(struct ipa *ipa)  in ipa_endpoint_suspend()  argument
1628 	if (!ipa->setup_complete)  in ipa_endpoint_suspend()
1631 	if (ipa->modem_netdev)  in ipa_endpoint_suspend()
1632 		ipa_modem_suspend(ipa->modem_netdev);  in ipa_endpoint_suspend()
1634 	ipa_cmd_pipeline_clear(ipa);  in ipa_endpoint_suspend()
1636 	ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);  in ipa_endpoint_suspend()
1637 	ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);  in ipa_endpoint_suspend()
1640 void ipa_endpoint_resume(struct ipa *ipa)  in ipa_endpoint_resume()  argument
1642 	if (!ipa->setup_complete)  in ipa_endpoint_resume()
1645 	ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);  in ipa_endpoint_resume()
1646 	ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);  in ipa_endpoint_resume()
1648 	if (ipa->modem_netdev)  in ipa_endpoint_resume()
1649 		ipa_modem_resume(ipa->modem_netdev);  in ipa_endpoint_resume()
1654 	struct gsi *gsi = &endpoint->ipa->gsi;  in ipa_endpoint_setup_one()
1676 	endpoint->ipa->set_up |= BIT(endpoint->endpoint_id);  in ipa_endpoint_setup_one()
1681 	endpoint->ipa->set_up &= ~BIT(endpoint->endpoint_id);  in ipa_endpoint_teardown_one()
1689 void ipa_endpoint_setup(struct ipa *ipa)  in ipa_endpoint_setup()  argument
1691 	u32 initialized = ipa->initialized;  in ipa_endpoint_setup()
1693 	ipa->set_up = 0;  in ipa_endpoint_setup()
1699 		ipa_endpoint_setup_one(&ipa->endpoint[endpoint_id]);  in ipa_endpoint_setup()
1703 void ipa_endpoint_teardown(struct ipa *ipa)  in ipa_endpoint_teardown()  argument
1705 	u32 set_up = ipa->set_up;  in ipa_endpoint_teardown()
1712 		ipa_endpoint_teardown_one(&ipa->endpoint[endpoint_id]);  in ipa_endpoint_teardown()
1714 	ipa->set_up = 0;  in ipa_endpoint_teardown()
1717 int ipa_endpoint_config(struct ipa *ipa)  in ipa_endpoint_config()  argument
1719 	struct device *dev = &ipa->pdev->dev;  in ipa_endpoint_config()
1738 	if (ipa->version < IPA_VERSION_3_5) {  in ipa_endpoint_config()
1739 		ipa->available = ~0;  in ipa_endpoint_config()
1746 	val = ioread32(ipa->reg_virt + IPA_REG_FLAVOR_0_OFFSET);  in ipa_endpoint_config()
1762 	ipa->available = rx_mask | tx_mask;  in ipa_endpoint_config()
1765 	if (ipa->initialized & ~ipa->available) {  in ipa_endpoint_config()
1767 			ipa->initialized & ~ipa->available);  in ipa_endpoint_config()
1771 	initialized = ipa->initialized;  in ipa_endpoint_config()
1779 		endpoint = &ipa->endpoint[endpoint_id];  in ipa_endpoint_config()
1790 void ipa_endpoint_deconfig(struct ipa *ipa)  in ipa_endpoint_deconfig()  argument
1792 	ipa->available = 0;	/* Nothing more to do */  in ipa_endpoint_deconfig()
1795 static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name,  in ipa_endpoint_init_one()  argument
1800 	endpoint = &ipa->endpoint[data->endpoint_id];  in ipa_endpoint_init_one()
1803 		ipa->channel_map[data->channel_id] = endpoint;  in ipa_endpoint_init_one()
1804 	ipa->name_map[name] = endpoint;  in ipa_endpoint_init_one()
1806 	endpoint->ipa = ipa;  in ipa_endpoint_init_one()
1813 	ipa->initialized |= BIT(endpoint->endpoint_id);  in ipa_endpoint_init_one()
1818 	endpoint->ipa->initialized &= ~BIT(endpoint->endpoint_id);  in ipa_endpoint_exit_one()
1823 void ipa_endpoint_exit(struct ipa *ipa)  in ipa_endpoint_exit()  argument
1825 	u32 initialized = ipa->initialized;  in ipa_endpoint_exit()
1832 		ipa_endpoint_exit_one(&ipa->endpoint[endpoint_id]);  in ipa_endpoint_exit()
1834 	memset(ipa->name_map, 0, sizeof(ipa->name_map));  in ipa_endpoint_exit()
1835 	memset(ipa->channel_map, 0, sizeof(ipa->channel_map));  in ipa_endpoint_exit()
1839 u32 ipa_endpoint_init(struct ipa *ipa, u32 count,  in ipa_endpoint_init()  argument
1845 	if (!ipa_endpoint_data_valid(ipa, count, data))  in ipa_endpoint_init()
1848 	ipa->initialized = 0;  in ipa_endpoint_init()
1855 		ipa_endpoint_init_one(ipa, name, data);  in ipa_endpoint_init()
1861 	if (!ipa_filter_map_valid(ipa, filter_map))  in ipa_endpoint_init()
1867 	ipa_endpoint_exit(ipa);  in ipa_endpoint_init()