227 static u32 gsi_channel_id(struct gsi_channel *channel)  in gsi_channel_id()  argument
229 	return channel - &channel->gsi->channel[0];  in gsi_channel_id()
417 static enum gsi_channel_state gsi_channel_state(struct gsi_channel *channel)  in gsi_channel_state()  argument
419 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_state()
420 	void *virt = channel->gsi->virt;  in gsi_channel_state()
430 gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode)  in gsi_channel_command()  argument
432 	struct completion *completion = &channel->completion;  in gsi_channel_command()
433 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_command()
434 	struct gsi *gsi = channel->gsi;  in gsi_channel_command()
445 		opcode, channel_id, gsi_channel_state(channel));  in gsi_channel_command()
453 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_alloc_command()  local
459 	state = gsi_channel_state(channel);  in gsi_channel_alloc_command()
465 	ret = gsi_channel_command(channel, GSI_CH_ALLOCATE);  in gsi_channel_alloc_command()
468 	state = gsi_channel_state(channel);  in gsi_channel_alloc_command()
478 static int gsi_channel_start_command(struct gsi_channel *channel)  in gsi_channel_start_command()  argument
480 	struct device *dev = channel->gsi->dev;  in gsi_channel_start_command()
484 	state = gsi_channel_state(channel);  in gsi_channel_start_command()
491 	ret = gsi_channel_command(channel, GSI_CH_START);  in gsi_channel_start_command()
494 	state = gsi_channel_state(channel);  in gsi_channel_start_command()
504 static int gsi_channel_stop_command(struct gsi_channel *channel)  in gsi_channel_stop_command()  argument
506 	struct device *dev = channel->gsi->dev;  in gsi_channel_stop_command()
510 	state = gsi_channel_state(channel);  in gsi_channel_stop_command()
524 	ret = gsi_channel_command(channel, GSI_CH_STOP);  in gsi_channel_stop_command()
527 	state = gsi_channel_state(channel);  in gsi_channel_stop_command()
541 static void gsi_channel_reset_command(struct gsi_channel *channel)  in gsi_channel_reset_command()  argument
543 	struct device *dev = channel->gsi->dev;  in gsi_channel_reset_command()
549 	state = gsi_channel_state(channel);  in gsi_channel_reset_command()
556 	ret = gsi_channel_command(channel, GSI_CH_RESET);  in gsi_channel_reset_command()
559 	state = gsi_channel_state(channel);  in gsi_channel_reset_command()
567 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_de_alloc_command()  local
572 	state = gsi_channel_state(channel);  in gsi_channel_de_alloc_command()
578 	ret = gsi_channel_command(channel, GSI_CH_DE_ALLOC);  in gsi_channel_de_alloc_command()
581 	state = gsi_channel_state(channel);  in gsi_channel_de_alloc_command()
647 static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel)  in gsi_channel_trans_last()  argument
649 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_channel_trans_last()
673 static void gsi_channel_trans_quiesce(struct gsi_channel *channel)  in gsi_channel_trans_quiesce()  argument
678 	trans = gsi_channel_trans_last(channel);  in gsi_channel_trans_quiesce()
686 static void gsi_channel_freeze(struct gsi_channel *channel)  in gsi_channel_freeze()  argument
688 	gsi_channel_trans_quiesce(channel);  in gsi_channel_freeze()
690 	napi_disable(&channel->napi);  in gsi_channel_freeze()
692 	gsi_irq_ieob_disable(channel->gsi, channel->evt_ring_id);  in gsi_channel_freeze()
696 static void gsi_channel_thaw(struct gsi_channel *channel)  in gsi_channel_thaw()  argument
698 	gsi_irq_ieob_enable(channel->gsi, channel->evt_ring_id);  in gsi_channel_thaw()
700 	napi_enable(&channel->napi);  in gsi_channel_thaw()
704 static void gsi_channel_program(struct gsi_channel *channel, bool doorbell)  in gsi_channel_program()  argument
706 	size_t size = channel->tre_ring.count * GSI_RING_ELEMENT_SIZE;  in gsi_channel_program()
707 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_program()
710 	struct gsi *gsi = channel->gsi;  in gsi_channel_program()
715 	channel->tre_ring.index = 0;  in gsi_channel_program()
719 	if (channel->toward_ipa)  in gsi_channel_program()
721 	val |= u32_encode_bits(channel->evt_ring_id, ERINDEX_FMASK);  in gsi_channel_program()
732 	val = channel->tre_ring.addr & GENMASK(31, 0);  in gsi_channel_program()
735 	val = channel->tre_ring.addr >> 32;  in gsi_channel_program()
739 	if (channel->command)  in gsi_channel_program()
749 	if (!channel->use_prefetch)  in gsi_channel_program()
780 static void gsi_channel_deprogram(struct gsi_channel *channel)  in gsi_channel_deprogram()  argument
788 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_start()  local
793 	ret = gsi_channel_start_command(channel);  in gsi_channel_start()
797 	gsi_channel_thaw(channel);  in gsi_channel_start()
805 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_stop()  local
809 	gsi_channel_freeze(channel);  in gsi_channel_stop()
812 	retries = channel->toward_ipa ? 0 : GSI_CHANNEL_STOP_RX_RETRIES;  in gsi_channel_stop()
817 		ret = gsi_channel_stop_command(channel);  in gsi_channel_stop()
827 		gsi_channel_thaw(channel);  in gsi_channel_stop()
835 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_reset()  local
839 	gsi_channel_reset_command(channel);  in gsi_channel_reset()
841 	if (legacy && !channel->toward_ipa)  in gsi_channel_reset()
842 		gsi_channel_reset_command(channel);  in gsi_channel_reset()
844 	gsi_channel_program(channel, legacy);  in gsi_channel_reset()
845 	gsi_channel_trans_cancel_pending(channel);  in gsi_channel_reset()
853 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_suspend()  local
858 	gsi_channel_freeze(channel);  in gsi_channel_suspend()
866 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_resume()  local
871 	gsi_channel_thaw(channel);  in gsi_channel_resume()
894 void gsi_channel_tx_queued(struct gsi_channel *channel)  in gsi_channel_tx_queued()  argument
899 	byte_count = channel->byte_count - channel->queued_byte_count;  in gsi_channel_tx_queued()
900 	trans_count = channel->trans_count - channel->queued_trans_count;  in gsi_channel_tx_queued()
901 	channel->queued_byte_count = channel->byte_count;  in gsi_channel_tx_queued()
902 	channel->queued_trans_count = channel->trans_count;  in gsi_channel_tx_queued()
904 	ipa_gsi_channel_tx_queued(channel->gsi, gsi_channel_id(channel),  in gsi_channel_tx_queued()
933 gsi_channel_tx_update(struct gsi_channel *channel, struct gsi_trans *trans)  in gsi_channel_tx_update()  argument
938 	byte_count -= channel->compl_byte_count;  in gsi_channel_tx_update()
939 	channel->compl_byte_count += byte_count;  in gsi_channel_tx_update()
940 	trans_count -= channel->compl_trans_count;  in gsi_channel_tx_update()
941 	channel->compl_trans_count += trans_count;  in gsi_channel_tx_update()
943 	ipa_gsi_channel_tx_completed(channel->gsi, gsi_channel_id(channel),  in gsi_channel_tx_update()
957 		struct gsi_channel *channel;  in gsi_isr_chan_ctrl()  local
961 		channel = &gsi->channel[channel_id];  in gsi_isr_chan_ctrl()
963 		complete(&channel->completion);  in gsi_isr_chan_ctrl()
994 		complete(&gsi->channel[channel_id].completion);  in gsi_isr_glob_chan_err()
1009 		u32 channel_id = gsi_channel_id(evt_ring->channel);  in gsi_isr_glob_evt_err()
1100 		napi_schedule(&gsi->evt_ring[evt_ring_id].channel->napi);  in gsi_isr_ieob()
1172 static struct gsi_trans *gsi_event_trans(struct gsi_channel *channel,  in gsi_event_trans()  argument
1180 	tre_index = gsi_ring_index(&channel->tre_ring, tre_offset);  in gsi_event_trans()
1182 	return gsi_channel_trans_mapped(channel, tre_index);  in gsi_event_trans()
1206 	struct gsi_channel *channel = evt_ring->channel;  in gsi_evt_ring_rx_update()  local
1216 	trans_info = &channel->trans_info;  in gsi_evt_ring_rx_update()
1225 	trans = gsi_event_trans(channel, event);  in gsi_evt_ring_rx_update()
1245 	channel->byte_count += byte_count;  in gsi_evt_ring_rx_update()
1246 	channel->trans_count++;  in gsi_evt_ring_rx_update()
1303 void gsi_channel_doorbell(struct gsi_channel *channel)  in gsi_channel_doorbell()  argument
1305 	struct gsi_ring *tre_ring = &channel->tre_ring;  in gsi_channel_doorbell()
1306 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_doorbell()
1307 	struct gsi *gsi = channel->gsi;  in gsi_channel_doorbell()
1316 static void gsi_channel_update(struct gsi_channel *channel)  in gsi_channel_update()  argument
1318 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_update()
1319 	struct gsi *gsi = channel->gsi;  in gsi_channel_update()
1341 	trans = gsi_event_trans(channel, gsi_ring_virt(ring, index - 1));  in gsi_channel_update()
1349 	if (channel->toward_ipa)  in gsi_channel_update()
1350 		gsi_channel_tx_update(channel, trans);  in gsi_channel_update()
1357 	gsi_evt_ring_doorbell(channel->gsi, channel->evt_ring_id, index);  in gsi_channel_update()
1374 static struct gsi_trans *gsi_channel_poll_one(struct gsi_channel *channel)  in gsi_channel_poll_one()  argument
1379 	trans = gsi_channel_trans_complete(channel);  in gsi_channel_poll_one()
1382 		gsi_channel_update(channel);  in gsi_channel_poll_one()
1383 		trans = gsi_channel_trans_complete(channel);  in gsi_channel_poll_one()
1406 	struct gsi_channel *channel;  in gsi_channel_poll()  local
1409 	channel = container_of(napi, struct gsi_channel, napi);  in gsi_channel_poll()
1414 		trans = gsi_channel_poll_one(channel);  in gsi_channel_poll()
1421 		napi_complete(&channel->napi);  in gsi_channel_poll()
1422 		gsi_irq_ieob_enable(channel->gsi, channel->evt_ring_id);  in gsi_channel_poll()
1458 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_setup_one()  local
1459 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_setup_one()
1462 	if (!channel->gsi)  in gsi_channel_setup_one()
1475 	gsi_channel_program(channel, legacy);  in gsi_channel_setup_one()
1477 	if (channel->toward_ipa)  in gsi_channel_setup_one()
1478 		netif_tx_napi_add(&gsi->dummy_dev, &channel->napi,  in gsi_channel_setup_one()
1481 		netif_napi_add(&gsi->dummy_dev, &channel->napi,  in gsi_channel_setup_one()
1496 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_teardown_one()  local
1497 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_teardown_one()
1499 	if (!channel->gsi)  in gsi_channel_teardown_one()
1502 	netif_napi_del(&channel->napi);  in gsi_channel_teardown_one()
1504 	gsi_channel_deprogram(channel);  in gsi_channel_teardown_one()
1571 		struct gsi_channel *channel = &gsi->channel[channel_id++];  in gsi_channel_setup()  local
1573 		if (!channel->gsi)  in gsi_channel_setup()
1704 static int gsi_channel_evt_ring_init(struct gsi_channel *channel)  in gsi_channel_evt_ring_init()  argument
1706 	struct gsi *gsi = channel->gsi;  in gsi_channel_evt_ring_init()
1713 	channel->evt_ring_id = ret;  in gsi_channel_evt_ring_init()
1715 	evt_ring = &gsi->evt_ring[channel->evt_ring_id];  in gsi_channel_evt_ring_init()
1716 	evt_ring->channel = channel;  in gsi_channel_evt_ring_init()
1718 	ret = gsi_ring_alloc(gsi, &evt_ring->ring, channel->event_count);  in gsi_channel_evt_ring_init()
1723 		ret, gsi_channel_id(channel));  in gsi_channel_evt_ring_init()
1725 	gsi_evt_ring_id_free(gsi, channel->evt_ring_id);  in gsi_channel_evt_ring_init()
1731 static void gsi_channel_evt_ring_exit(struct gsi_channel *channel)  in gsi_channel_evt_ring_exit()  argument
1733 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_evt_ring_exit()
1734 	struct gsi *gsi = channel->gsi;  in gsi_channel_evt_ring_exit()
1779 	if (!data->channel.tlv_count ||  in gsi_channel_data_valid()
1780 	    data->channel.tlv_count > GSI_TLV_MAX) {  in gsi_channel_data_valid()
1782 			channel_id, data->channel.tlv_count, GSI_TLV_MAX);  in gsi_channel_data_valid()
1791 	if (data->channel.tre_count < 2 * data->channel.tlv_count - 1) {  in gsi_channel_data_valid()
1793 			channel_id, data->channel.tlv_count,  in gsi_channel_data_valid()
1794 			data->channel.tre_count);  in gsi_channel_data_valid()
1798 	if (!is_power_of_2(data->channel.tre_count)) {  in gsi_channel_data_valid()
1800 			channel_id, data->channel.tre_count);  in gsi_channel_data_valid()
1804 	if (!is_power_of_2(data->channel.event_count)) {  in gsi_channel_data_valid()
1806 			channel_id, data->channel.event_count);  in gsi_channel_data_valid()
1819 	struct gsi_channel *channel;  in gsi_channel_init_one()  local
1827 	if (data->channel.tre_count > data->channel.event_count) {  in gsi_channel_init_one()
1828 		tre_count = data->channel.event_count;  in gsi_channel_init_one()
1832 		tre_count = data->channel.tre_count;  in gsi_channel_init_one()
1835 	channel = &gsi->channel[data->channel_id];  in gsi_channel_init_one()
1836 	memset(channel, 0, sizeof(*channel));  in gsi_channel_init_one()
1838 	channel->gsi = gsi;  in gsi_channel_init_one()
1839 	channel->toward_ipa = data->toward_ipa;  in gsi_channel_init_one()
1840 	channel->command = command;  in gsi_channel_init_one()
1841 	channel->use_prefetch = command && prefetch;  in gsi_channel_init_one()
1842 	channel->tlv_count = data->channel.tlv_count;  in gsi_channel_init_one()
1843 	channel->tre_count = tre_count;  in gsi_channel_init_one()
1844 	channel->event_count = data->channel.event_count;  in gsi_channel_init_one()
1845 	init_completion(&channel->completion);  in gsi_channel_init_one()
1847 	ret = gsi_channel_evt_ring_init(channel);  in gsi_channel_init_one()
1851 	ret = gsi_ring_alloc(gsi, &channel->tre_ring, data->channel.tre_count);  in gsi_channel_init_one()
1865 		ret = ipa_cmd_pool_init(channel, tre_max);  in gsi_channel_init_one()
1870 	gsi_channel_trans_exit(channel);  in gsi_channel_init_one()
1872 	gsi_ring_free(gsi, &channel->tre_ring);  in gsi_channel_init_one()
1874 	gsi_channel_evt_ring_exit(channel);  in gsi_channel_init_one()
1876 	channel->gsi = NULL;	/* Mark it not (fully) initialized */  in gsi_channel_init_one()
1882 static void gsi_channel_exit_one(struct gsi_channel *channel)  in gsi_channel_exit_one()  argument
1884 	if (!channel->gsi)  in gsi_channel_exit_one()
1887 	if (channel->command)  in gsi_channel_exit_one()
1888 		ipa_cmd_pool_exit(channel);  in gsi_channel_exit_one()
1889 	gsi_channel_trans_exit(channel);  in gsi_channel_exit_one()
1890 	gsi_ring_free(channel->gsi, &channel->tre_ring);  in gsi_channel_exit_one()
1891 	gsi_channel_evt_ring_exit(channel);  in gsi_channel_exit_one()
1934 		gsi_channel_exit_one(&gsi->channel[data->channel_id]);  in gsi_channel_init()
1947 		gsi_channel_exit_one(&gsi->channel[channel_id]);  in gsi_channel_exit()
2059 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_tre_max()  local
2062 	return channel->tre_count - (channel->tlv_count - 1);  in gsi_channel_tre_max()
2068 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_trans_tre_max()  local
2070 	return channel->tlv_count;  in gsi_channel_trans_tre_max()