174 static u32 gsi_channel_id(struct gsi_channel *channel)  in gsi_channel_id()  argument
176 	return channel - &channel->gsi->channel[0];  in gsi_channel_id()
180 static bool gsi_channel_initialized(struct gsi_channel *channel)  in gsi_channel_initialized()  argument
182 	return !!channel->gsi;  in gsi_channel_initialized()
465 static enum gsi_channel_state gsi_channel_state(struct gsi_channel *channel)  in gsi_channel_state()  argument
467 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_state()
468 	void __iomem *virt = channel->gsi->virt;  in gsi_channel_state()
478 gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode)  in gsi_channel_command()  argument
480 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_command()
481 	struct gsi *gsi = channel->gsi;  in gsi_channel_command()
499 		opcode, channel_id, gsi_channel_state(channel));  in gsi_channel_command()
505 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_alloc_command()  local
510 	state = gsi_channel_state(channel);  in gsi_channel_alloc_command()
517 	gsi_channel_command(channel, GSI_CH_ALLOCATE);  in gsi_channel_alloc_command()
520 	state = gsi_channel_state(channel);  in gsi_channel_alloc_command()
531 static int gsi_channel_start_command(struct gsi_channel *channel)  in gsi_channel_start_command()  argument
533 	struct device *dev = channel->gsi->dev;  in gsi_channel_start_command()
536 	state = gsi_channel_state(channel);  in gsi_channel_start_command()
540 			gsi_channel_id(channel), state);  in gsi_channel_start_command()
544 	gsi_channel_command(channel, GSI_CH_START);  in gsi_channel_start_command()
547 	state = gsi_channel_state(channel);  in gsi_channel_start_command()
552 		gsi_channel_id(channel), state);  in gsi_channel_start_command()
558 static int gsi_channel_stop_command(struct gsi_channel *channel)  in gsi_channel_stop_command()  argument
560 	struct device *dev = channel->gsi->dev;  in gsi_channel_stop_command()
563 	state = gsi_channel_state(channel);  in gsi_channel_stop_command()
574 			gsi_channel_id(channel), state);  in gsi_channel_stop_command()
578 	gsi_channel_command(channel, GSI_CH_STOP);  in gsi_channel_stop_command()
581 	state = gsi_channel_state(channel);  in gsi_channel_stop_command()
590 		gsi_channel_id(channel), state);  in gsi_channel_stop_command()
596 static void gsi_channel_reset_command(struct gsi_channel *channel)  in gsi_channel_reset_command()  argument
598 	struct device *dev = channel->gsi->dev;  in gsi_channel_reset_command()
604 	state = gsi_channel_state(channel);  in gsi_channel_reset_command()
610 				gsi_channel_id(channel), state);  in gsi_channel_reset_command()
614 	gsi_channel_command(channel, GSI_CH_RESET);  in gsi_channel_reset_command()
617 	state = gsi_channel_state(channel);  in gsi_channel_reset_command()
620 			gsi_channel_id(channel), state);  in gsi_channel_reset_command()
626 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_de_alloc_command()  local
630 	state = gsi_channel_state(channel);  in gsi_channel_de_alloc_command()
637 	gsi_channel_command(channel, GSI_CH_DE_ALLOC);  in gsi_channel_de_alloc_command()
640 	state = gsi_channel_state(channel);  in gsi_channel_de_alloc_command()
710 static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel)  in gsi_channel_trans_last()  argument
712 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_channel_trans_last()
717 	if (channel->toward_ipa && pending_id != trans_info->free_id) {  in gsi_channel_trans_last()
737 	trans = &trans_info->trans[trans_id % channel->tre_count];  in gsi_channel_trans_last()
744 static void gsi_channel_trans_quiesce(struct gsi_channel *channel)  in gsi_channel_trans_quiesce()  argument
749 	trans = gsi_channel_trans_last(channel);  in gsi_channel_trans_quiesce()
757 static void gsi_channel_program(struct gsi_channel *channel, bool doorbell)  in gsi_channel_program()  argument
759 	size_t size = channel->tre_ring.count * GSI_RING_ELEMENT_SIZE;  in gsi_channel_program()
760 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_program()
763 	struct gsi *gsi = channel->gsi;  in gsi_channel_program()
769 	if (channel->toward_ipa)  in gsi_channel_program()
771 	val |= u32_encode_bits(channel->evt_ring_id, ERINDEX_FMASK);  in gsi_channel_program()
782 	val = lower_32_bits(channel->tre_ring.addr);  in gsi_channel_program()
784 	val = upper_32_bits(channel->tre_ring.addr);  in gsi_channel_program()
788 	if (channel->command)  in gsi_channel_program()
801 	if (gsi->version >= IPA_VERSION_4_0 && !channel->command) {  in gsi_channel_program()
817 	gpi->max_outstanding_tre = channel->trans_tre_max *  in gsi_channel_program()
841 static int __gsi_channel_start(struct gsi_channel *channel, bool resume)  in __gsi_channel_start()  argument
843 	struct gsi *gsi = channel->gsi;  in __gsi_channel_start()
852 	ret = gsi_channel_start_command(channel);  in __gsi_channel_start()
862 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_start()  local
866 	napi_enable(&channel->napi);  in gsi_channel_start()
867 	gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id);  in gsi_channel_start()
869 	ret = __gsi_channel_start(channel, false);  in gsi_channel_start()
871 		gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id);  in gsi_channel_start()
872 		napi_disable(&channel->napi);  in gsi_channel_start()
878 static int gsi_channel_stop_retry(struct gsi_channel *channel)  in gsi_channel_stop_retry()  argument
884 		ret = gsi_channel_stop_command(channel);  in gsi_channel_stop_retry()
893 static int __gsi_channel_stop(struct gsi_channel *channel, bool suspend)  in __gsi_channel_stop()  argument
895 	struct gsi *gsi = channel->gsi;  in __gsi_channel_stop()
899 	gsi_channel_trans_quiesce(channel);  in __gsi_channel_stop()
907 	ret = gsi_channel_stop_retry(channel);  in __gsi_channel_stop()
917 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_stop()  local
920 	ret = __gsi_channel_stop(channel, false);  in gsi_channel_stop()
925 	gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id);  in gsi_channel_stop()
926 	napi_disable(&channel->napi);  in gsi_channel_stop()
934 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_reset()  local
938 	gsi_channel_reset_command(channel);  in gsi_channel_reset()
940 	if (gsi->version < IPA_VERSION_4_0 && !channel->toward_ipa)  in gsi_channel_reset()
941 		gsi_channel_reset_command(channel);  in gsi_channel_reset()
944 	channel->tre_ring.index = 0;  in gsi_channel_reset()
945 	gsi_channel_program(channel, doorbell);  in gsi_channel_reset()
946 	gsi_channel_trans_cancel_pending(channel);  in gsi_channel_reset()
954 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_suspend()  local
957 	ret = __gsi_channel_stop(channel, true);  in gsi_channel_suspend()
962 	napi_synchronize(&channel->napi);  in gsi_channel_suspend()
970 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_resume()  local
972 	return __gsi_channel_start(channel, true);  in gsi_channel_resume()
989 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_tx_committed()  local
991 	channel->trans_count++;  in gsi_trans_tx_committed()
992 	channel->byte_count += trans->len;  in gsi_trans_tx_committed()
994 	trans->trans_count = channel->trans_count;  in gsi_trans_tx_committed()
995 	trans->byte_count = channel->byte_count;  in gsi_trans_tx_committed()
1002 	struct gsi_channel *channel;  in gsi_trans_tx_queued()  local
1006 	channel = &gsi->channel[channel_id];  in gsi_trans_tx_queued()
1008 	byte_count = channel->byte_count - channel->queued_byte_count;  in gsi_trans_tx_queued()
1009 	trans_count = channel->trans_count - channel->queued_trans_count;  in gsi_trans_tx_queued()
1010 	channel->queued_byte_count = channel->byte_count;  in gsi_trans_tx_queued()
1011 	channel->queued_trans_count = channel->trans_count;  in gsi_trans_tx_queued()
1035 	struct gsi_channel *channel;  in gsi_trans_tx_completed()  local
1039 	channel = &gsi->channel[channel_id];  in gsi_trans_tx_completed()
1040 	trans_count = trans->trans_count - channel->compl_trans_count;  in gsi_trans_tx_completed()
1041 	byte_count = trans->byte_count - channel->compl_byte_count;  in gsi_trans_tx_completed()
1043 	channel->compl_trans_count += trans_count;  in gsi_trans_tx_completed()
1044 	channel->compl_byte_count += byte_count;  in gsi_trans_tx_completed()
1104 		u32 channel_id = gsi_channel_id(evt_ring->channel);  in gsi_isr_glob_evt_err()
1228 		napi_schedule(&gsi->evt_ring[evt_ring_id].channel->napi);  in gsi_isr_ieob()
1318 	struct gsi_channel *channel;  in gsi_event_trans()  local
1323 	channel = &gsi->channel[channel_id];  in gsi_event_trans()
1324 	if (WARN(!channel->gsi, "event has bad channel %u\n", channel_id))  in gsi_event_trans()
1329 	tre_index = gsi_ring_index(&channel->tre_ring, tre_offset);  in gsi_event_trans()
1331 	trans = gsi_channel_trans_mapped(channel, tre_index);  in gsi_event_trans()
1465 void gsi_channel_doorbell(struct gsi_channel *channel)  in gsi_channel_doorbell()  argument
1467 	struct gsi_ring *tre_ring = &channel->tre_ring;  in gsi_channel_doorbell()
1468 	u32 channel_id = gsi_channel_id(channel);  in gsi_channel_doorbell()
1469 	struct gsi *gsi = channel->gsi;  in gsi_channel_doorbell()
1478 void gsi_channel_update(struct gsi_channel *channel)  in gsi_channel_update()  argument
1480 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_update()
1481 	struct gsi *gsi = channel->gsi;  in gsi_channel_update()
1524 static struct gsi_trans *gsi_channel_poll_one(struct gsi_channel *channel)  in gsi_channel_poll_one()  argument
1529 	trans = gsi_channel_trans_complete(channel);  in gsi_channel_poll_one()
1550 	struct gsi_channel *channel;  in gsi_channel_poll()  local
1553 	channel = container_of(napi, struct gsi_channel, napi);  in gsi_channel_poll()
1557 		trans = gsi_channel_poll_one(channel);  in gsi_channel_poll()
1564 		gsi_irq_ieob_enable_one(channel->gsi, channel->evt_ring_id);  in gsi_channel_poll()
1586 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_setup_one()  local
1587 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_setup_one()
1590 	if (!gsi_channel_initialized(channel))  in gsi_channel_setup_one()
1603 	gsi_channel_program(channel, true);  in gsi_channel_setup_one()
1605 	if (channel->toward_ipa)  in gsi_channel_setup_one()
1606 		netif_napi_add_tx(&gsi->dummy_dev, &channel->napi,  in gsi_channel_setup_one()
1609 		netif_napi_add(&gsi->dummy_dev, &channel->napi,  in gsi_channel_setup_one()
1624 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_teardown_one()  local
1625 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_teardown_one()
1627 	if (!gsi_channel_initialized(channel))  in gsi_channel_teardown_one()
1630 	netif_napi_del(&channel->napi);  in gsi_channel_teardown_one()
1751 		struct gsi_channel *channel = &gsi->channel[channel_id++];  in gsi_channel_setup()  local
1753 		if (!gsi_channel_initialized(channel))  in gsi_channel_setup()
1959 static int gsi_channel_evt_ring_init(struct gsi_channel *channel)  in gsi_channel_evt_ring_init()  argument
1961 	struct gsi *gsi = channel->gsi;  in gsi_channel_evt_ring_init()
1968 	channel->evt_ring_id = ret;  in gsi_channel_evt_ring_init()
1970 	evt_ring = &gsi->evt_ring[channel->evt_ring_id];  in gsi_channel_evt_ring_init()
1971 	evt_ring->channel = channel;  in gsi_channel_evt_ring_init()
1973 	ret = gsi_ring_alloc(gsi, &evt_ring->ring, channel->event_count);  in gsi_channel_evt_ring_init()
1978 		ret, gsi_channel_id(channel));  in gsi_channel_evt_ring_init()
1980 	gsi_evt_ring_id_free(gsi, channel->evt_ring_id);  in gsi_channel_evt_ring_init()
1986 static void gsi_channel_evt_ring_exit(struct gsi_channel *channel)  in gsi_channel_evt_ring_exit()  argument
1988 	u32 evt_ring_id = channel->evt_ring_id;  in gsi_channel_evt_ring_exit()
1989 	struct gsi *gsi = channel->gsi;  in gsi_channel_evt_ring_exit()
2021 	channel_data = &data->channel;  in gsi_channel_data_valid()
2069 	struct gsi_channel *channel;  in gsi_channel_init_one()  local
2077 	if (data->channel.tre_count > data->channel.event_count) {  in gsi_channel_init_one()
2078 		tre_count = data->channel.event_count;  in gsi_channel_init_one()
2082 		tre_count = data->channel.tre_count;  in gsi_channel_init_one()
2085 	channel = &gsi->channel[data->channel_id];  in gsi_channel_init_one()
2086 	memset(channel, 0, sizeof(*channel));  in gsi_channel_init_one()
2088 	channel->gsi = gsi;  in gsi_channel_init_one()
2089 	channel->toward_ipa = data->toward_ipa;  in gsi_channel_init_one()
2090 	channel->command = command;  in gsi_channel_init_one()
2091 	channel->trans_tre_max = data->channel.tlv_count;  in gsi_channel_init_one()
2092 	channel->tre_count = tre_count;  in gsi_channel_init_one()
2093 	channel->event_count = data->channel.event_count;  in gsi_channel_init_one()
2095 	ret = gsi_channel_evt_ring_init(channel);  in gsi_channel_init_one()
2099 	ret = gsi_ring_alloc(gsi, &channel->tre_ring, data->channel.tre_count);  in gsi_channel_init_one()
2113 		ret = ipa_cmd_pool_init(channel, tre_max);  in gsi_channel_init_one()
2118 	gsi_channel_trans_exit(channel);  in gsi_channel_init_one()
2120 	gsi_ring_free(gsi, &channel->tre_ring);  in gsi_channel_init_one()
2122 	gsi_channel_evt_ring_exit(channel);  in gsi_channel_init_one()
2124 	channel->gsi = NULL;	/* Mark it not (fully) initialized */  in gsi_channel_init_one()
2130 static void gsi_channel_exit_one(struct gsi_channel *channel)  in gsi_channel_exit_one()  argument
2132 	if (!gsi_channel_initialized(channel))  in gsi_channel_exit_one()
2135 	if (channel->command)  in gsi_channel_exit_one()
2136 		ipa_cmd_pool_exit(channel);  in gsi_channel_exit_one()
2137 	gsi_channel_trans_exit(channel);  in gsi_channel_exit_one()
2138 	gsi_ring_free(channel->gsi, &channel->tre_ring);  in gsi_channel_exit_one()
2139 	gsi_channel_evt_ring_exit(channel);  in gsi_channel_exit_one()
2186 		gsi_channel_exit_one(&gsi->channel[data->channel_id]);  in gsi_channel_init()
2198 		gsi_channel_exit_one(&gsi->channel[channel_id]);  in gsi_channel_exit()
2303 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_tre_max()  local
2306 	return channel->tre_count - (channel->trans_tre_max - 1);  in gsi_channel_tre_max()