485 struct gchan {  struct
503 	struct gchan gchan[MAX_CHANNELS_PER_GPII];  argument
531 	struct gchan *gchan;  member
545 static inline struct gchan *to_gchan(struct dma_chan *dma_chan)  in to_gchan()
547 	return container_of(dma_chan, struct gchan, vc.chan);  in to_gchan()
686 static int gpi_send_cmd(struct gpii *gpii, struct gchan *gchan,  in gpi_send_cmd()  argument
697 		chid = gchan->chid;  in gpi_send_cmd()
706 	cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg;  in gpi_send_cmd()
722 	if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state)  in gpi_send_cmd()
732 static inline void gpi_write_ch_db(struct gchan *gchan,  in gpi_write_ch_db()  argument
735 	struct gpii *gpii = gchan->gpii;  in gpi_write_ch_db()
739 	gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp);  in gpi_write_ch_db()
767 	struct gchan *gchan;  in gpi_process_ch_ctrl_irq()  local
778 		gchan = &gpii->gchan[chid];  in gpi_process_ch_ctrl_irq()
779 		state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg +  in gpi_process_ch_ctrl_irq()
790 		gchan->ch_state = state;  in gpi_process_ch_ctrl_irq()
797 		if (gchan->ch_state != CH_STATE_STOP_IN_PROC)  in gpi_process_ch_ctrl_irq()
929 static void gpi_process_imed_data_event(struct gchan *gchan,  in gpi_process_imed_data_event()  argument
932 	struct gpii *gpii = gchan->gpii;  in gpi_process_imed_data_event()
933 	struct gpi_ring *ch_ring = &gchan->ch_ring;  in gpi_process_imed_data_event()
944 	if (gchan->pm_state != ACTIVE_STATE) {  in gpi_process_imed_data_event()
946 			TO_GPI_PM_STR(gchan->pm_state));  in gpi_process_imed_data_event()
950 	spin_lock_irqsave(&gchan->vc.lock, flags);  in gpi_process_imed_data_event()
951 	vd = vchan_next_desc(&gchan->vc);  in gpi_process_imed_data_event()
956 		spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_process_imed_data_event()
971 	spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_process_imed_data_event()
1003 	spin_lock_irqsave(&gchan->vc.lock, flags);  in gpi_process_imed_data_event()
1005 	spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_process_imed_data_event()
1011 static void gpi_process_xfer_compl_event(struct gchan *gchan,  in gpi_process_xfer_compl_event()  argument
1014 	struct gpii *gpii = gchan->gpii;  in gpi_process_xfer_compl_event()
1015 	struct gpi_ring *ch_ring = &gchan->ch_ring;  in gpi_process_xfer_compl_event()
1024 	if (unlikely(gchan->pm_state != ACTIVE_STATE)) {  in gpi_process_xfer_compl_event()
1026 			TO_GPI_PM_STR(gchan->pm_state));  in gpi_process_xfer_compl_event()
1030 	spin_lock_irqsave(&gchan->vc.lock, flags);  in gpi_process_xfer_compl_event()
1031 	vd = vchan_next_desc(&gchan->vc);  in gpi_process_xfer_compl_event()
1035 		spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_process_xfer_compl_event()
1046 	spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_process_xfer_compl_event()
1082 	spin_lock_irqsave(&gchan->vc.lock, flags);  in gpi_process_xfer_compl_event()
1084 	spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_process_xfer_compl_event()
1096 	struct gchan *gchan;  in gpi_process_events()  local
1116 				gchan = &gpii->gchan[chid];  in gpi_process_events()
1117 				gpi_process_xfer_compl_event(gchan,  in gpi_process_events()
1124 				gchan = &gpii->gchan[chid];  in gpi_process_events()
1125 				gpi_process_imed_data_event(gchan,  in gpi_process_events()
1170 static void gpi_mark_stale_events(struct gchan *gchan)  in gpi_mark_stale_events()  argument
1172 	struct gpii *gpii = gchan->gpii;  in gpi_mark_stale_events()
1185 		if (chid == gchan->chid)  in gpi_mark_stale_events()
1196 static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd)  in gpi_reset_chan()  argument
1198 	struct gpii *gpii = gchan->gpii;  in gpi_reset_chan()
1199 	struct gpi_ring *ch_ring = &gchan->ch_ring;  in gpi_reset_chan()
1204 	ret = gpi_send_cmd(gpii, gchan, gpi_cmd);  in gpi_reset_chan()
1220 	gpi_mark_stale_events(gchan);  in gpi_reset_chan()
1223 	spin_lock_irqsave(&gchan->vc.lock, flags);  in gpi_reset_chan()
1224 	vchan_get_all_descriptors(&gchan->vc, &list);  in gpi_reset_chan()
1225 	spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_reset_chan()
1227 	vchan_dma_desc_free_list(&gchan->vc, &list);  in gpi_reset_chan()
1232 static int gpi_start_chan(struct gchan *gchan)  in gpi_start_chan()  argument
1234 	struct gpii *gpii = gchan->gpii;  in gpi_start_chan()
1237 	ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_START);  in gpi_start_chan()
1246 	gchan->pm_state = ACTIVE_STATE;  in gpi_start_chan()
1252 static int gpi_stop_chan(struct gchan *gchan)  in gpi_stop_chan()  argument
1254 	struct gpii *gpii = gchan->gpii;  in gpi_stop_chan()
1257 	ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_STOP);  in gpi_stop_chan()
1268 static int gpi_alloc_chan(struct gchan *chan, bool send_alloc_cmd)  in gpi_alloc_chan()
1455 static void gpi_queue_xfer(struct gpii *gpii, struct gchan *gchan,  in gpi_queue_xfer()  argument
1462 	ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre);  in gpi_queue_xfer()
1476 	struct gchan *gchan = to_gchan(chan);  in gpi_terminate_all()  local
1477 	struct gpii *gpii = gchan->gpii;  in gpi_terminate_all()
1487 	schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0;  in gpi_terminate_all()
1488 	echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII;  in gpi_terminate_all()
1492 		gchan = &gpii->gchan[i];  in gpi_terminate_all()
1496 		gchan->pm_state = PREPARE_TERMINATE;  in gpi_terminate_all()
1500 		ret = gpi_stop_chan(gchan);  in gpi_terminate_all()
1505 		gchan = &gpii->gchan[i];  in gpi_terminate_all()
1507 		ret = gpi_reset_chan(gchan, GPI_CH_CMD_RESET);  in gpi_terminate_all()
1514 		ret = gpi_alloc_chan(gchan, false);  in gpi_terminate_all()
1523 		gchan = &gpii->gchan[i];  in gpi_terminate_all()
1525 		ret = gpi_start_chan(gchan);  in gpi_terminate_all()
1540 	struct gchan *gchan = to_gchan(chan);  in gpi_pause()  local
1541 	struct gpii *gpii = gchan->gpii;  in gpi_pause()
1558 		ret = gpi_stop_chan(&gpii->gchan[i]);  in gpi_pause()
1581 	struct gchan *gchan = to_gchan(chan);  in gpi_resume()  local
1582 	struct gpii *gpii = gchan->gpii;  in gpi_resume()
1596 		ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START);  in gpi_resume()
1623 	struct gchan *gchan = to_gchan(chan);  in gpi_peripheral_config()  local
1628 	gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT);  in gpi_peripheral_config()
1629 	if (!gchan->config)  in gpi_peripheral_config()
1632 	memcpy(gchan->config, config->peripheral_config, config->peripheral_size);  in gpi_peripheral_config()
1637 static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc,  in gpi_create_i2c_tre()
1712 static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc,  in gpi_create_spi_tre()
1795 	struct gchan *gchan = to_gchan(chan);  in gpi_prep_slave_sg()  local
1796 	struct gpii *gpii = gchan->gpii;  in gpi_prep_slave_sg()
1798 	struct gpi_ring *ch_ring = &gchan->ch_ring;  in gpi_prep_slave_sg()
1816 	set_config = *(u32 *)gchan->config;  in gpi_prep_slave_sg()
1834 	if (gchan->protocol == QCOM_GPI_SPI) {  in gpi_prep_slave_sg()
1835 		i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction);  in gpi_prep_slave_sg()
1836 	} else if (gchan->protocol == QCOM_GPI_I2C) {  in gpi_prep_slave_sg()
1837 		i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction);  in gpi_prep_slave_sg()
1839 		dev_err(dev, "invalid peripheral: %d\n", gchan->protocol);  in gpi_prep_slave_sg()
1845 	gpi_desc->gchan = gchan;  in gpi_prep_slave_sg()
1849 	return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags);  in gpi_prep_slave_sg()
1855 	struct gchan *gchan = to_gchan(chan);  in gpi_issue_pending()  local
1856 	struct gpii *gpii = gchan->gpii;  in gpi_issue_pending()
1860 	struct gpi_ring *ch_ring = &gchan->ch_ring;  in gpi_issue_pending()
1867 	spin_lock_irqsave(&gchan->vc.lock, flags);  in gpi_issue_pending()
1868 	if (vchan_issue_pending(&gchan->vc))  in gpi_issue_pending()
1869 		vd = list_last_entry(&gchan->vc.desc_issued,  in gpi_issue_pending()
1871 	spin_unlock_irqrestore(&gchan->vc.lock, flags);  in gpi_issue_pending()
1882 		gpi_queue_xfer(gpii, gchan, tre, &wp);  in gpi_issue_pending()
1886 	gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db);  in gpi_issue_pending()
1890 static int gpi_ch_init(struct gchan *gchan)  in gpi_ch_init()  argument
1892 	struct gpii *gpii = gchan->gpii;  in gpi_ch_init()
1897 	gchan->pm_state = CONFIG_STATE;  in gpi_ch_init()
1901 		if (gpii->gchan[i].pm_state != CONFIG_STATE)  in gpi_ch_init()
1905 	if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) {  in gpi_ch_init()
1907 			gpii->gchan[0].protocol, gpii->gchan[1].protocol);  in gpi_ch_init()
1938 		ret = gpi_alloc_chan(&gpii->gchan[i], true);  in gpi_ch_init()
1947 		ret = gpi_start_chan(&gpii->gchan[i]);  in gpi_ch_init()
1957 		gpi_stop_chan(&gpii->gchan[i]);  in gpi_ch_init()
1958 		gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);  in gpi_ch_init()
1963 		gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);  in gpi_ch_init()
1975 	struct gchan *gchan = to_gchan(chan);  in gpi_free_chan_resources()  local
1976 	struct gpii *gpii = gchan->gpii;  in gpi_free_chan_resources()
1982 	cur_state = gchan->pm_state;  in gpi_free_chan_resources()
1986 	gchan->pm_state = PREPARE_TERMINATE;  in gpi_free_chan_resources()
1991 		gpi_stop_chan(gchan);  in gpi_free_chan_resources()
1993 		ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);  in gpi_free_chan_resources()
1997 		gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);  in gpi_free_chan_resources()
2001 	gpi_free_ring(&gchan->ch_ring, gpii);  in gpi_free_chan_resources()
2002 	vchan_free_chan_resources(&gchan->vc);  in gpi_free_chan_resources()
2003 	kfree(gchan->config);  in gpi_free_chan_resources()
2006 	gchan->pm_state = DISABLE_STATE;  in gpi_free_chan_resources()
2011 		if (gpii->gchan[i].ch_ring.configured)  in gpi_free_chan_resources()
2045 	struct gchan *gchan = to_gchan(chan);  in gpi_alloc_chan_resources()  local
2046 	struct gpii *gpii = gchan->gpii;  in gpi_alloc_chan_resources()
2052 	ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES,  in gpi_alloc_chan_resources()
2057 	ret = gpi_ch_init(gchan);  in gpi_alloc_chan_resources()
2070 	struct gchan *tx_chan, *rx_chan;  in gpi_find_avail_gpii()
2078 		tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];  in gpi_find_avail_gpii()
2079 		rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];  in gpi_find_avail_gpii()
2092 		tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];  in gpi_find_avail_gpii()
2093 		rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];  in gpi_find_avail_gpii()
2115 	struct gchan *gchan;  in gpi_of_dma_xlate()  local
2138 	gchan = &gpi_dev->gpiis[gpii].gchan[chid];  in gpi_of_dma_xlate()
2139 	if (gchan->vc.chan.client_count) {  in gpi_of_dma_xlate()
2141 			gpii, chid, gchan->seid);  in gpi_of_dma_xlate()
2145 	gchan->seid = seid;  in gpi_of_dma_xlate()
2146 	gchan->protocol = args->args[2];  in gpi_of_dma_xlate()
2148 	return dma_get_slave_channel(&gchan->vc.chan);  in gpi_of_dma_xlate()
2222 			struct gchan *gchan = &gpii->gchan[chan];  in gpi_probe()  local
2225 			gchan->ch_cntxt_base_reg = gpi_dev->ee_base +  in gpi_probe()
2227 			gchan->ch_cntxt_db_reg = gpi_dev->ee_base +  in gpi_probe()
2229 			gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i);  in gpi_probe()
2232 			vchan_init(&gchan->vc, &gpi_dev->dma_device);  in gpi_probe()
2233 			gchan->vc.desc_free = gpi_desc_free;  in gpi_probe()
2234 			gchan->chid = chan;  in gpi_probe()
2235 			gchan->gpii = gpii;  in gpi_probe()
2236 			gchan->dir = GPII_CHAN_DIR[chan];  in gpi_probe()