Lines Matching refs:gsi
176 return channel - &channel->gsi->channel[0]; in gsi_channel_id()
182 return !!channel->gsi; in gsi_channel_initialized()
186 static void gsi_irq_type_update(struct gsi *gsi, u32 val) in gsi_irq_type_update() argument
188 gsi->type_enabled_bitmap = val; in gsi_irq_type_update()
189 iowrite32(val, gsi->virt + GSI_CNTXT_TYPE_IRQ_MSK_OFFSET); in gsi_irq_type_update()
192 static void gsi_irq_type_enable(struct gsi *gsi, enum gsi_irq_type_id type_id) in gsi_irq_type_enable() argument
194 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | BIT(type_id)); in gsi_irq_type_enable()
197 static void gsi_irq_type_disable(struct gsi *gsi, enum gsi_irq_type_id type_id) in gsi_irq_type_disable() argument
199 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap & ~BIT(type_id)); in gsi_irq_type_disable()
207 static void gsi_irq_ev_ctrl_enable(struct gsi *gsi, u32 evt_ring_id) in gsi_irq_ev_ctrl_enable() argument
215 iowrite32(~0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET); in gsi_irq_ev_ctrl_enable()
217 iowrite32(val, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET); in gsi_irq_ev_ctrl_enable()
218 gsi_irq_type_enable(gsi, GSI_EV_CTRL); in gsi_irq_ev_ctrl_enable()
222 static void gsi_irq_ev_ctrl_disable(struct gsi *gsi) in gsi_irq_ev_ctrl_disable() argument
224 gsi_irq_type_disable(gsi, GSI_EV_CTRL); in gsi_irq_ev_ctrl_disable()
225 iowrite32(0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET); in gsi_irq_ev_ctrl_disable()
233 static void gsi_irq_ch_ctrl_enable(struct gsi *gsi, u32 channel_id) in gsi_irq_ch_ctrl_enable() argument
241 iowrite32(~0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET); in gsi_irq_ch_ctrl_enable()
243 iowrite32(val, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET); in gsi_irq_ch_ctrl_enable()
244 gsi_irq_type_enable(gsi, GSI_CH_CTRL); in gsi_irq_ch_ctrl_enable()
248 static void gsi_irq_ch_ctrl_disable(struct gsi *gsi) in gsi_irq_ch_ctrl_disable() argument
250 gsi_irq_type_disable(gsi, GSI_CH_CTRL); in gsi_irq_ch_ctrl_disable()
251 iowrite32(0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET); in gsi_irq_ch_ctrl_disable()
254 static void gsi_irq_ieob_enable_one(struct gsi *gsi, u32 evt_ring_id) in gsi_irq_ieob_enable_one() argument
256 bool enable_ieob = !gsi->ieob_enabled_bitmap; in gsi_irq_ieob_enable_one()
259 gsi->ieob_enabled_bitmap |= BIT(evt_ring_id); in gsi_irq_ieob_enable_one()
260 val = gsi->ieob_enabled_bitmap; in gsi_irq_ieob_enable_one()
261 iowrite32(val, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); in gsi_irq_ieob_enable_one()
265 gsi_irq_type_enable(gsi, GSI_IEOB); in gsi_irq_ieob_enable_one()
268 static void gsi_irq_ieob_disable(struct gsi *gsi, u32 event_mask) in gsi_irq_ieob_disable() argument
272 gsi->ieob_enabled_bitmap &= ~event_mask; in gsi_irq_ieob_disable()
275 if (!gsi->ieob_enabled_bitmap) in gsi_irq_ieob_disable()
276 gsi_irq_type_disable(gsi, GSI_IEOB); in gsi_irq_ieob_disable()
278 val = gsi->ieob_enabled_bitmap; in gsi_irq_ieob_disable()
279 iowrite32(val, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); in gsi_irq_ieob_disable()
282 static void gsi_irq_ieob_disable_one(struct gsi *gsi, u32 evt_ring_id) in gsi_irq_ieob_disable_one() argument
284 gsi_irq_ieob_disable(gsi, BIT(evt_ring_id)); in gsi_irq_ieob_disable_one()
288 static void gsi_irq_enable(struct gsi *gsi) in gsi_irq_enable() argument
295 iowrite32(BIT(ERROR_INT), gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); in gsi_irq_enable()
296 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | BIT(GSI_GLOB_EE)); in gsi_irq_enable()
306 iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); in gsi_irq_enable()
307 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | BIT(GSI_GENERAL)); in gsi_irq_enable()
311 static void gsi_irq_disable(struct gsi *gsi) in gsi_irq_disable() argument
313 gsi_irq_type_update(gsi, 0); in gsi_irq_disable()
316 iowrite32(0, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); in gsi_irq_disable()
317 iowrite32(0, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); in gsi_irq_disable()
343 static bool gsi_command(struct gsi *gsi, u32 reg, u32 val) in gsi_command() argument
346 struct completion *completion = &gsi->completion; in gsi_command()
350 iowrite32(val, gsi->virt + reg); in gsi_command()
357 gsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_state() argument
361 val = ioread32(gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id)); in gsi_evt_ring_state()
367 static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id, in gsi_evt_ring_command() argument
370 struct device *dev = gsi->dev; in gsi_evt_ring_command()
375 gsi_irq_ev_ctrl_enable(gsi, evt_ring_id); in gsi_evt_ring_command()
380 timeout = !gsi_command(gsi, GSI_EV_CH_CMD_OFFSET, val); in gsi_evt_ring_command()
382 gsi_irq_ev_ctrl_disable(gsi); in gsi_evt_ring_command()
388 opcode, evt_ring_id, gsi_evt_ring_state(gsi, evt_ring_id)); in gsi_evt_ring_command()
392 static int gsi_evt_ring_alloc_command(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_alloc_command() argument
397 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_alloc_command()
399 dev_err(gsi->dev, "event ring %u bad state %u before alloc\n", in gsi_evt_ring_alloc_command()
404 gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_ALLOCATE); in gsi_evt_ring_alloc_command()
407 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_alloc_command()
411 dev_err(gsi->dev, "event ring %u bad state %u after alloc\n", in gsi_evt_ring_alloc_command()
418 static void gsi_evt_ring_reset_command(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_reset_command() argument
422 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_reset_command()
425 dev_err(gsi->dev, "event ring %u bad state %u before reset\n", in gsi_evt_ring_reset_command()
430 gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_RESET); in gsi_evt_ring_reset_command()
433 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_reset_command()
437 dev_err(gsi->dev, "event ring %u bad state %u after reset\n", in gsi_evt_ring_reset_command()
442 static void gsi_evt_ring_de_alloc_command(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_de_alloc_command() argument
446 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_de_alloc_command()
448 dev_err(gsi->dev, "event ring %u state %u before dealloc\n", in gsi_evt_ring_de_alloc_command()
453 gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_DE_ALLOC); in gsi_evt_ring_de_alloc_command()
456 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_de_alloc_command()
460 dev_err(gsi->dev, "event ring %u bad state %u after dealloc\n", in gsi_evt_ring_de_alloc_command()
468 void __iomem *virt = channel->gsi->virt; in gsi_channel_state()
481 struct gsi *gsi = channel->gsi; in gsi_channel_command() local
482 struct device *dev = gsi->dev; in gsi_channel_command()
487 gsi_irq_ch_ctrl_enable(gsi, channel_id); in gsi_channel_command()
491 timeout = !gsi_command(gsi, GSI_CH_CMD_OFFSET, val); in gsi_channel_command()
493 gsi_irq_ch_ctrl_disable(gsi); in gsi_channel_command()
503 static int gsi_channel_alloc_command(struct gsi *gsi, u32 channel_id) in gsi_channel_alloc_command() argument
505 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_alloc_command()
506 struct device *dev = gsi->dev; in gsi_channel_alloc_command()
533 struct device *dev = channel->gsi->dev; in gsi_channel_start_command()
560 struct device *dev = channel->gsi->dev; in gsi_channel_stop_command()
598 struct device *dev = channel->gsi->dev; in gsi_channel_reset_command()
624 static void gsi_channel_de_alloc_command(struct gsi *gsi, u32 channel_id) in gsi_channel_de_alloc_command() argument
626 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_de_alloc_command()
627 struct device *dev = gsi->dev; in gsi_channel_de_alloc_command()
652 static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index) in gsi_evt_ring_doorbell() argument
654 struct gsi_ring *ring = &gsi->evt_ring[evt_ring_id].ring; in gsi_evt_ring_doorbell()
661 iowrite32(val, gsi->virt + GSI_EV_CH_E_DOORBELL_0_OFFSET(evt_ring_id)); in gsi_evt_ring_doorbell()
665 static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_program() argument
667 struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_evt_ring_program()
676 iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
679 val = ev_r_length_encoded(gsi->version, size); in gsi_evt_ring_program()
680 iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_1_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
687 iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_2_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
689 iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_3_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
694 iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_8_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
697 iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_9_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
698 iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_10_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
699 iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_11_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
702 iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_12_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
703 iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_13_OFFSET(evt_ring_id)); in gsi_evt_ring_program()
706 gsi_evt_ring_doorbell(gsi, evt_ring_id, ring->index); in gsi_evt_ring_program()
763 struct gsi *gsi = channel->gsi; in gsi_channel_program() local
768 val = chtype_protocol_encoded(gsi->version, GSI_CHANNEL_TYPE_GPI); in gsi_channel_program()
773 iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_0_OFFSET(channel_id)); in gsi_channel_program()
775 val = r_length_encoded(gsi->version, size); in gsi_channel_program()
776 iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_1_OFFSET(channel_id)); in gsi_channel_program()
783 iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_2_OFFSET(channel_id)); in gsi_channel_program()
785 iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_3_OFFSET(channel_id)); in gsi_channel_program()
795 if (gsi->version < IPA_VERSION_4_0 && doorbell) in gsi_channel_program()
801 if (gsi->version >= IPA_VERSION_4_0 && !channel->command) { in gsi_channel_program()
803 if (gsi->version < IPA_VERSION_4_5) in gsi_channel_program()
810 if (gsi->version >= IPA_VERSION_4_9) in gsi_channel_program()
813 iowrite32(val, gsi->virt + GSI_CH_C_QOS_OFFSET(channel_id)); in gsi_channel_program()
822 iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_0_OFFSET(channel_id)); in gsi_channel_program()
825 iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_1_OFFSET(channel_id)); in gsi_channel_program()
828 iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_2_OFFSET(channel_id)); in gsi_channel_program()
834 val = ioread32(gsi->virt + GSI_CH_C_SCRATCH_3_OFFSET(channel_id)); in gsi_channel_program()
836 iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_3_OFFSET(channel_id)); in gsi_channel_program()
843 struct gsi *gsi = channel->gsi; in __gsi_channel_start() local
847 if (resume && gsi->version < IPA_VERSION_4_0) in __gsi_channel_start()
850 mutex_lock(&gsi->mutex); in __gsi_channel_start()
854 mutex_unlock(&gsi->mutex); in __gsi_channel_start()
860 int gsi_channel_start(struct gsi *gsi, u32 channel_id) in gsi_channel_start() argument
862 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_start()
867 gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); in gsi_channel_start()
871 gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); in gsi_channel_start()
895 struct gsi *gsi = channel->gsi; in __gsi_channel_stop() local
902 if (suspend && gsi->version < IPA_VERSION_4_0) in __gsi_channel_stop()
905 mutex_lock(&gsi->mutex); in __gsi_channel_stop()
909 mutex_unlock(&gsi->mutex); in __gsi_channel_stop()
915 int gsi_channel_stop(struct gsi *gsi, u32 channel_id) in gsi_channel_stop() argument
917 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_stop()
925 gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); in gsi_channel_stop()
932 void gsi_channel_reset(struct gsi *gsi, u32 channel_id, bool doorbell) in gsi_channel_reset() argument
934 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_reset()
936 mutex_lock(&gsi->mutex); in gsi_channel_reset()
940 if (gsi->version < IPA_VERSION_4_0 && !channel->toward_ipa) in gsi_channel_reset()
948 mutex_unlock(&gsi->mutex); in gsi_channel_reset()
952 int gsi_channel_suspend(struct gsi *gsi, u32 channel_id) in gsi_channel_suspend() argument
954 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_suspend()
968 int gsi_channel_resume(struct gsi *gsi, u32 channel_id) in gsi_channel_resume() argument
970 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_resume()
976 void gsi_suspend(struct gsi *gsi) in gsi_suspend() argument
978 disable_irq(gsi->irq); in gsi_suspend()
982 void gsi_resume(struct gsi *gsi) in gsi_resume() argument
984 enable_irq(gsi->irq); in gsi_resume()
989 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_tx_committed()
1001 struct gsi *gsi = trans->gsi; in gsi_trans_tx_queued() local
1006 channel = &gsi->channel[channel_id]; in gsi_trans_tx_queued()
1013 ipa_gsi_channel_tx_queued(gsi, channel_id, trans_count, byte_count); in gsi_trans_tx_queued()
1034 struct gsi *gsi = trans->gsi; in gsi_trans_tx_completed() local
1039 channel = &gsi->channel[channel_id]; in gsi_trans_tx_completed()
1046 ipa_gsi_channel_tx_completed(gsi, channel_id, trans_count, byte_count); in gsi_trans_tx_completed()
1050 static void gsi_isr_chan_ctrl(struct gsi *gsi) in gsi_isr_chan_ctrl() argument
1054 channel_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_CH_IRQ_OFFSET); in gsi_isr_chan_ctrl()
1055 iowrite32(channel_mask, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET); in gsi_isr_chan_ctrl()
1062 complete(&gsi->completion); in gsi_isr_chan_ctrl()
1067 static void gsi_isr_evt_ctrl(struct gsi *gsi) in gsi_isr_evt_ctrl() argument
1071 event_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_OFFSET); in gsi_isr_evt_ctrl()
1072 iowrite32(event_mask, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET); in gsi_isr_evt_ctrl()
1079 complete(&gsi->completion); in gsi_isr_evt_ctrl()
1085 gsi_isr_glob_chan_err(struct gsi *gsi, u32 err_ee, u32 channel_id, u32 code) in gsi_isr_glob_chan_err() argument
1088 dev_err(gsi->dev, "channel %u out of resources\n", channel_id); in gsi_isr_glob_chan_err()
1089 complete(&gsi->completion); in gsi_isr_glob_chan_err()
1094 dev_err(gsi->dev, "channel %u global error ee 0x%08x code 0x%08x\n", in gsi_isr_glob_chan_err()
1100 gsi_isr_glob_evt_err(struct gsi *gsi, u32 err_ee, u32 evt_ring_id, u32 code) in gsi_isr_glob_evt_err() argument
1103 struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_isr_glob_evt_err()
1106 complete(&gsi->completion); in gsi_isr_glob_evt_err()
1107 dev_err(gsi->dev, "evt_ring for channel %u out of resources\n", in gsi_isr_glob_evt_err()
1113 dev_err(gsi->dev, "event ring %u global error ee %u code 0x%08x\n", in gsi_isr_glob_evt_err()
1118 static void gsi_isr_glob_err(struct gsi *gsi) in gsi_isr_glob_err() argument
1127 val = ioread32(gsi->virt + GSI_ERROR_LOG_OFFSET); in gsi_isr_glob_err()
1128 iowrite32(0, gsi->virt + GSI_ERROR_LOG_OFFSET); in gsi_isr_glob_err()
1129 iowrite32(~0, gsi->virt + GSI_ERROR_LOG_CLR_OFFSET); in gsi_isr_glob_err()
1137 gsi_isr_glob_chan_err(gsi, ee, which, code); in gsi_isr_glob_err()
1139 gsi_isr_glob_evt_err(gsi, ee, which, code); in gsi_isr_glob_err()
1141 dev_err(gsi->dev, "unexpected global error 0x%08x\n", type); in gsi_isr_glob_err()
1145 static void gsi_isr_gp_int1(struct gsi *gsi) in gsi_isr_gp_int1() argument
1169 val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); in gsi_isr_gp_int1()
1175 gsi->result = 0; in gsi_isr_gp_int1()
1179 gsi->result = -EAGAIN; in gsi_isr_gp_int1()
1183 dev_err(gsi->dev, "global INT1 generic result %u\n", result); in gsi_isr_gp_int1()
1184 gsi->result = -EIO; in gsi_isr_gp_int1()
1188 complete(&gsi->completion); in gsi_isr_gp_int1()
1192 static void gsi_isr_glob_ee(struct gsi *gsi) in gsi_isr_glob_ee() argument
1196 val = ioread32(gsi->virt + GSI_CNTXT_GLOB_IRQ_STTS_OFFSET); in gsi_isr_glob_ee()
1199 gsi_isr_glob_err(gsi); in gsi_isr_glob_ee()
1201 iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_CLR_OFFSET); in gsi_isr_glob_ee()
1207 gsi_isr_gp_int1(gsi); in gsi_isr_glob_ee()
1211 dev_err(gsi->dev, "unexpected global interrupt 0x%08x\n", val); in gsi_isr_glob_ee()
1215 static void gsi_isr_ieob(struct gsi *gsi) in gsi_isr_ieob() argument
1219 event_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_OFFSET); in gsi_isr_ieob()
1220 gsi_irq_ieob_disable(gsi, event_mask); in gsi_isr_ieob()
1221 iowrite32(event_mask, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET); in gsi_isr_ieob()
1228 napi_schedule(&gsi->evt_ring[evt_ring_id].channel->napi); in gsi_isr_ieob()
1233 static void gsi_isr_general(struct gsi *gsi) in gsi_isr_general() argument
1235 struct device *dev = gsi->dev; in gsi_isr_general()
1238 val = ioread32(gsi->virt + GSI_CNTXT_GSI_IRQ_STTS_OFFSET); in gsi_isr_general()
1239 iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_CLR_OFFSET); in gsi_isr_general()
1254 struct gsi *gsi = dev_id; in gsi_isr() local
1259 while ((intr_mask = ioread32(gsi->virt + GSI_CNTXT_TYPE_IRQ_OFFSET))) { in gsi_isr()
1268 gsi_isr_chan_ctrl(gsi); in gsi_isr()
1271 gsi_isr_evt_ctrl(gsi); in gsi_isr()
1274 gsi_isr_glob_ee(gsi); in gsi_isr()
1277 gsi_isr_ieob(gsi); in gsi_isr()
1280 gsi_isr_general(gsi); in gsi_isr()
1283 dev_err(gsi->dev, in gsi_isr()
1291 dev_err(gsi->dev, "interrupt flood\n"); in gsi_isr()
1300 static int gsi_irq_init(struct gsi *gsi, struct platform_device *pdev) in gsi_irq_init() argument
1308 gsi->irq = ret; in gsi_irq_init()
1315 gsi_event_trans(struct gsi *gsi, struct gsi_event *event) in gsi_event_trans() argument
1323 channel = &gsi->channel[channel_id]; in gsi_event_trans()
1324 if (WARN(!channel->gsi, "event has bad channel %u\n", channel_id)) in gsi_event_trans()
1363 static void gsi_evt_ring_update(struct gsi *gsi, u32 evt_ring_id, u32 index) in gsi_evt_ring_update() argument
1365 struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_evt_ring_update()
1390 trans = gsi_event_trans(gsi, event); in gsi_evt_ring_update()
1409 gsi_evt_ring_doorbell(gsi, evt_ring_id, index); in gsi_evt_ring_update()
1413 static int gsi_ring_alloc(struct gsi *gsi, struct gsi_ring *ring, u32 count) in gsi_ring_alloc() argument
1416 struct device *dev = gsi->dev; in gsi_ring_alloc()
1435 static void gsi_ring_free(struct gsi *gsi, struct gsi_ring *ring) in gsi_ring_free() argument
1439 dma_free_coherent(gsi->dev, size, ring->virt, ring->addr); in gsi_ring_free()
1443 static int gsi_evt_ring_id_alloc(struct gsi *gsi) in gsi_evt_ring_id_alloc() argument
1447 if (gsi->event_bitmap == ~0U) { in gsi_evt_ring_id_alloc()
1448 dev_err(gsi->dev, "event rings exhausted\n"); in gsi_evt_ring_id_alloc()
1452 evt_ring_id = ffz(gsi->event_bitmap); in gsi_evt_ring_id_alloc()
1453 gsi->event_bitmap |= BIT(evt_ring_id); in gsi_evt_ring_id_alloc()
1459 static void gsi_evt_ring_id_free(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_id_free() argument
1461 gsi->event_bitmap &= ~BIT(evt_ring_id); in gsi_evt_ring_id_free()
1469 struct gsi *gsi = channel->gsi; in gsi_channel_doorbell() local
1474 iowrite32(val, gsi->virt + GSI_CH_C_DOORBELL_0_OFFSET(channel_id)); in gsi_channel_doorbell()
1481 struct gsi *gsi = channel->gsi; in gsi_channel_update() local
1488 evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_channel_update()
1495 index = gsi_ring_index(ring, ioread32(gsi->virt + offset)); in gsi_channel_update()
1500 trans = gsi_event_trans(gsi, gsi_ring_virt(ring, index - 1)); in gsi_channel_update()
1509 gsi_evt_ring_update(gsi, evt_ring_id, index); in gsi_channel_update()
1564 gsi_irq_ieob_enable_one(channel->gsi, channel->evt_ring_id); in gsi_channel_poll()
1584 static int gsi_channel_setup_one(struct gsi *gsi, u32 channel_id) in gsi_channel_setup_one() argument
1586 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_setup_one()
1593 ret = gsi_evt_ring_alloc_command(gsi, evt_ring_id); in gsi_channel_setup_one()
1597 gsi_evt_ring_program(gsi, evt_ring_id); in gsi_channel_setup_one()
1599 ret = gsi_channel_alloc_command(gsi, channel_id); in gsi_channel_setup_one()
1606 netif_napi_add_tx(&gsi->dummy_dev, &channel->napi, in gsi_channel_setup_one()
1609 netif_napi_add(&gsi->dummy_dev, &channel->napi, in gsi_channel_setup_one()
1616 gsi_evt_ring_de_alloc_command(gsi, evt_ring_id); in gsi_channel_setup_one()
1622 static void gsi_channel_teardown_one(struct gsi *gsi, u32 channel_id) in gsi_channel_teardown_one() argument
1624 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_teardown_one()
1632 gsi_channel_de_alloc_command(gsi, channel_id); in gsi_channel_teardown_one()
1633 gsi_evt_ring_reset_command(gsi, evt_ring_id); in gsi_channel_teardown_one()
1634 gsi_evt_ring_de_alloc_command(gsi, evt_ring_id); in gsi_channel_teardown_one()
1641 static int gsi_generic_command(struct gsi *gsi, u32 channel_id, in gsi_generic_command() argument
1658 iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); in gsi_generic_command()
1661 val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); in gsi_generic_command()
1663 iowrite32(val, gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); in gsi_generic_command()
1671 timeout = !gsi_command(gsi, GSI_GENERIC_CMD_OFFSET, val); in gsi_generic_command()
1674 iowrite32(BIT(ERROR_INT), gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); in gsi_generic_command()
1677 return gsi->result; in gsi_generic_command()
1679 dev_err(gsi->dev, "GSI generic command %u to channel %u timed out\n", in gsi_generic_command()
1685 static int gsi_modem_channel_alloc(struct gsi *gsi, u32 channel_id) in gsi_modem_channel_alloc() argument
1687 return gsi_generic_command(gsi, channel_id, in gsi_modem_channel_alloc()
1691 static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id) in gsi_modem_channel_halt() argument
1697 ret = gsi_generic_command(gsi, channel_id, in gsi_modem_channel_halt()
1702 dev_err(gsi->dev, "error %d halting modem channel %u\n", in gsi_modem_channel_halt()
1708 gsi_modem_channel_flow_control(struct gsi *gsi, u32 channel_id, bool enable) in gsi_modem_channel_flow_control() argument
1719 if (!enable && gsi->version >= IPA_VERSION_4_11) in gsi_modem_channel_flow_control()
1723 ret = gsi_generic_command(gsi, channel_id, command, 0); in gsi_modem_channel_flow_control()
1727 dev_err(gsi->dev, in gsi_modem_channel_flow_control()
1733 static int gsi_channel_setup(struct gsi *gsi) in gsi_channel_setup() argument
1739 gsi_irq_enable(gsi); in gsi_channel_setup()
1741 mutex_lock(&gsi->mutex); in gsi_channel_setup()
1744 ret = gsi_channel_setup_one(gsi, channel_id); in gsi_channel_setup()
1747 } while (++channel_id < gsi->channel_count); in gsi_channel_setup()
1751 struct gsi_channel *channel = &gsi->channel[channel_id++]; in gsi_channel_setup()
1757 dev_err(gsi->dev, "channel %u not supported by hardware\n", in gsi_channel_setup()
1759 channel_id = gsi->channel_count; in gsi_channel_setup()
1764 mask = gsi->modem_channel_bitmap; in gsi_channel_setup()
1768 ret = gsi_modem_channel_alloc(gsi, modem_channel_id); in gsi_channel_setup()
1776 mutex_unlock(&gsi->mutex); in gsi_channel_setup()
1782 mask ^= gsi->modem_channel_bitmap; in gsi_channel_setup()
1788 gsi_modem_channel_halt(gsi, channel_id); in gsi_channel_setup()
1793 gsi_channel_teardown_one(gsi, channel_id); in gsi_channel_setup()
1795 mutex_unlock(&gsi->mutex); in gsi_channel_setup()
1797 gsi_irq_disable(gsi); in gsi_channel_setup()
1803 static void gsi_channel_teardown(struct gsi *gsi) in gsi_channel_teardown() argument
1805 u32 mask = gsi->modem_channel_bitmap; in gsi_channel_teardown()
1808 mutex_lock(&gsi->mutex); in gsi_channel_teardown()
1815 gsi_modem_channel_halt(gsi, channel_id); in gsi_channel_teardown()
1818 channel_id = gsi->channel_count - 1; in gsi_channel_teardown()
1820 gsi_channel_teardown_one(gsi, channel_id); in gsi_channel_teardown()
1823 mutex_unlock(&gsi->mutex); in gsi_channel_teardown()
1825 gsi_irq_disable(gsi); in gsi_channel_teardown()
1829 static int gsi_irq_setup(struct gsi *gsi) in gsi_irq_setup() argument
1834 iowrite32(1, gsi->virt + GSI_CNTXT_INTSET_OFFSET); in gsi_irq_setup()
1837 gsi_irq_type_update(gsi, 0); in gsi_irq_setup()
1840 iowrite32(0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET); in gsi_irq_setup()
1841 iowrite32(0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET); in gsi_irq_setup()
1842 iowrite32(0, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); in gsi_irq_setup()
1843 iowrite32(0, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); in gsi_irq_setup()
1846 if (gsi->version > IPA_VERSION_3_1) { in gsi_irq_setup()
1851 iowrite32(0, gsi->virt_raw + offset); in gsi_irq_setup()
1853 iowrite32(0, gsi->virt_raw + offset); in gsi_irq_setup()
1856 iowrite32(0, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); in gsi_irq_setup()
1858 ret = request_irq(gsi->irq, gsi_isr, 0, "gsi", gsi); in gsi_irq_setup()
1860 dev_err(gsi->dev, "error %d requesting \"gsi\" IRQ\n", ret); in gsi_irq_setup()
1865 static void gsi_irq_teardown(struct gsi *gsi) in gsi_irq_teardown() argument
1867 free_irq(gsi->irq, gsi); in gsi_irq_teardown()
1871 static int gsi_ring_setup(struct gsi *gsi) in gsi_ring_setup() argument
1873 struct device *dev = gsi->dev; in gsi_ring_setup()
1877 if (gsi->version < IPA_VERSION_3_5_1) { in gsi_ring_setup()
1879 gsi->channel_count = GSI_CHANNEL_COUNT_MAX; in gsi_ring_setup()
1880 gsi->evt_ring_count = GSI_EVT_RING_COUNT_MAX; in gsi_ring_setup()
1885 val = ioread32(gsi->virt + GSI_GSI_HW_PARAM_2_OFFSET); in gsi_ring_setup()
1897 gsi->channel_count = count; in gsi_ring_setup()
1910 gsi->evt_ring_count = count; in gsi_ring_setup()
1916 int gsi_setup(struct gsi *gsi) in gsi_setup() argument
1922 val = ioread32(gsi->virt + GSI_GSI_STATUS_OFFSET); in gsi_setup()
1924 dev_err(gsi->dev, "GSI has not been enabled\n"); in gsi_setup()
1928 ret = gsi_irq_setup(gsi); in gsi_setup()
1932 ret = gsi_ring_setup(gsi); /* No matching teardown required */ in gsi_setup()
1937 iowrite32(0, gsi->virt + GSI_ERROR_LOG_OFFSET); in gsi_setup()
1939 ret = gsi_channel_setup(gsi); in gsi_setup()
1946 gsi_irq_teardown(gsi); in gsi_setup()
1952 void gsi_teardown(struct gsi *gsi) in gsi_teardown() argument
1954 gsi_channel_teardown(gsi); in gsi_teardown()
1955 gsi_irq_teardown(gsi); in gsi_teardown()
1961 struct gsi *gsi = channel->gsi; in gsi_channel_evt_ring_init() local
1965 ret = gsi_evt_ring_id_alloc(gsi); in gsi_channel_evt_ring_init()
1970 evt_ring = &gsi->evt_ring[channel->evt_ring_id]; in gsi_channel_evt_ring_init()
1973 ret = gsi_ring_alloc(gsi, &evt_ring->ring, channel->event_count); in gsi_channel_evt_ring_init()
1977 dev_err(gsi->dev, "error %d allocating channel %u event ring\n", in gsi_channel_evt_ring_init()
1980 gsi_evt_ring_id_free(gsi, channel->evt_ring_id); in gsi_channel_evt_ring_init()
1989 struct gsi *gsi = channel->gsi; in gsi_channel_evt_ring_exit() local
1992 evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_channel_evt_ring_exit()
1993 gsi_ring_free(gsi, &evt_ring->ring); in gsi_channel_evt_ring_exit()
1994 gsi_evt_ring_id_free(gsi, evt_ring_id); in gsi_channel_evt_ring_exit()
1997 static bool gsi_channel_data_valid(struct gsi *gsi, bool command, in gsi_channel_data_valid() argument
2002 struct device *dev = gsi->dev; in gsi_channel_data_valid()
2065 static int gsi_channel_init_one(struct gsi *gsi, in gsi_channel_init_one() argument
2073 if (!gsi_channel_data_valid(gsi, command, data)) in gsi_channel_init_one()
2079 dev_warn(gsi->dev, "channel %u limited to %u TREs\n", in gsi_channel_init_one()
2085 channel = &gsi->channel[data->channel_id]; in gsi_channel_init_one()
2088 channel->gsi = gsi; in gsi_channel_init_one()
2099 ret = gsi_ring_alloc(gsi, &channel->tre_ring, data->channel.tre_count); in gsi_channel_init_one()
2101 dev_err(gsi->dev, "error %d allocating channel %u ring\n", in gsi_channel_init_one()
2106 ret = gsi_channel_trans_init(gsi, data->channel_id); in gsi_channel_init_one()
2111 u32 tre_max = gsi_channel_tre_max(gsi, data->channel_id); in gsi_channel_init_one()
2120 gsi_ring_free(gsi, &channel->tre_ring); in gsi_channel_init_one()
2124 channel->gsi = NULL; /* Mark it not (fully) initialized */ in gsi_channel_init_one()
2138 gsi_ring_free(channel->gsi, &channel->tre_ring); in gsi_channel_exit_one()
2143 static int gsi_channel_init(struct gsi *gsi, u32 count, in gsi_channel_init() argument
2151 modem_alloc = gsi->version == IPA_VERSION_4_2; in gsi_channel_init()
2153 gsi->event_bitmap = gsi_event_bitmap_init(GSI_EVT_RING_COUNT_MAX); in gsi_channel_init()
2154 gsi->ieob_enabled_bitmap = 0; in gsi_channel_init()
2166 gsi->modem_channel_bitmap |= in gsi_channel_init()
2171 ret = gsi_channel_init_one(gsi, &data[i], command); in gsi_channel_init()
2183 gsi->modem_channel_bitmap &= ~BIT(data[i].channel_id); in gsi_channel_init()
2186 gsi_channel_exit_one(&gsi->channel[data->channel_id]); in gsi_channel_init()
2193 static void gsi_channel_exit(struct gsi *gsi) in gsi_channel_exit() argument
2198 gsi_channel_exit_one(&gsi->channel[channel_id]); in gsi_channel_exit()
2200 gsi->modem_channel_bitmap = 0; in gsi_channel_exit()
2204 int gsi_init(struct gsi *gsi, struct platform_device *pdev, in gsi_init() argument
2216 gsi->dev = dev; in gsi_init()
2217 gsi->version = version; in gsi_init()
2222 init_dummy_netdev(&gsi->dummy_dev); in gsi_init()
2238 adjust = gsi->version < IPA_VERSION_4_5 ? 0 : GSI_EE_REG_ADJUST; in gsi_init()
2245 gsi->virt_raw = ioremap(res->start, size); in gsi_init()
2246 if (!gsi->virt_raw) { in gsi_init()
2251 gsi->virt = gsi->virt_raw - adjust; in gsi_init()
2253 init_completion(&gsi->completion); in gsi_init()
2255 ret = gsi_irq_init(gsi, pdev); /* No matching exit required */ in gsi_init()
2259 ret = gsi_channel_init(gsi, count, data); in gsi_init()
2263 mutex_init(&gsi->mutex); in gsi_init()
2268 iounmap(gsi->virt_raw); in gsi_init()
2274 void gsi_exit(struct gsi *gsi) in gsi_exit() argument
2276 mutex_destroy(&gsi->mutex); in gsi_exit()
2277 gsi_channel_exit(gsi); in gsi_exit()
2278 iounmap(gsi->virt_raw); in gsi_exit()
2301 u32 gsi_channel_tre_max(struct gsi *gsi, u32 channel_id) in gsi_channel_tre_max() argument
2303 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_tre_max()