Lines Matching refs:gsi
171 return channel - &channel->gsi->channel[0]; in gsi_channel_id()
177 return !!channel->gsi; in gsi_channel_initialized()
197 static void gsi_irq_type_update(struct gsi *gsi, u32 val) in gsi_irq_type_update() argument
199 const struct reg *reg = gsi_reg(gsi, CNTXT_TYPE_IRQ_MSK); in gsi_irq_type_update()
201 gsi->type_enabled_bitmap = val; in gsi_irq_type_update()
202 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_irq_type_update()
205 static void gsi_irq_type_enable(struct gsi *gsi, enum gsi_irq_type_id type_id) in gsi_irq_type_enable() argument
207 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | type_id); in gsi_irq_type_enable()
210 static void gsi_irq_type_disable(struct gsi *gsi, enum gsi_irq_type_id type_id) in gsi_irq_type_disable() argument
212 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap & ~type_id); in gsi_irq_type_disable()
220 static void gsi_irq_ev_ctrl_enable(struct gsi *gsi, u32 evt_ring_id) in gsi_irq_ev_ctrl_enable() argument
229 reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_CLR); in gsi_irq_ev_ctrl_enable()
230 iowrite32(~0, gsi->virt + reg_offset(reg)); in gsi_irq_ev_ctrl_enable()
232 reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK); in gsi_irq_ev_ctrl_enable()
233 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_irq_ev_ctrl_enable()
234 gsi_irq_type_enable(gsi, GSI_EV_CTRL); in gsi_irq_ev_ctrl_enable()
238 static void gsi_irq_ev_ctrl_disable(struct gsi *gsi) in gsi_irq_ev_ctrl_disable() argument
242 gsi_irq_type_disable(gsi, GSI_EV_CTRL); in gsi_irq_ev_ctrl_disable()
244 reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK); in gsi_irq_ev_ctrl_disable()
245 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_ev_ctrl_disable()
253 static void gsi_irq_ch_ctrl_enable(struct gsi *gsi, u32 channel_id) in gsi_irq_ch_ctrl_enable() argument
262 reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_CLR); in gsi_irq_ch_ctrl_enable()
263 iowrite32(~0, gsi->virt + reg_offset(reg)); in gsi_irq_ch_ctrl_enable()
265 reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK); in gsi_irq_ch_ctrl_enable()
266 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_irq_ch_ctrl_enable()
268 gsi_irq_type_enable(gsi, GSI_CH_CTRL); in gsi_irq_ch_ctrl_enable()
272 static void gsi_irq_ch_ctrl_disable(struct gsi *gsi) in gsi_irq_ch_ctrl_disable() argument
276 gsi_irq_type_disable(gsi, GSI_CH_CTRL); in gsi_irq_ch_ctrl_disable()
278 reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK); in gsi_irq_ch_ctrl_disable()
279 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_ch_ctrl_disable()
282 static void gsi_irq_ieob_enable_one(struct gsi *gsi, u32 evt_ring_id) in gsi_irq_ieob_enable_one() argument
284 bool enable_ieob = !gsi->ieob_enabled_bitmap; in gsi_irq_ieob_enable_one()
288 gsi->ieob_enabled_bitmap |= BIT(evt_ring_id); in gsi_irq_ieob_enable_one()
290 reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK); in gsi_irq_ieob_enable_one()
291 val = gsi->ieob_enabled_bitmap; in gsi_irq_ieob_enable_one()
292 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_irq_ieob_enable_one()
296 gsi_irq_type_enable(gsi, GSI_IEOB); in gsi_irq_ieob_enable_one()
299 static void gsi_irq_ieob_disable(struct gsi *gsi, u32 event_mask) in gsi_irq_ieob_disable() argument
304 gsi->ieob_enabled_bitmap &= ~event_mask; in gsi_irq_ieob_disable()
307 if (!gsi->ieob_enabled_bitmap) in gsi_irq_ieob_disable()
308 gsi_irq_type_disable(gsi, GSI_IEOB); in gsi_irq_ieob_disable()
310 reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK); in gsi_irq_ieob_disable()
311 val = gsi->ieob_enabled_bitmap; in gsi_irq_ieob_disable()
312 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_irq_ieob_disable()
315 static void gsi_irq_ieob_disable_one(struct gsi *gsi, u32 evt_ring_id) in gsi_irq_ieob_disable_one() argument
317 gsi_irq_ieob_disable(gsi, BIT(evt_ring_id)); in gsi_irq_ieob_disable_one()
321 static void gsi_irq_enable(struct gsi *gsi) in gsi_irq_enable() argument
329 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); in gsi_irq_enable()
330 iowrite32(ERROR_INT, gsi->virt + reg_offset(reg)); in gsi_irq_enable()
332 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | GSI_GLOB_EE); in gsi_irq_enable()
339 reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN); in gsi_irq_enable()
343 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_irq_enable()
345 gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | GSI_GENERAL); in gsi_irq_enable()
349 static void gsi_irq_disable(struct gsi *gsi) in gsi_irq_disable() argument
353 gsi_irq_type_update(gsi, 0); in gsi_irq_disable()
356 reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN); in gsi_irq_disable()
357 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_disable()
359 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); in gsi_irq_disable()
360 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_disable()
386 static bool gsi_command(struct gsi *gsi, u32 reg, u32 val) in gsi_command() argument
389 struct completion *completion = &gsi->completion; in gsi_command()
393 iowrite32(val, gsi->virt + reg); in gsi_command()
400 gsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_state() argument
402 const struct reg *reg = gsi_reg(gsi, EV_CH_E_CNTXT_0); in gsi_evt_ring_state()
405 val = ioread32(gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_state()
411 static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id, in gsi_evt_ring_command() argument
414 struct device *dev = gsi->dev; in gsi_evt_ring_command()
420 gsi_irq_ev_ctrl_enable(gsi, evt_ring_id); in gsi_evt_ring_command()
422 reg = gsi_reg(gsi, EV_CH_CMD); in gsi_evt_ring_command()
426 timeout = !gsi_command(gsi, reg_offset(reg), val); in gsi_evt_ring_command()
428 gsi_irq_ev_ctrl_disable(gsi); in gsi_evt_ring_command()
434 opcode, evt_ring_id, gsi_evt_ring_state(gsi, evt_ring_id)); in gsi_evt_ring_command()
438 static int gsi_evt_ring_alloc_command(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_alloc_command() argument
443 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_alloc_command()
445 dev_err(gsi->dev, "event ring %u bad state %u before alloc\n", in gsi_evt_ring_alloc_command()
450 gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_ALLOCATE); in gsi_evt_ring_alloc_command()
453 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_alloc_command()
457 dev_err(gsi->dev, "event ring %u bad state %u after alloc\n", in gsi_evt_ring_alloc_command()
464 static void gsi_evt_ring_reset_command(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_reset_command() argument
468 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_reset_command()
471 dev_err(gsi->dev, "event ring %u bad state %u before reset\n", in gsi_evt_ring_reset_command()
476 gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_RESET); in gsi_evt_ring_reset_command()
479 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_reset_command()
483 dev_err(gsi->dev, "event ring %u bad state %u after reset\n", in gsi_evt_ring_reset_command()
488 static void gsi_evt_ring_de_alloc_command(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_de_alloc_command() argument
492 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_de_alloc_command()
494 dev_err(gsi->dev, "event ring %u state %u before dealloc\n", in gsi_evt_ring_de_alloc_command()
499 gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_DE_ALLOC); in gsi_evt_ring_de_alloc_command()
502 state = gsi_evt_ring_state(gsi, evt_ring_id); in gsi_evt_ring_de_alloc_command()
506 dev_err(gsi->dev, "event ring %u bad state %u after dealloc\n", in gsi_evt_ring_de_alloc_command()
513 const struct reg *reg = gsi_reg(channel->gsi, CH_C_CNTXT_0); in gsi_channel_state()
515 struct gsi *gsi = channel->gsi; in gsi_channel_state() local
516 void __iomem *virt = gsi->virt; in gsi_channel_state()
519 reg = gsi_reg(gsi, CH_C_CNTXT_0); in gsi_channel_state()
530 struct gsi *gsi = channel->gsi; in gsi_channel_command() local
531 struct device *dev = gsi->dev; in gsi_channel_command()
537 gsi_irq_ch_ctrl_enable(gsi, channel_id); in gsi_channel_command()
539 reg = gsi_reg(gsi, CH_CMD); in gsi_channel_command()
543 timeout = !gsi_command(gsi, reg_offset(reg), val); in gsi_channel_command()
545 gsi_irq_ch_ctrl_disable(gsi); in gsi_channel_command()
555 static int gsi_channel_alloc_command(struct gsi *gsi, u32 channel_id) in gsi_channel_alloc_command() argument
557 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_alloc_command()
558 struct device *dev = gsi->dev; in gsi_channel_alloc_command()
585 struct device *dev = channel->gsi->dev; in gsi_channel_start_command()
612 struct device *dev = channel->gsi->dev; in gsi_channel_stop_command()
650 struct device *dev = channel->gsi->dev; in gsi_channel_reset_command()
676 static void gsi_channel_de_alloc_command(struct gsi *gsi, u32 channel_id) in gsi_channel_de_alloc_command() argument
678 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_de_alloc_command()
679 struct device *dev = gsi->dev; in gsi_channel_de_alloc_command()
704 static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index) in gsi_evt_ring_doorbell() argument
706 const struct reg *reg = gsi_reg(gsi, EV_CH_E_DOORBELL_0); in gsi_evt_ring_doorbell()
707 struct gsi_ring *ring = &gsi->evt_ring[evt_ring_id].ring; in gsi_evt_ring_doorbell()
714 iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_doorbell()
718 static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_program() argument
720 struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_evt_ring_program()
725 reg = gsi_reg(gsi, EV_CH_E_CNTXT_0); in gsi_evt_ring_program()
731 iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
733 reg = gsi_reg(gsi, EV_CH_E_CNTXT_1); in gsi_evt_ring_program()
735 iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
741 reg = gsi_reg(gsi, EV_CH_E_CNTXT_2); in gsi_evt_ring_program()
743 iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
745 reg = gsi_reg(gsi, EV_CH_E_CNTXT_3); in gsi_evt_ring_program()
747 iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
750 reg = gsi_reg(gsi, EV_CH_E_CNTXT_8); in gsi_evt_ring_program()
754 iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
757 reg = gsi_reg(gsi, EV_CH_E_CNTXT_9); in gsi_evt_ring_program()
758 iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
760 reg = gsi_reg(gsi, EV_CH_E_CNTXT_10); in gsi_evt_ring_program()
761 iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
763 reg = gsi_reg(gsi, EV_CH_E_CNTXT_11); in gsi_evt_ring_program()
764 iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
767 reg = gsi_reg(gsi, EV_CH_E_CNTXT_12); in gsi_evt_ring_program()
768 iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
770 reg = gsi_reg(gsi, EV_CH_E_CNTXT_13); in gsi_evt_ring_program()
771 iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); in gsi_evt_ring_program()
774 gsi_evt_ring_doorbell(gsi, evt_ring_id, ring->index); in gsi_evt_ring_program()
831 struct gsi *gsi = channel->gsi; in gsi_channel_program() local
837 reg = gsi_reg(gsi, CH_C_CNTXT_0); in gsi_channel_program()
840 val = ch_c_cntxt_0_type_encode(gsi->version, reg, GSI_CHANNEL_TYPE_GPI); in gsi_channel_program()
843 if (gsi->version < IPA_VERSION_5_0) in gsi_channel_program()
846 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
848 reg = gsi_reg(gsi, CH_C_CNTXT_1); in gsi_channel_program()
850 if (gsi->version >= IPA_VERSION_5_0) in gsi_channel_program()
852 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
858 reg = gsi_reg(gsi, CH_C_CNTXT_2); in gsi_channel_program()
860 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
862 reg = gsi_reg(gsi, CH_C_CNTXT_3); in gsi_channel_program()
864 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
866 reg = gsi_reg(gsi, CH_C_QOS); in gsi_channel_program()
876 if (gsi->version < IPA_VERSION_4_0 && doorbell) in gsi_channel_program()
882 if (gsi->version >= IPA_VERSION_4_0 && !channel->command) { in gsi_channel_program()
884 if (gsi->version < IPA_VERSION_4_5) in gsi_channel_program()
890 if (gsi->version >= IPA_VERSION_4_9) in gsi_channel_program()
893 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
901 reg = gsi_reg(gsi, CH_C_SCRATCH_0); in gsi_channel_program()
903 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
905 reg = gsi_reg(gsi, CH_C_SCRATCH_1); in gsi_channel_program()
907 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
909 reg = gsi_reg(gsi, CH_C_SCRATCH_2); in gsi_channel_program()
911 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_program()
917 reg = gsi_reg(gsi, CH_C_SCRATCH_3); in gsi_channel_program()
919 val = ioread32(gsi->virt + offset); in gsi_channel_program()
921 iowrite32(val, gsi->virt + offset); in gsi_channel_program()
928 struct gsi *gsi = channel->gsi; in __gsi_channel_start() local
932 if (resume && gsi->version < IPA_VERSION_4_0) in __gsi_channel_start()
935 mutex_lock(&gsi->mutex); in __gsi_channel_start()
939 mutex_unlock(&gsi->mutex); in __gsi_channel_start()
945 int gsi_channel_start(struct gsi *gsi, u32 channel_id) in gsi_channel_start() argument
947 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_start()
952 gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); in gsi_channel_start()
956 gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); in gsi_channel_start()
980 struct gsi *gsi = channel->gsi; in __gsi_channel_stop() local
987 if (suspend && gsi->version < IPA_VERSION_4_0) in __gsi_channel_stop()
990 mutex_lock(&gsi->mutex); in __gsi_channel_stop()
994 mutex_unlock(&gsi->mutex); in __gsi_channel_stop()
1000 int gsi_channel_stop(struct gsi *gsi, u32 channel_id) in gsi_channel_stop() argument
1002 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_stop()
1010 gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); in gsi_channel_stop()
1017 void gsi_channel_reset(struct gsi *gsi, u32 channel_id, bool doorbell) in gsi_channel_reset() argument
1019 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_reset()
1021 mutex_lock(&gsi->mutex); in gsi_channel_reset()
1025 if (gsi->version < IPA_VERSION_4_0 && !channel->toward_ipa) in gsi_channel_reset()
1033 mutex_unlock(&gsi->mutex); in gsi_channel_reset()
1037 int gsi_channel_suspend(struct gsi *gsi, u32 channel_id) in gsi_channel_suspend() argument
1039 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_suspend()
1053 int gsi_channel_resume(struct gsi *gsi, u32 channel_id) in gsi_channel_resume() argument
1055 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_resume()
1061 void gsi_suspend(struct gsi *gsi) in gsi_suspend() argument
1063 disable_irq(gsi->irq); in gsi_suspend()
1067 void gsi_resume(struct gsi *gsi) in gsi_resume() argument
1069 enable_irq(gsi->irq); in gsi_resume()
1074 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_tx_committed()
1086 struct gsi *gsi = trans->gsi; in gsi_trans_tx_queued() local
1091 channel = &gsi->channel[channel_id]; in gsi_trans_tx_queued()
1098 ipa_gsi_channel_tx_queued(gsi, channel_id, trans_count, byte_count); in gsi_trans_tx_queued()
1119 struct gsi *gsi = trans->gsi; in gsi_trans_tx_completed() local
1124 channel = &gsi->channel[channel_id]; in gsi_trans_tx_completed()
1131 ipa_gsi_channel_tx_completed(gsi, channel_id, trans_count, byte_count); in gsi_trans_tx_completed()
1135 static void gsi_isr_chan_ctrl(struct gsi *gsi) in gsi_isr_chan_ctrl() argument
1140 reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ); in gsi_isr_chan_ctrl()
1141 channel_mask = ioread32(gsi->virt + reg_offset(reg)); in gsi_isr_chan_ctrl()
1143 reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_CLR); in gsi_isr_chan_ctrl()
1144 iowrite32(channel_mask, gsi->virt + reg_offset(reg)); in gsi_isr_chan_ctrl()
1151 complete(&gsi->completion); in gsi_isr_chan_ctrl()
1156 static void gsi_isr_evt_ctrl(struct gsi *gsi) in gsi_isr_evt_ctrl() argument
1161 reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ); in gsi_isr_evt_ctrl()
1162 event_mask = ioread32(gsi->virt + reg_offset(reg)); in gsi_isr_evt_ctrl()
1164 reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_CLR); in gsi_isr_evt_ctrl()
1165 iowrite32(event_mask, gsi->virt + reg_offset(reg)); in gsi_isr_evt_ctrl()
1172 complete(&gsi->completion); in gsi_isr_evt_ctrl()
1178 gsi_isr_glob_chan_err(struct gsi *gsi, u32 err_ee, u32 channel_id, u32 code) in gsi_isr_glob_chan_err() argument
1181 dev_err(gsi->dev, "channel %u out of resources\n", channel_id); in gsi_isr_glob_chan_err()
1182 complete(&gsi->completion); in gsi_isr_glob_chan_err()
1187 dev_err(gsi->dev, "channel %u global error ee 0x%08x code 0x%08x\n", in gsi_isr_glob_chan_err()
1193 gsi_isr_glob_evt_err(struct gsi *gsi, u32 err_ee, u32 evt_ring_id, u32 code) in gsi_isr_glob_evt_err() argument
1196 struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_isr_glob_evt_err()
1199 complete(&gsi->completion); in gsi_isr_glob_evt_err()
1200 dev_err(gsi->dev, "evt_ring for channel %u out of resources\n", in gsi_isr_glob_evt_err()
1206 dev_err(gsi->dev, "event ring %u global error ee %u code 0x%08x\n", in gsi_isr_glob_evt_err()
1211 static void gsi_isr_glob_err(struct gsi *gsi) in gsi_isr_glob_err() argument
1223 log_reg = gsi_reg(gsi, ERROR_LOG); in gsi_isr_glob_err()
1225 val = ioread32(gsi->virt + offset); in gsi_isr_glob_err()
1226 iowrite32(0, gsi->virt + offset); in gsi_isr_glob_err()
1228 clr_reg = gsi_reg(gsi, ERROR_LOG_CLR); in gsi_isr_glob_err()
1229 iowrite32(~0, gsi->virt + reg_offset(clr_reg)); in gsi_isr_glob_err()
1238 gsi_isr_glob_chan_err(gsi, ee, which, code); in gsi_isr_glob_err()
1240 gsi_isr_glob_evt_err(gsi, ee, which, code); in gsi_isr_glob_err()
1242 dev_err(gsi->dev, "unexpected global error 0x%08x\n", type); in gsi_isr_glob_err()
1246 static void gsi_isr_gp_int1(struct gsi *gsi) in gsi_isr_gp_int1() argument
1271 reg = gsi_reg(gsi, CNTXT_SCRATCH_0); in gsi_isr_gp_int1()
1272 val = ioread32(gsi->virt + reg_offset(reg)); in gsi_isr_gp_int1()
1278 gsi->result = 0; in gsi_isr_gp_int1()
1282 gsi->result = -EAGAIN; in gsi_isr_gp_int1()
1286 dev_err(gsi->dev, "global INT1 generic result %u\n", result); in gsi_isr_gp_int1()
1287 gsi->result = -EIO; in gsi_isr_gp_int1()
1291 complete(&gsi->completion); in gsi_isr_gp_int1()
1295 static void gsi_isr_glob_ee(struct gsi *gsi) in gsi_isr_glob_ee() argument
1300 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_STTS); in gsi_isr_glob_ee()
1301 val = ioread32(gsi->virt + reg_offset(reg)); in gsi_isr_glob_ee()
1304 gsi_isr_glob_err(gsi); in gsi_isr_glob_ee()
1306 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_CLR); in gsi_isr_glob_ee()
1307 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_isr_glob_ee()
1313 gsi_isr_gp_int1(gsi); in gsi_isr_glob_ee()
1317 dev_err(gsi->dev, "unexpected global interrupt 0x%08x\n", val); in gsi_isr_glob_ee()
1321 static void gsi_isr_ieob(struct gsi *gsi) in gsi_isr_ieob() argument
1326 reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ); in gsi_isr_ieob()
1327 event_mask = ioread32(gsi->virt + reg_offset(reg)); in gsi_isr_ieob()
1329 gsi_irq_ieob_disable(gsi, event_mask); in gsi_isr_ieob()
1331 reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_CLR); in gsi_isr_ieob()
1332 iowrite32(event_mask, gsi->virt + reg_offset(reg)); in gsi_isr_ieob()
1339 napi_schedule(&gsi->evt_ring[evt_ring_id].channel->napi); in gsi_isr_ieob()
1344 static void gsi_isr_general(struct gsi *gsi) in gsi_isr_general() argument
1346 struct device *dev = gsi->dev; in gsi_isr_general()
1350 reg = gsi_reg(gsi, CNTXT_GSI_IRQ_STTS); in gsi_isr_general()
1351 val = ioread32(gsi->virt + reg_offset(reg)); in gsi_isr_general()
1353 reg = gsi_reg(gsi, CNTXT_GSI_IRQ_CLR); in gsi_isr_general()
1354 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_isr_general()
1369 struct gsi *gsi = dev_id; in gsi_isr() local
1375 reg = gsi_reg(gsi, CNTXT_TYPE_IRQ); in gsi_isr()
1379 while ((intr_mask = ioread32(gsi->virt + offset))) { in gsi_isr()
1391 gsi_isr_chan_ctrl(gsi); in gsi_isr()
1394 gsi_isr_evt_ctrl(gsi); in gsi_isr()
1397 gsi_isr_glob_ee(gsi); in gsi_isr()
1400 gsi_isr_ieob(gsi); in gsi_isr()
1403 gsi_isr_general(gsi); in gsi_isr()
1406 dev_err(gsi->dev, in gsi_isr()
1414 dev_err(gsi->dev, "interrupt flood\n"); in gsi_isr()
1423 static int gsi_irq_init(struct gsi *gsi, struct platform_device *pdev) in gsi_irq_init() argument
1431 gsi->irq = ret; in gsi_irq_init()
1438 gsi_event_trans(struct gsi *gsi, struct gsi_event *event) in gsi_event_trans() argument
1446 channel = &gsi->channel[channel_id]; in gsi_event_trans()
1447 if (WARN(!channel->gsi, "event has bad channel %u\n", channel_id)) in gsi_event_trans()
1486 static void gsi_evt_ring_update(struct gsi *gsi, u32 evt_ring_id, u32 index) in gsi_evt_ring_update() argument
1488 struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_evt_ring_update()
1513 trans = gsi_event_trans(gsi, event); in gsi_evt_ring_update()
1532 gsi_evt_ring_doorbell(gsi, evt_ring_id, index); in gsi_evt_ring_update()
1536 static int gsi_ring_alloc(struct gsi *gsi, struct gsi_ring *ring, u32 count) in gsi_ring_alloc() argument
1539 struct device *dev = gsi->dev; in gsi_ring_alloc()
1558 static void gsi_ring_free(struct gsi *gsi, struct gsi_ring *ring) in gsi_ring_free() argument
1562 dma_free_coherent(gsi->dev, size, ring->virt, ring->addr); in gsi_ring_free()
1566 static int gsi_evt_ring_id_alloc(struct gsi *gsi) in gsi_evt_ring_id_alloc() argument
1570 if (gsi->event_bitmap == ~0U) { in gsi_evt_ring_id_alloc()
1571 dev_err(gsi->dev, "event rings exhausted\n"); in gsi_evt_ring_id_alloc()
1575 evt_ring_id = ffz(gsi->event_bitmap); in gsi_evt_ring_id_alloc()
1576 gsi->event_bitmap |= BIT(evt_ring_id); in gsi_evt_ring_id_alloc()
1582 static void gsi_evt_ring_id_free(struct gsi *gsi, u32 evt_ring_id) in gsi_evt_ring_id_free() argument
1584 gsi->event_bitmap &= ~BIT(evt_ring_id); in gsi_evt_ring_id_free()
1592 struct gsi *gsi = channel->gsi; in gsi_channel_doorbell() local
1596 reg = gsi_reg(gsi, CH_C_DOORBELL_0); in gsi_channel_doorbell()
1599 iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); in gsi_channel_doorbell()
1606 struct gsi *gsi = channel->gsi; in gsi_channel_update() local
1614 evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_channel_update()
1620 reg = gsi_reg(gsi, EV_CH_E_CNTXT_4); in gsi_channel_update()
1622 index = gsi_ring_index(ring, ioread32(gsi->virt + offset)); in gsi_channel_update()
1627 trans = gsi_event_trans(gsi, gsi_ring_virt(ring, index - 1)); in gsi_channel_update()
1636 gsi_evt_ring_update(gsi, evt_ring_id, index); in gsi_channel_update()
1691 gsi_irq_ieob_enable_one(channel->gsi, channel->evt_ring_id); in gsi_channel_poll()
1711 static int gsi_channel_setup_one(struct gsi *gsi, u32 channel_id) in gsi_channel_setup_one() argument
1713 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_setup_one()
1720 ret = gsi_evt_ring_alloc_command(gsi, evt_ring_id); in gsi_channel_setup_one()
1724 gsi_evt_ring_program(gsi, evt_ring_id); in gsi_channel_setup_one()
1726 ret = gsi_channel_alloc_command(gsi, channel_id); in gsi_channel_setup_one()
1733 netif_napi_add_tx(&gsi->dummy_dev, &channel->napi, in gsi_channel_setup_one()
1736 netif_napi_add(&gsi->dummy_dev, &channel->napi, in gsi_channel_setup_one()
1743 gsi_evt_ring_de_alloc_command(gsi, evt_ring_id); in gsi_channel_setup_one()
1749 static void gsi_channel_teardown_one(struct gsi *gsi, u32 channel_id) in gsi_channel_teardown_one() argument
1751 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_teardown_one()
1759 gsi_channel_de_alloc_command(gsi, channel_id); in gsi_channel_teardown_one()
1760 gsi_evt_ring_reset_command(gsi, evt_ring_id); in gsi_channel_teardown_one()
1761 gsi_evt_ring_de_alloc_command(gsi, evt_ring_id); in gsi_channel_teardown_one()
1768 static int gsi_generic_command(struct gsi *gsi, u32 channel_id, in gsi_generic_command() argument
1786 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); in gsi_generic_command()
1788 iowrite32(val, gsi->virt + reg_offset(reg)); in gsi_generic_command()
1791 reg = gsi_reg(gsi, CNTXT_SCRATCH_0); in gsi_generic_command()
1793 val = ioread32(gsi->virt + offset); in gsi_generic_command()
1796 iowrite32(val, gsi->virt + offset); in gsi_generic_command()
1799 reg = gsi_reg(gsi, GENERIC_CMD); in gsi_generic_command()
1803 if (gsi->version >= IPA_VERSION_4_11) in gsi_generic_command()
1806 timeout = !gsi_command(gsi, reg_offset(reg), val); in gsi_generic_command()
1809 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); in gsi_generic_command()
1810 iowrite32(ERROR_INT, gsi->virt + reg_offset(reg)); in gsi_generic_command()
1813 return gsi->result; in gsi_generic_command()
1815 dev_err(gsi->dev, "GSI generic command %u to channel %u timed out\n", in gsi_generic_command()
1821 static int gsi_modem_channel_alloc(struct gsi *gsi, u32 channel_id) in gsi_modem_channel_alloc() argument
1823 return gsi_generic_command(gsi, channel_id, in gsi_modem_channel_alloc()
1827 static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id) in gsi_modem_channel_halt() argument
1833 ret = gsi_generic_command(gsi, channel_id, in gsi_modem_channel_halt()
1838 dev_err(gsi->dev, "error %d halting modem channel %u\n", in gsi_modem_channel_halt()
1844 gsi_modem_channel_flow_control(struct gsi *gsi, u32 channel_id, bool enable) in gsi_modem_channel_flow_control() argument
1855 if (!enable && gsi->version >= IPA_VERSION_4_11) in gsi_modem_channel_flow_control()
1859 ret = gsi_generic_command(gsi, channel_id, command, 0); in gsi_modem_channel_flow_control()
1863 dev_err(gsi->dev, in gsi_modem_channel_flow_control()
1869 static int gsi_channel_setup(struct gsi *gsi) in gsi_channel_setup() argument
1875 gsi_irq_enable(gsi); in gsi_channel_setup()
1877 mutex_lock(&gsi->mutex); in gsi_channel_setup()
1880 ret = gsi_channel_setup_one(gsi, channel_id); in gsi_channel_setup()
1883 } while (++channel_id < gsi->channel_count); in gsi_channel_setup()
1887 struct gsi_channel *channel = &gsi->channel[channel_id++]; in gsi_channel_setup()
1893 dev_err(gsi->dev, "channel %u not supported by hardware\n", in gsi_channel_setup()
1895 channel_id = gsi->channel_count; in gsi_channel_setup()
1900 mask = gsi->modem_channel_bitmap; in gsi_channel_setup()
1904 ret = gsi_modem_channel_alloc(gsi, modem_channel_id); in gsi_channel_setup()
1912 mutex_unlock(&gsi->mutex); in gsi_channel_setup()
1918 mask ^= gsi->modem_channel_bitmap; in gsi_channel_setup()
1924 gsi_modem_channel_halt(gsi, channel_id); in gsi_channel_setup()
1929 gsi_channel_teardown_one(gsi, channel_id); in gsi_channel_setup()
1931 mutex_unlock(&gsi->mutex); in gsi_channel_setup()
1933 gsi_irq_disable(gsi); in gsi_channel_setup()
1939 static void gsi_channel_teardown(struct gsi *gsi) in gsi_channel_teardown() argument
1941 u32 mask = gsi->modem_channel_bitmap; in gsi_channel_teardown()
1944 mutex_lock(&gsi->mutex); in gsi_channel_teardown()
1951 gsi_modem_channel_halt(gsi, channel_id); in gsi_channel_teardown()
1954 channel_id = gsi->channel_count - 1; in gsi_channel_teardown()
1956 gsi_channel_teardown_one(gsi, channel_id); in gsi_channel_teardown()
1959 mutex_unlock(&gsi->mutex); in gsi_channel_teardown()
1961 gsi_irq_disable(gsi); in gsi_channel_teardown()
1965 static int gsi_irq_setup(struct gsi *gsi) in gsi_irq_setup() argument
1971 reg = gsi_reg(gsi, CNTXT_INTSET); in gsi_irq_setup()
1972 iowrite32(reg_bit(reg, INTYPE), gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1975 gsi_irq_type_update(gsi, 0); in gsi_irq_setup()
1978 reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK); in gsi_irq_setup()
1979 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1981 reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK); in gsi_irq_setup()
1982 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1984 reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); in gsi_irq_setup()
1985 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1987 reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK); in gsi_irq_setup()
1988 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1991 if (gsi->version > IPA_VERSION_3_1) { in gsi_irq_setup()
1992 reg = gsi_reg(gsi, INTER_EE_SRC_CH_IRQ_MSK); in gsi_irq_setup()
1993 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1995 reg = gsi_reg(gsi, INTER_EE_SRC_EV_CH_IRQ_MSK); in gsi_irq_setup()
1996 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
1999 reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN); in gsi_irq_setup()
2000 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_irq_setup()
2002 ret = request_irq(gsi->irq, gsi_isr, 0, "gsi", gsi); in gsi_irq_setup()
2004 dev_err(gsi->dev, "error %d requesting \"gsi\" IRQ\n", ret); in gsi_irq_setup()
2009 static void gsi_irq_teardown(struct gsi *gsi) in gsi_irq_teardown() argument
2011 free_irq(gsi->irq, gsi); in gsi_irq_teardown()
2015 static int gsi_ring_setup(struct gsi *gsi) in gsi_ring_setup() argument
2017 struct device *dev = gsi->dev; in gsi_ring_setup()
2022 if (gsi->version < IPA_VERSION_3_5_1) { in gsi_ring_setup()
2024 gsi->channel_count = GSI_CHANNEL_COUNT_MAX; in gsi_ring_setup()
2025 gsi->evt_ring_count = GSI_EVT_RING_COUNT_MAX; in gsi_ring_setup()
2030 reg = gsi_reg(gsi, HW_PARAM_2); in gsi_ring_setup()
2031 val = ioread32(gsi->virt + reg_offset(reg)); in gsi_ring_setup()
2043 gsi->channel_count = count; in gsi_ring_setup()
2045 if (gsi->version < IPA_VERSION_5_0) { in gsi_ring_setup()
2048 reg = gsi_reg(gsi, HW_PARAM_4); in gsi_ring_setup()
2061 gsi->evt_ring_count = count; in gsi_ring_setup()
2067 int gsi_setup(struct gsi *gsi) in gsi_setup() argument
2074 reg = gsi_reg(gsi, GSI_STATUS); in gsi_setup()
2075 val = ioread32(gsi->virt + reg_offset(reg)); in gsi_setup()
2077 dev_err(gsi->dev, "GSI has not been enabled\n"); in gsi_setup()
2081 ret = gsi_irq_setup(gsi); in gsi_setup()
2085 ret = gsi_ring_setup(gsi); /* No matching teardown required */ in gsi_setup()
2090 reg = gsi_reg(gsi, ERROR_LOG); in gsi_setup()
2091 iowrite32(0, gsi->virt + reg_offset(reg)); in gsi_setup()
2093 ret = gsi_channel_setup(gsi); in gsi_setup()
2100 gsi_irq_teardown(gsi); in gsi_setup()
2106 void gsi_teardown(struct gsi *gsi) in gsi_teardown() argument
2108 gsi_channel_teardown(gsi); in gsi_teardown()
2109 gsi_irq_teardown(gsi); in gsi_teardown()
2115 struct gsi *gsi = channel->gsi; in gsi_channel_evt_ring_init() local
2119 ret = gsi_evt_ring_id_alloc(gsi); in gsi_channel_evt_ring_init()
2124 evt_ring = &gsi->evt_ring[channel->evt_ring_id]; in gsi_channel_evt_ring_init()
2127 ret = gsi_ring_alloc(gsi, &evt_ring->ring, channel->event_count); in gsi_channel_evt_ring_init()
2131 dev_err(gsi->dev, "error %d allocating channel %u event ring\n", in gsi_channel_evt_ring_init()
2134 gsi_evt_ring_id_free(gsi, channel->evt_ring_id); in gsi_channel_evt_ring_init()
2143 struct gsi *gsi = channel->gsi; in gsi_channel_evt_ring_exit() local
2146 evt_ring = &gsi->evt_ring[evt_ring_id]; in gsi_channel_evt_ring_exit()
2147 gsi_ring_free(gsi, &evt_ring->ring); in gsi_channel_evt_ring_exit()
2148 gsi_evt_ring_id_free(gsi, evt_ring_id); in gsi_channel_evt_ring_exit()
2151 static bool gsi_channel_data_valid(struct gsi *gsi, bool command, in gsi_channel_data_valid() argument
2156 struct device *dev = gsi->dev; in gsi_channel_data_valid()
2219 static int gsi_channel_init_one(struct gsi *gsi, in gsi_channel_init_one() argument
2227 if (!gsi_channel_data_valid(gsi, command, data)) in gsi_channel_init_one()
2233 dev_warn(gsi->dev, "channel %u limited to %u TREs\n", in gsi_channel_init_one()
2239 channel = &gsi->channel[data->channel_id]; in gsi_channel_init_one()
2242 channel->gsi = gsi; in gsi_channel_init_one()
2253 ret = gsi_ring_alloc(gsi, &channel->tre_ring, data->channel.tre_count); in gsi_channel_init_one()
2255 dev_err(gsi->dev, "error %d allocating channel %u ring\n", in gsi_channel_init_one()
2260 ret = gsi_channel_trans_init(gsi, data->channel_id); in gsi_channel_init_one()
2265 u32 tre_max = gsi_channel_tre_max(gsi, data->channel_id); in gsi_channel_init_one()
2274 gsi_ring_free(gsi, &channel->tre_ring); in gsi_channel_init_one()
2278 channel->gsi = NULL; /* Mark it not (fully) initialized */ in gsi_channel_init_one()
2292 gsi_ring_free(channel->gsi, &channel->tre_ring); in gsi_channel_exit_one()
2297 static int gsi_channel_init(struct gsi *gsi, u32 count, in gsi_channel_init() argument
2305 modem_alloc = gsi->version == IPA_VERSION_4_2; in gsi_channel_init()
2307 gsi->event_bitmap = gsi_event_bitmap_init(GSI_EVT_RING_COUNT_MAX); in gsi_channel_init()
2308 gsi->ieob_enabled_bitmap = 0; in gsi_channel_init()
2320 gsi->modem_channel_bitmap |= in gsi_channel_init()
2325 ret = gsi_channel_init_one(gsi, &data[i], command); in gsi_channel_init()
2337 gsi->modem_channel_bitmap &= ~BIT(data[i].channel_id); in gsi_channel_init()
2340 gsi_channel_exit_one(&gsi->channel[data->channel_id]); in gsi_channel_init()
2347 static void gsi_channel_exit(struct gsi *gsi) in gsi_channel_exit() argument
2352 gsi_channel_exit_one(&gsi->channel[channel_id]); in gsi_channel_exit()
2354 gsi->modem_channel_bitmap = 0; in gsi_channel_exit()
2358 int gsi_init(struct gsi *gsi, struct platform_device *pdev, in gsi_init() argument
2366 gsi->dev = &pdev->dev; in gsi_init()
2367 gsi->version = version; in gsi_init()
2372 init_dummy_netdev(&gsi->dummy_dev); in gsi_init()
2373 init_completion(&gsi->completion); in gsi_init()
2375 ret = gsi_reg_init(gsi, pdev); in gsi_init()
2379 ret = gsi_irq_init(gsi, pdev); /* No matching exit required */ in gsi_init()
2383 ret = gsi_channel_init(gsi, count, data); in gsi_init()
2387 mutex_init(&gsi->mutex); in gsi_init()
2392 gsi_reg_exit(gsi); in gsi_init()
2398 void gsi_exit(struct gsi *gsi) in gsi_exit() argument
2400 mutex_destroy(&gsi->mutex); in gsi_exit()
2401 gsi_channel_exit(gsi); in gsi_exit()
2402 gsi_reg_exit(gsi); in gsi_exit()
2425 u32 gsi_channel_tre_max(struct gsi *gsi, u32 channel_id) in gsi_channel_tre_max() argument
2427 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_tre_max()