25 static int ipc_imem_msg_send_device_sleep(struct iosm_imem *ipc_imem, u32 state)  in ipc_imem_msg_send_device_sleep()  argument
32 	ipc_imem->device_sleep = state;  in ipc_imem_msg_send_device_sleep()
34 	return ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_device_sleep()
38 static bool ipc_imem_dl_skb_alloc(struct iosm_imem *ipc_imem,  in ipc_imem_dl_skb_alloc()  argument
45 	return ipc_protocol_dl_td_prepare(ipc_imem->ipc_protocol, pipe);  in ipc_imem_dl_skb_alloc()
51 static int ipc_imem_tq_td_alloc_timer(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_td_alloc_timer()  argument
59 		struct ipc_pipe *pipe = &ipc_imem->channels[i].dl_pipe;  in ipc_imem_tq_td_alloc_timer()
64 		while (ipc_imem_dl_skb_alloc(ipc_imem, pipe))  in ipc_imem_tq_td_alloc_timer()
72 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_td_alloc_timer()
76 		ipc_imem->hrtimer_period =  in ipc_imem_tq_td_alloc_timer()
78 		if (!hrtimer_active(&ipc_imem->td_alloc_timer))  in ipc_imem_tq_td_alloc_timer()
79 			hrtimer_start(&ipc_imem->td_alloc_timer,  in ipc_imem_tq_td_alloc_timer()
80 				      ipc_imem->hrtimer_period,  in ipc_imem_tq_td_alloc_timer()
88 	struct iosm_imem *ipc_imem =  in ipc_imem_td_alloc_timer_cb()  local
91 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_alloc_timer, 0, NULL,  in ipc_imem_td_alloc_timer_cb()
97 static int ipc_imem_tq_fast_update_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_fast_update_timer_cb()  argument
100 	ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_fast_update_timer_cb()
109 	struct iosm_imem *ipc_imem =  in ipc_imem_fast_update_timer_cb()  local
112 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_fast_update_timer_cb, 0,  in ipc_imem_fast_update_timer_cb()
117 static int ipc_imem_tq_adb_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_adb_timer_cb()  argument
120 	ipc_mux_ul_adb_finish(ipc_imem->mux);  in ipc_imem_tq_adb_timer_cb()
127 	struct iosm_imem *ipc_imem =  in ipc_imem_adb_timer_cb()  local
130 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_adb_timer_cb, 0,  in ipc_imem_adb_timer_cb()
135 static int ipc_imem_setup_cp_mux_cap_init(struct iosm_imem *ipc_imem,  in ipc_imem_setup_cp_mux_cap_init()  argument
138 	ipc_mmio_update_cp_capability(ipc_imem->mmio);  in ipc_imem_setup_cp_mux_cap_init()
140 	if (ipc_imem->mmio->mux_protocol == MUX_UNKNOWN) {  in ipc_imem_setup_cp_mux_cap_init()
141 		dev_err(ipc_imem->dev, "Failed to get Mux capability.");  in ipc_imem_setup_cp_mux_cap_init()
145 	cfg->protocol = ipc_imem->mmio->mux_protocol;  in ipc_imem_setup_cp_mux_cap_init()
147 	cfg->ul_flow = (ipc_imem->mmio->has_ul_flow_credit == 1) ?  in ipc_imem_setup_cp_mux_cap_init()
159 void ipc_imem_msg_send_feature_set(struct iosm_imem *ipc_imem,  in ipc_imem_msg_send_feature_set()  argument
166 		ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_feature_set()
170 		ipc_protocol_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_feature_set()
178 void ipc_imem_td_update_timer_start(struct iosm_imem *ipc_imem)  in ipc_imem_td_update_timer_start()  argument
181 	if (!ipc_imem->enter_runtime || ipc_imem->td_update_timer_suspended) {  in ipc_imem_td_update_timer_start()
183 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_td_update_timer_start()
188 	if (!hrtimer_active(&ipc_imem->tdupdate_timer)) {  in ipc_imem_td_update_timer_start()
189 		ipc_imem->hrtimer_period =  in ipc_imem_td_update_timer_start()
191 		if (!hrtimer_active(&ipc_imem->tdupdate_timer))  in ipc_imem_td_update_timer_start()
192 			hrtimer_start(&ipc_imem->tdupdate_timer,  in ipc_imem_td_update_timer_start()
193 				      ipc_imem->hrtimer_period,  in ipc_imem_td_update_timer_start()
208 void ipc_imem_adb_timer_start(struct iosm_imem *ipc_imem)  in ipc_imem_adb_timer_start()  argument
210 	if (!hrtimer_active(&ipc_imem->adb_timer)) {  in ipc_imem_adb_timer_start()
211 		ipc_imem->hrtimer_period =  in ipc_imem_adb_timer_start()
213 		hrtimer_start(&ipc_imem->adb_timer,  in ipc_imem_adb_timer_start()
214 			      ipc_imem->hrtimer_period,  in ipc_imem_adb_timer_start()
219 bool ipc_imem_ul_write_td(struct iosm_imem *ipc_imem)  in ipc_imem_ul_write_td()  argument
229 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_ul_write_td()
230 		channel = &ipc_imem->channels[i];  in ipc_imem_ul_write_td()
243 				ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
247 				ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
255 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
262 void ipc_imem_ipc_init_check(struct iosm_imem *ipc_imem)  in ipc_imem_ipc_init_check()  argument
266 	ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;  in ipc_imem_ipc_init_check()
269 	ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_ipc_init_check()
273 		if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_ipc_init_check()
274 		    ipc_imem->ipc_requested_state) {  in ipc_imem_ipc_init_check()
276 			ipc_mmio_config(ipc_imem->mmio);  in ipc_imem_ipc_init_check()
279 			ipc_imem->ipc_requested_state =  in ipc_imem_ipc_init_check()
281 			ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_ipc_init_check()
290 	dev_err(ipc_imem->dev, "%s: ipc_status(%d) ne. IPC_MEM_DEVICE_IPC_INIT",  in ipc_imem_ipc_init_check()
291 		ipc_imem_phase_get_string(ipc_imem->phase),  in ipc_imem_ipc_init_check()
292 		ipc_mmio_get_ipc_state(ipc_imem->mmio));  in ipc_imem_ipc_init_check()
294 	ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_TIMEOUT);  in ipc_imem_ipc_init_check()
298 static void ipc_imem_dl_skb_process(struct iosm_imem *ipc_imem,  in ipc_imem_dl_skb_process()  argument
310 		ipc_pcie_addr_unmap(ipc_imem->pcie, IPC_CB(skb)->len,  in ipc_imem_dl_skb_process()
314 			ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink,  in ipc_imem_dl_skb_process()
316 		else if (ipc_is_trace_channel(ipc_imem, port_id))  in ipc_imem_dl_skb_process()
317 			ipc_trace_port_rx(ipc_imem, skb);  in ipc_imem_dl_skb_process()
319 			wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port,  in ipc_imem_dl_skb_process()
325 			ipc_mux_dl_decode(ipc_imem->mux, skb);  in ipc_imem_dl_skb_process()
328 		dev_err(ipc_imem->dev, "Invalid channel type");  in ipc_imem_dl_skb_process()
334 static void ipc_imem_dl_pipe_process(struct iosm_imem *ipc_imem,  in ipc_imem_dl_pipe_process()  argument
345 	ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,  in ipc_imem_dl_pipe_process()
358 		skb = ipc_protocol_dl_td_process(ipc_imem->ipc_protocol, pipe);  in ipc_imem_dl_pipe_process()
361 		ipc_imem_dl_skb_process(ipc_imem, pipe, skb);  in ipc_imem_dl_pipe_process()
365 	while (ipc_imem_dl_skb_alloc(ipc_imem, pipe))  in ipc_imem_dl_pipe_process()
370 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_dl_pipe_process()
375 		ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);  in ipc_imem_dl_pipe_process()
383 		ipc_imem->hrtimer_period =  in ipc_imem_dl_pipe_process()
385 		hrtimer_start(&ipc_imem->fast_update_timer,  in ipc_imem_dl_pipe_process()
386 			      ipc_imem->hrtimer_period, HRTIMER_MODE_REL);  in ipc_imem_dl_pipe_process()
389 	if (ipc_imem->app_notify_dl_pend)  in ipc_imem_dl_pipe_process()
390 		complete(&ipc_imem->dl_pend_sem);  in ipc_imem_dl_pipe_process()
394 static void ipc_imem_ul_pipe_process(struct iosm_imem *ipc_imem,  in ipc_imem_ul_pipe_process()  argument
405 	ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,  in ipc_imem_ul_pipe_process()
417 		skb = ipc_protocol_ul_td_process(ipc_imem->ipc_protocol, pipe);  in ipc_imem_ul_pipe_process()
431 				ipc_mux_ul_encoded_process(ipc_imem->mux, skb);  in ipc_imem_ul_pipe_process()
433 				dev_err(ipc_imem->dev,  in ipc_imem_ul_pipe_process()
437 			ipc_pcie_kfree_skb(ipc_imem->pcie, skb);  in ipc_imem_ul_pipe_process()
443 		ipc_mux_check_n_restart_tx(ipc_imem->mux);  in ipc_imem_ul_pipe_process()
445 	if (ipc_imem->app_notify_ul_pend)  in ipc_imem_ul_pipe_process()
446 		complete(&ipc_imem->ul_pend_sem);  in ipc_imem_ul_pipe_process()
450 static void ipc_imem_rom_irq_exec(struct iosm_imem *ipc_imem)  in ipc_imem_rom_irq_exec()  argument
454 	channel = ipc_imem->ipc_devlink->devlink_sio.channel;  in ipc_imem_rom_irq_exec()
455 	ipc_imem->rom_exit_code = ipc_mmio_get_rom_exit_code(ipc_imem->mmio);  in ipc_imem_rom_irq_exec()
460 static int ipc_imem_tq_td_update_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_td_update_timer_cb()  argument
463 	ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_td_update_timer_cb()
469 static void ipc_imem_slp_control_exec(struct iosm_imem *ipc_imem)  in ipc_imem_slp_control_exec()  argument
472 	if (ipc_protocol_pm_dev_sleep_handle(ipc_imem->ipc_protocol) &&  in ipc_imem_slp_control_exec()
473 	    hrtimer_active(&ipc_imem->tdupdate_timer)) {  in ipc_imem_slp_control_exec()
475 		ipc_imem_tq_td_update_timer_cb(ipc_imem, 0, NULL, 0);  in ipc_imem_slp_control_exec()
477 		ipc_imem_hrtimer_stop(&ipc_imem->tdupdate_timer);  in ipc_imem_slp_control_exec()
479 		ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);  in ipc_imem_slp_control_exec()
484 static int ipc_imem_tq_startup_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_startup_timer_cb()  argument
488 	if (ipc_imem_phase_update(ipc_imem) != IPC_P_RUN)  in ipc_imem_tq_startup_timer_cb()
491 	if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_tq_startup_timer_cb()
493 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;  in ipc_imem_tq_startup_timer_cb()
495 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_tq_startup_timer_cb()
498 		ipc_imem->hrtimer_period = ktime_set(0, 100 * 1000UL * 1000ULL);  in ipc_imem_tq_startup_timer_cb()
500 		if (!hrtimer_active(&ipc_imem->startup_timer))  in ipc_imem_tq_startup_timer_cb()
501 			hrtimer_start(&ipc_imem->startup_timer,  in ipc_imem_tq_startup_timer_cb()
502 				      ipc_imem->hrtimer_period,  in ipc_imem_tq_startup_timer_cb()
504 	} else if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_tq_startup_timer_cb()
507 		ipc_imem_hrtimer_stop(&ipc_imem->startup_timer);  in ipc_imem_tq_startup_timer_cb()
510 		ipc_mmio_config(ipc_imem->mmio);  in ipc_imem_tq_startup_timer_cb()
511 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_RUNNING;  in ipc_imem_tq_startup_timer_cb()
512 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_tq_startup_timer_cb()
522 	struct iosm_imem *ipc_imem =  in ipc_imem_startup_timer_cb()  local
525 	if (ktime_to_ns(ipc_imem->hrtimer_period)) {  in ipc_imem_startup_timer_cb()
526 		hrtimer_forward_now(&ipc_imem->startup_timer,  in ipc_imem_startup_timer_cb()
527 				    ipc_imem->hrtimer_period);  in ipc_imem_startup_timer_cb()
531 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_startup_timer_cb, 0,  in ipc_imem_startup_timer_cb()
538 ipc_imem_get_exec_stage_buffered(struct iosm_imem *ipc_imem)  in ipc_imem_get_exec_stage_buffered()  argument
540 	return (ipc_imem->phase == IPC_P_RUN &&  in ipc_imem_get_exec_stage_buffered()
541 		ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) ?  in ipc_imem_get_exec_stage_buffered()
542 		       ipc_protocol_get_ap_exec_stage(ipc_imem->ipc_protocol) :  in ipc_imem_get_exec_stage_buffered()
543 		       ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_get_exec_stage_buffered()
547 static int ipc_imem_send_mdm_rdy_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_send_mdm_rdy_cb()  argument
551 		ipc_imem_get_exec_stage_buffered(ipc_imem);  in ipc_imem_send_mdm_rdy_cb()
554 		ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);  in ipc_imem_send_mdm_rdy_cb()
566 	struct iosm_imem *ipc_imem;  in ipc_imem_run_state_worker()  local
569 	ipc_imem = container_of(instance, struct iosm_imem, run_state_worker);  in ipc_imem_run_state_worker()
571 	if (ipc_imem->phase != IPC_P_RUN) {  in ipc_imem_run_state_worker()
572 		dev_err(ipc_imem->dev,  in ipc_imem_run_state_worker()
577 	if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))  in ipc_imem_run_state_worker()
578 		ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_run_state_worker()
580 	if (!ipc_imem_setup_cp_mux_cap_init(ipc_imem, &mux_cfg))  in ipc_imem_run_state_worker()
581 		ipc_imem->mux = ipc_mux_init(&mux_cfg, ipc_imem);  in ipc_imem_run_state_worker()
583 	ipc_imem_wwan_channel_init(ipc_imem, mux_cfg.protocol);  in ipc_imem_run_state_worker()
584 	if (ipc_imem->mux)  in ipc_imem_run_state_worker()
585 		ipc_imem->mux->wwan = ipc_imem->wwan;  in ipc_imem_run_state_worker()
589 			ipc_imem->ipc_port[ctrl_chl_idx] = NULL;  in ipc_imem_run_state_worker()
590 			if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7360_ID &&  in ipc_imem_run_state_worker()
596 				ipc_imem_channel_init(ipc_imem, IPC_CTYPE_CTRL,  in ipc_imem_run_state_worker()
599 				ipc_imem->ipc_port[ctrl_chl_idx] =  in ipc_imem_run_state_worker()
600 					ipc_port_init(ipc_imem, chnl_cfg_port);  in ipc_imem_run_state_worker()
606 	ipc_debugfs_init(ipc_imem);  in ipc_imem_run_state_worker()
608 	ipc_task_queue_send_task(ipc_imem, ipc_imem_send_mdm_rdy_cb, 0, NULL, 0,  in ipc_imem_run_state_worker()
614 	set_bit(FULLY_FUNCTIONAL, &ipc_imem->flag);  in ipc_imem_run_state_worker()
620 static void ipc_imem_handle_irq(struct iosm_imem *ipc_imem, int irq)  in ipc_imem_handle_irq()  argument
629 		ipc_imem->ev_irq_pending[irq] = false;  in ipc_imem_handle_irq()
632 	old_phase = ipc_imem->phase;  in ipc_imem_handle_irq()
635 		dev_dbg(ipc_imem->dev,  in ipc_imem_handle_irq()
642 	phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_handle_irq()
646 		if (!ipc_imem->enter_runtime) {  in ipc_imem_handle_irq()
648 			ipc_imem->enter_runtime = 1;  in ipc_imem_handle_irq()
653 			ipc_imem_msg_send_device_sleep(ipc_imem,  in ipc_imem_handle_irq()
654 						       ipc_imem->device_sleep);  in ipc_imem_handle_irq()
656 			ipc_imem_msg_send_feature_set(ipc_imem,  in ipc_imem_handle_irq()
662 			ipc_protocol_get_ipc_status(ipc_imem->ipc_protocol);  in ipc_imem_handle_irq()
665 		if (ipc_imem->ipc_status != curr_ipc_status) {  in ipc_imem_handle_irq()
666 			ipc_imem->ipc_status = curr_ipc_status;  in ipc_imem_handle_irq()
668 			if (ipc_imem->ipc_status ==  in ipc_imem_handle_irq()
670 				schedule_work(&ipc_imem->run_state_worker);  in ipc_imem_handle_irq()
675 		ipc_imem_slp_control_exec(ipc_imem);  in ipc_imem_handle_irq()
681 		dev_err(ipc_imem->dev, "confused phase %s",  in ipc_imem_handle_irq()
697 		ipc_imem_rom_irq_exec(ipc_imem);  in ipc_imem_handle_irq()
705 	ipc_protocol_msg_process(ipc_imem, irq);  in ipc_imem_handle_irq()
709 		struct ipc_pipe *ul_pipe = &ipc_imem->channels[i].ul_pipe;  in ipc_imem_handle_irq()
710 		struct ipc_pipe *dl_pipe = &ipc_imem->channels[i].dl_pipe;  in ipc_imem_handle_irq()
714 			ipc_imem_dl_pipe_process(ipc_imem, dl_pipe);  in ipc_imem_handle_irq()
721 			ipc_imem_ul_pipe_process(ipc_imem, ul_pipe);  in ipc_imem_handle_irq()
725 	if (ipc_mux_ul_data_encode(ipc_imem->mux)) {  in ipc_imem_handle_irq()
726 		ipc_imem_td_update_timer_start(ipc_imem);  in ipc_imem_handle_irq()
727 		if (ipc_imem->mux->protocol == MUX_AGGREGATION)  in ipc_imem_handle_irq()
728 			ipc_imem_adb_timer_start(ipc_imem);  in ipc_imem_handle_irq()
734 	ul_pending |= ipc_imem_ul_write_td(ipc_imem);  in ipc_imem_handle_irq()
738 		ipc_imem->hrtimer_period =  in ipc_imem_handle_irq()
740 		if (!hrtimer_active(&ipc_imem->tdupdate_timer))  in ipc_imem_handle_irq()
741 			hrtimer_start(&ipc_imem->tdupdate_timer,  in ipc_imem_handle_irq()
742 				      ipc_imem->hrtimer_period,  in ipc_imem_handle_irq()
751 	    ipc_imem->ipc_requested_state == IPC_MEM_DEVICE_IPC_RUNNING &&  in ipc_imem_handle_irq()
752 	    ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_handle_irq()
754 		complete(&ipc_imem->ipc_devlink->devlink_sio.channel->ul_sem);  in ipc_imem_handle_irq()
758 	ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_DONT_CARE;  in ipc_imem_handle_irq()
761 		ipc_imem->hrtimer_period =  in ipc_imem_handle_irq()
763 		if (!hrtimer_active(&ipc_imem->td_alloc_timer))  in ipc_imem_handle_irq()
764 			hrtimer_start(&ipc_imem->td_alloc_timer,  in ipc_imem_handle_irq()
765 				      ipc_imem->hrtimer_period,  in ipc_imem_handle_irq()
771 static int ipc_imem_tq_irq_cb(struct iosm_imem *ipc_imem, int arg, void *msg,  in ipc_imem_tq_irq_cb()  argument
774 	ipc_imem_handle_irq(ipc_imem, arg);  in ipc_imem_tq_irq_cb()
779 void ipc_imem_ul_send(struct iosm_imem *ipc_imem)  in ipc_imem_ul_send()  argument
782 	if (ipc_imem_ul_write_td(ipc_imem))  in ipc_imem_ul_send()
783 		ipc_imem_td_update_timer_start(ipc_imem);  in ipc_imem_ul_send()
787 static enum ipc_phase ipc_imem_phase_update_check(struct iosm_imem *ipc_imem,  in ipc_imem_phase_update_check()  argument
792 		if (ipc_imem->phase != IPC_P_ROM) {  in ipc_imem_phase_update_check()
794 			ipc_uevent_send(ipc_imem->dev, UEVENT_ROM_READY);  in ipc_imem_phase_update_check()
797 		ipc_imem->phase = IPC_P_ROM;  in ipc_imem_phase_update_check()
801 		ipc_imem->phase = IPC_P_PSI;  in ipc_imem_phase_update_check()
805 		ipc_imem->phase = IPC_P_EBL;  in ipc_imem_phase_update_check()
809 		if (ipc_imem->phase != IPC_P_RUN &&  in ipc_imem_phase_update_check()
810 		    ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) {  in ipc_imem_phase_update_check()
811 			ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);  in ipc_imem_phase_update_check()
813 		ipc_imem->phase = IPC_P_RUN;  in ipc_imem_phase_update_check()
817 		if (ipc_imem->phase != IPC_P_CRASH)  in ipc_imem_phase_update_check()
818 			ipc_uevent_send(ipc_imem->dev, UEVENT_CRASH);  in ipc_imem_phase_update_check()
820 		ipc_imem->phase = IPC_P_CRASH;  in ipc_imem_phase_update_check()
824 		if (ipc_imem->phase != IPC_P_CD_READY)  in ipc_imem_phase_update_check()
825 			ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY);  in ipc_imem_phase_update_check()
826 		ipc_imem->phase = IPC_P_CD_READY;  in ipc_imem_phase_update_check()
833 		ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);  in ipc_imem_phase_update_check()
837 	return ipc_imem->phase;  in ipc_imem_phase_update_check()
841 static bool ipc_imem_pipe_open(struct iosm_imem *ipc_imem,  in ipc_imem_pipe_open()  argument
848 	if (ipc_protocol_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_pipe_open()
856 static int ipc_imem_tq_pipe_td_alloc(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_pipe_td_alloc()  argument
864 		processed |= ipc_imem_dl_skb_alloc(ipc_imem, dl_pipe);  in ipc_imem_tq_pipe_td_alloc()
870 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, arg);  in ipc_imem_tq_pipe_td_alloc()
878 	struct iosm_imem *ipc_imem =  in ipc_imem_td_update_timer_cb()  local
881 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_update_timer_cb, 0,  in ipc_imem_td_update_timer_cb()
887 enum ipc_phase ipc_imem_phase_update(struct iosm_imem *ipc_imem)  in ipc_imem_phase_update()  argument
890 				ipc_imem_get_exec_stage_buffered(ipc_imem);  in ipc_imem_phase_update()
892 	return ipc_imem->phase == IPC_P_OFF_REQ ?  in ipc_imem_phase_update()
893 		       ipc_imem->phase :  in ipc_imem_phase_update()
894 		       ipc_imem_phase_update_check(ipc_imem, exec_stage);  in ipc_imem_phase_update()
929 void ipc_imem_pipe_close(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe)  in ipc_imem_pipe_close()  argument
934 	ipc_protocol_msg_send(ipc_imem->ipc_protocol, IPC_MSG_PREP_PIPE_CLOSE,  in ipc_imem_pipe_close()
937 	ipc_imem_pipe_cleanup(ipc_imem, pipe);  in ipc_imem_pipe_close()
940 void ipc_imem_channel_close(struct iosm_imem *ipc_imem, int channel_id)  in ipc_imem_channel_close()  argument
944 	if (channel_id < 0 || channel_id >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_close()
945 		dev_err(ipc_imem->dev, "invalid channel id %d", channel_id);  in ipc_imem_channel_close()
949 	channel = &ipc_imem->channels[channel_id];  in ipc_imem_channel_close()
952 		dev_err(ipc_imem->dev, "ch[%d]: invalid channel state %d",  in ipc_imem_channel_close()
962 	if (ipc_imem->phase == IPC_P_RUN) {  in ipc_imem_channel_close()
963 		ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_close()
964 		ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_close()
967 	ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_close()
968 	ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_close()
974 struct ipc_mem_channel *ipc_imem_channel_open(struct iosm_imem *ipc_imem,  in ipc_imem_channel_open()  argument
980 		dev_err(ipc_imem->dev, "invalid channel ID: %d", channel_id);  in ipc_imem_channel_open()
984 	channel = &ipc_imem->channels[channel_id];  in ipc_imem_channel_open()
988 	if (!ipc_imem_pipe_open(ipc_imem, &channel->ul_pipe))  in ipc_imem_channel_open()
991 	if (!ipc_imem_pipe_open(ipc_imem, &channel->dl_pipe))  in ipc_imem_channel_open()
995 	if (ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_pipe_td_alloc, db_id,  in ipc_imem_channel_open()
997 		dev_err(ipc_imem->dev, "td allocation failed : %d", channel_id);  in ipc_imem_channel_open()
1004 	ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_open()
1006 	ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_open()
1012 void ipc_imem_pm_suspend(struct iosm_imem *ipc_imem)  in ipc_imem_pm_suspend()  argument
1014 	ipc_protocol_suspend(ipc_imem->ipc_protocol);  in ipc_imem_pm_suspend()
1017 void ipc_imem_pm_s2idle_sleep(struct iosm_imem *ipc_imem, bool sleep)  in ipc_imem_pm_s2idle_sleep()  argument
1019 	ipc_protocol_s2idle_sleep(ipc_imem->ipc_protocol, sleep);  in ipc_imem_pm_s2idle_sleep()
1022 void ipc_imem_pm_resume(struct iosm_imem *ipc_imem)  in ipc_imem_pm_resume()  argument
1026 	if (ipc_protocol_resume(ipc_imem->ipc_protocol)) {  in ipc_imem_pm_resume()
1027 		stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_pm_resume()
1028 		ipc_imem_phase_update_check(ipc_imem, stage);  in ipc_imem_pm_resume()
1038 int ipc_imem_channel_alloc(struct iosm_imem *ipc_imem, int index,  in ipc_imem_channel_alloc()  argument
1045 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_channel_alloc()
1046 		channel = &ipc_imem->channels[i];  in ipc_imem_channel_alloc()
1051 	if (i >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_alloc()
1052 		dev_dbg(ipc_imem->dev,  in ipc_imem_channel_alloc()
1058 	if (ipc_imem->channels[i].state != IMEM_CHANNEL_FREE) {  in ipc_imem_channel_alloc()
1059 		dev_dbg(ipc_imem->dev, "channel is in use");  in ipc_imem_channel_alloc()
1073 void ipc_imem_channel_init(struct iosm_imem *ipc_imem, enum ipc_ctype ctype,  in ipc_imem_channel_init()  argument
1080 		dev_err(ipc_imem->dev, "invalid pipe: ul_pipe=%d, dl_pipe=%d",  in ipc_imem_channel_init()
1085 	if (ipc_imem->nr_of_channels >= IPC_MEM_MAX_CHANNELS) {  in ipc_imem_channel_init()
1086 		dev_err(ipc_imem->dev, "too many channels");  in ipc_imem_channel_init()
1090 	channel = &ipc_imem->channels[ipc_imem->nr_of_channels];  in ipc_imem_channel_init()
1091 	channel->channel_id = ipc_imem->nr_of_channels;  in ipc_imem_channel_init()
1096 	ipc_imem->nr_of_channels++;  in ipc_imem_channel_init()
1098 	ipc_imem_channel_update(ipc_imem, channel->channel_id, chnl_cfg,  in ipc_imem_channel_init()
1106 void ipc_imem_channel_update(struct iosm_imem *ipc_imem, int id,  in ipc_imem_channel_update()  argument
1111 	if (id < 0 || id >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_update()
1112 		dev_err(ipc_imem->dev, "invalid channel id %d", id);  in ipc_imem_channel_update()
1116 	channel = &ipc_imem->channels[id];  in ipc_imem_channel_update()
1120 		dev_err(ipc_imem->dev, "invalid channel state %d",  in ipc_imem_channel_update()
1146 static void ipc_imem_channel_reset(struct iosm_imem *ipc_imem)  in ipc_imem_channel_reset()  argument
1150 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_channel_reset()
1153 		channel = &ipc_imem->channels[i];  in ipc_imem_channel_reset()
1155 		ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_reset()
1156 		ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_reset()
1162 void ipc_imem_pipe_cleanup(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe)  in ipc_imem_pipe_cleanup()  argument
1173 		ipc_pcie_kfree_skb(ipc_imem->pcie, skb);  in ipc_imem_pipe_cleanup()
1175 	ipc_protocol_pipe_cleanup(ipc_imem->ipc_protocol, pipe);  in ipc_imem_pipe_cleanup()
1179 static void ipc_imem_device_ipc_uninit(struct iosm_imem *ipc_imem)  in ipc_imem_device_ipc_uninit()  argument
1187 	if (ipc_pcie_check_data_link_active(ipc_imem->pcie)) {  in ipc_imem_device_ipc_uninit()
1192 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_device_ipc_uninit()
1194 		ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);  in ipc_imem_device_ipc_uninit()
1204 			ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);  in ipc_imem_device_ipc_uninit()
1209 void ipc_imem_cleanup(struct iosm_imem *ipc_imem)  in ipc_imem_cleanup()  argument
1211 	ipc_imem->phase = IPC_P_OFF_REQ;  in ipc_imem_cleanup()
1214 	ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_NOT_READY);  in ipc_imem_cleanup()
1216 	hrtimer_cancel(&ipc_imem->td_alloc_timer);  in ipc_imem_cleanup()
1217 	hrtimer_cancel(&ipc_imem->tdupdate_timer);  in ipc_imem_cleanup()
1218 	hrtimer_cancel(&ipc_imem->fast_update_timer);  in ipc_imem_cleanup()
1219 	hrtimer_cancel(&ipc_imem->startup_timer);  in ipc_imem_cleanup()
1222 	cancel_work_sync(&ipc_imem->run_state_worker);  in ipc_imem_cleanup()
1224 	if (test_and_clear_bit(FULLY_FUNCTIONAL, &ipc_imem->flag)) {  in ipc_imem_cleanup()
1225 		ipc_mux_deinit(ipc_imem->mux);  in ipc_imem_cleanup()
1226 		ipc_debugfs_deinit(ipc_imem);  in ipc_imem_cleanup()
1227 		ipc_wwan_deinit(ipc_imem->wwan);  in ipc_imem_cleanup()
1228 		ipc_port_deinit(ipc_imem->ipc_port);  in ipc_imem_cleanup()
1231 	if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))  in ipc_imem_cleanup()
1232 		ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_cleanup()
1234 	ipc_imem_device_ipc_uninit(ipc_imem);  in ipc_imem_cleanup()
1235 	ipc_imem_channel_reset(ipc_imem);  in ipc_imem_cleanup()
1237 	ipc_protocol_deinit(ipc_imem->ipc_protocol);  in ipc_imem_cleanup()
1238 	ipc_task_deinit(ipc_imem->ipc_task);  in ipc_imem_cleanup()
1240 	kfree(ipc_imem->ipc_task);  in ipc_imem_cleanup()
1241 	kfree(ipc_imem->mmio);  in ipc_imem_cleanup()
1243 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_cleanup()
1251 static int ipc_imem_config(struct iosm_imem *ipc_imem)  in ipc_imem_config()  argument
1256 	init_completion(&ipc_imem->ul_pend_sem);  in ipc_imem_config()
1258 	init_completion(&ipc_imem->dl_pend_sem);  in ipc_imem_config()
1261 	ipc_imem->ipc_status = IPC_MEM_DEVICE_IPC_UNINIT;  in ipc_imem_config()
1262 	ipc_imem->enter_runtime = 0;  in ipc_imem_config()
1264 	phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_config()
1269 		ipc_imem->hrtimer_period = ktime_set(0, 1000 * 1000 * 1000ULL);  in ipc_imem_config()
1271 		if (!hrtimer_active(&ipc_imem->startup_timer))  in ipc_imem_config()
1272 			hrtimer_start(&ipc_imem->startup_timer,  in ipc_imem_config()
1273 				      ipc_imem->hrtimer_period,  in ipc_imem_config()
1281 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_UNINIT;  in ipc_imem_config()
1284 		if (ipc_imem->ipc_requested_state ==  in ipc_imem_config()
1285 		    ipc_mmio_get_ipc_state(ipc_imem->mmio)) {  in ipc_imem_config()
1286 			ipc_imem_ipc_init_check(ipc_imem);  in ipc_imem_config()
1290 		dev_err(ipc_imem->dev,  in ipc_imem_config()
1292 			ipc_mmio_get_ipc_state(ipc_imem->mmio));  in ipc_imem_config()
1296 		dev_dbg(ipc_imem->dev,  in ipc_imem_config()
1301 		dev_err(ipc_imem->dev, "unexpected operation phase %d", phase);  in ipc_imem_config()
1305 	complete(&ipc_imem->dl_pend_sem);  in ipc_imem_config()
1306 	complete(&ipc_imem->ul_pend_sem);  in ipc_imem_config()
1307 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_config()
1315 	struct iosm_imem *ipc_imem = kzalloc(sizeof(*pcie->imem), GFP_KERNEL);  in ipc_imem_init()  local
1318 	if (!ipc_imem)  in ipc_imem_init()
1322 	ipc_imem->pcie = pcie;  in ipc_imem_init()
1323 	ipc_imem->dev = dev;  in ipc_imem_init()
1325 	ipc_imem->pci_device_id = device_id;  in ipc_imem_init()
1327 	ipc_imem->cp_version = 0;  in ipc_imem_init()
1328 	ipc_imem->device_sleep = IPC_HOST_SLEEP_ENTER_SLEEP;  in ipc_imem_init()
1331 	ipc_imem->nr_of_channels = 0;  in ipc_imem_init()
1334 	ipc_imem->mmio = ipc_mmio_init(mmio, ipc_imem->dev);  in ipc_imem_init()
1335 	if (!ipc_imem->mmio) {  in ipc_imem_init()
1336 		dev_err(ipc_imem->dev, "failed to initialize mmio region");  in ipc_imem_init()
1340 	ipc_imem->ipc_task = kzalloc(sizeof(*ipc_imem->ipc_task),  in ipc_imem_init()
1344 	if (!ipc_imem->ipc_task)  in ipc_imem_init()
1347 	if (ipc_task_init(ipc_imem->ipc_task))  in ipc_imem_init()
1350 	ipc_imem->ipc_task->dev = ipc_imem->dev;  in ipc_imem_init()
1352 	INIT_WORK(&ipc_imem->run_state_worker, ipc_imem_run_state_worker);  in ipc_imem_init()
1354 	ipc_imem->ipc_protocol = ipc_protocol_init(ipc_imem);  in ipc_imem_init()
1356 	if (!ipc_imem->ipc_protocol)  in ipc_imem_init()
1360 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_init()
1362 	hrtimer_init(&ipc_imem->startup_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1364 	ipc_imem->startup_timer.function = ipc_imem_startup_timer_cb;  in ipc_imem_init()
1366 	hrtimer_init(&ipc_imem->tdupdate_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1368 	ipc_imem->tdupdate_timer.function = ipc_imem_td_update_timer_cb;  in ipc_imem_init()
1370 	hrtimer_init(&ipc_imem->fast_update_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1372 	ipc_imem->fast_update_timer.function = ipc_imem_fast_update_timer_cb;  in ipc_imem_init()
1374 	hrtimer_init(&ipc_imem->td_alloc_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1376 	ipc_imem->td_alloc_timer.function = ipc_imem_td_alloc_timer_cb;  in ipc_imem_init()
1378 	hrtimer_init(&ipc_imem->adb_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in ipc_imem_init()
1379 	ipc_imem->adb_timer.function = ipc_imem_adb_timer_cb;  in ipc_imem_init()
1381 	if (ipc_imem_config(ipc_imem)) {  in ipc_imem_init()
1382 		dev_err(ipc_imem->dev, "failed to initialize the imem");  in ipc_imem_init()
1386 	stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_init()
1389 		ipc_imem->ipc_devlink = ipc_devlink_init(ipc_imem);  in ipc_imem_init()
1390 		if (!ipc_imem->ipc_devlink) {  in ipc_imem_init()
1391 			dev_err(ipc_imem->dev, "Devlink register failed");  in ipc_imem_init()
1395 		if (ipc_flash_link_establish(ipc_imem))  in ipc_imem_init()
1398 		set_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag);  in ipc_imem_init()
1400 	return ipc_imem;  in ipc_imem_init()
1402 	ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_init()
1404 	hrtimer_cancel(&ipc_imem->td_alloc_timer);  in ipc_imem_init()
1405 	hrtimer_cancel(&ipc_imem->fast_update_timer);  in ipc_imem_init()
1406 	hrtimer_cancel(&ipc_imem->tdupdate_timer);  in ipc_imem_init()
1407 	hrtimer_cancel(&ipc_imem->startup_timer);  in ipc_imem_init()
1409 	cancel_work_sync(&ipc_imem->run_state_worker);  in ipc_imem_init()
1410 	ipc_task_deinit(ipc_imem->ipc_task);  in ipc_imem_init()
1412 	kfree(ipc_imem->ipc_task);  in ipc_imem_init()
1414 	kfree(ipc_imem->mmio);  in ipc_imem_init()
1416 	kfree(ipc_imem);  in ipc_imem_init()
1420 void ipc_imem_irq_process(struct iosm_imem *ipc_imem, int irq)  in ipc_imem_irq_process()  argument
1423 	if (ipc_imem && !ipc_imem->ev_irq_pending[irq]) {  in ipc_imem_irq_process()
1424 		ipc_imem->ev_irq_pending[irq] = true;  in ipc_imem_irq_process()
1425 		ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_irq_cb, irq,  in ipc_imem_irq_process()
1430 void ipc_imem_td_update_timer_suspend(struct iosm_imem *ipc_imem, bool suspend)  in ipc_imem_td_update_timer_suspend()  argument
1432 	ipc_imem->td_update_timer_suspended = suspend;  in ipc_imem_td_update_timer_suspend()
1438 static int ipc_imem_devlink_trigger_chip_info_cb(struct iosm_imem *ipc_imem,  in ipc_imem_devlink_trigger_chip_info_cb()  argument
1448 	stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_devlink_trigger_chip_info_cb()
1450 		dev_err(ipc_imem->dev,  in ipc_imem_devlink_trigger_chip_info_cb()
1455 	size = ipc_imem->mmio->chip_info_size;  in ipc_imem_devlink_trigger_chip_info_cb()
1459 	skb = ipc_pcie_alloc_local_skb(ipc_imem->pcie, GFP_ATOMIC, size);  in ipc_imem_devlink_trigger_chip_info_cb()
1461 		dev_err(ipc_imem->dev, "exhausted skbuf kernel DL memory");  in ipc_imem_devlink_trigger_chip_info_cb()
1466 	ipc_mmio_copy_chip_info(ipc_imem->mmio, skb_put(skb, size), size);  in ipc_imem_devlink_trigger_chip_info_cb()
1468 	dev_dbg(ipc_imem->dev, "execution_stage[%X] eq. BOOT", stage);  in ipc_imem_devlink_trigger_chip_info_cb()
1469 	ipc_imem->phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_devlink_trigger_chip_info_cb()
1470 	ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb);  in ipc_imem_devlink_trigger_chip_info_cb()
1476 int ipc_imem_devlink_trigger_chip_info(struct iosm_imem *ipc_imem)  in ipc_imem_devlink_trigger_chip_info()  argument
1478 	return ipc_task_queue_send_task(ipc_imem,  in ipc_imem_devlink_trigger_chip_info()