131 	struct stm32_qspi *qspi = (struct stm32_qspi *)dev_id;  in stm32_qspi_irq()  local
134 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_irq()
135 	sr = readl_relaxed(qspi->io_base + QSPI_SR);  in stm32_qspi_irq()
140 		writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_irq()
141 		complete(&qspi->match_completion);  in stm32_qspi_irq()
149 		writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_irq()
150 		complete(&qspi->data_completion);  in stm32_qspi_irq()
166 static int stm32_qspi_tx_poll(struct stm32_qspi *qspi,  in stm32_qspi_tx_poll()  argument
184 		ret = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR,  in stm32_qspi_tx_poll()
188 			dev_err(qspi->dev, "fifo timeout (len:%d stat:%#x)\n",  in stm32_qspi_tx_poll()
192 		tx_fifo(buf++, qspi->io_base + QSPI_DR);  in stm32_qspi_tx_poll()
198 static int stm32_qspi_tx_mm(struct stm32_qspi *qspi,  in stm32_qspi_tx_mm()  argument
201 	memcpy_fromio(op->data.buf.in, qspi->mm_base + op->addr.val,  in stm32_qspi_tx_mm()
213 static int stm32_qspi_tx_dma(struct stm32_qspi *qspi,  in stm32_qspi_tx_dma()  argument
226 		dma_ch = qspi->dma_chrx;  in stm32_qspi_tx_dma()
229 		dma_ch = qspi->dma_chtx;  in stm32_qspi_tx_dma()
236 	err = spi_controller_dma_map_mem_op_data(qspi->ctrl, op, &sgt);  in stm32_qspi_tx_dma()
247 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_tx_dma()
249 	reinit_completion(&qspi->dma_completion);  in stm32_qspi_tx_dma()
251 	desc->callback_param = &qspi->dma_completion;  in stm32_qspi_tx_dma()
259 	writel_relaxed(cr | CR_DMAEN, qspi->io_base + QSPI_CR);  in stm32_qspi_tx_dma()
262 	if (!wait_for_completion_timeout(&qspi->dma_completion,  in stm32_qspi_tx_dma()
270 	writel_relaxed(cr & ~CR_DMAEN, qspi->io_base + QSPI_CR);  in stm32_qspi_tx_dma()
272 	spi_controller_dma_unmap_mem_op_data(qspi->ctrl, op, &sgt);  in stm32_qspi_tx_dma()
277 static int stm32_qspi_tx(struct stm32_qspi *qspi, const struct spi_mem_op *op)  in stm32_qspi_tx()  argument
282 	if (qspi->fmode == CCR_FMODE_MM)  in stm32_qspi_tx()
283 		return stm32_qspi_tx_mm(qspi, op);  in stm32_qspi_tx()
284 	else if (((op->data.dir == SPI_MEM_DATA_IN && qspi->dma_chrx) ||  in stm32_qspi_tx()
285 		 (op->data.dir == SPI_MEM_DATA_OUT && qspi->dma_chtx)) &&  in stm32_qspi_tx()
287 		if (!stm32_qspi_tx_dma(qspi, op))  in stm32_qspi_tx()
290 	return stm32_qspi_tx_poll(qspi, op);  in stm32_qspi_tx()
293 static int stm32_qspi_wait_nobusy(struct stm32_qspi *qspi)  in stm32_qspi_wait_nobusy()  argument
297 	return readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR, sr,  in stm32_qspi_wait_nobusy()
302 static int stm32_qspi_wait_cmd(struct stm32_qspi *qspi)  in stm32_qspi_wait_cmd()  argument
307 	if ((readl_relaxed(qspi->io_base + QSPI_SR) & SR_TCF) ||  in stm32_qspi_wait_cmd()
308 	    qspi->fmode == CCR_FMODE_APM)  in stm32_qspi_wait_cmd()
311 	reinit_completion(&qspi->data_completion);  in stm32_qspi_wait_cmd()
312 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_wait_cmd()
313 	writel_relaxed(cr | CR_TCIE | CR_TEIE, qspi->io_base + QSPI_CR);  in stm32_qspi_wait_cmd()
315 	if (!wait_for_completion_timeout(&qspi->data_completion,  in stm32_qspi_wait_cmd()
319 		sr = readl_relaxed(qspi->io_base + QSPI_SR);  in stm32_qspi_wait_cmd()
326 	writel_relaxed(FCR_CTCF | FCR_CTEF, qspi->io_base + QSPI_FCR);  in stm32_qspi_wait_cmd()
328 		err = stm32_qspi_wait_nobusy(qspi);  in stm32_qspi_wait_cmd()
333 static int stm32_qspi_wait_poll_status(struct stm32_qspi *qspi)  in stm32_qspi_wait_poll_status()  argument
337 	reinit_completion(&qspi->match_completion);  in stm32_qspi_wait_poll_status()
338 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_wait_poll_status()
339 	writel_relaxed(cr | CR_SMIE, qspi->io_base + QSPI_CR);  in stm32_qspi_wait_poll_status()
341 	if (!wait_for_completion_timeout(&qspi->match_completion,  in stm32_qspi_wait_poll_status()
342 				msecs_to_jiffies(qspi->status_timeout)))  in stm32_qspi_wait_poll_status()
345 	writel_relaxed(FCR_CSMF, qspi->io_base + QSPI_FCR);  in stm32_qspi_wait_poll_status()
360 	struct stm32_qspi *qspi = spi_controller_get_devdata(spi->master);  in stm32_qspi_send()  local
361 	struct stm32_qspi_flash *flash = &qspi->flash[spi_get_chipselect(spi, 0)];  in stm32_qspi_send()
365 	dev_dbg(qspi->dev, "cmd:%#x mode:%d.%d.%d.%d addr:%#llx len:%#x\n",  in stm32_qspi_send()
370 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_send()
374 	writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_send()
378 			       qspi->io_base + QSPI_DLR);  in stm32_qspi_send()
380 	ccr = qspi->fmode;  in stm32_qspi_send()
400 	writel_relaxed(ccr, qspi->io_base + QSPI_CCR);  in stm32_qspi_send()
402 	if (op->addr.nbytes && qspi->fmode != CCR_FMODE_MM)  in stm32_qspi_send()
403 		writel_relaxed(op->addr.val, qspi->io_base + QSPI_AR);  in stm32_qspi_send()
405 	if (qspi->fmode == CCR_FMODE_APM)  in stm32_qspi_send()
406 		err_poll_status = stm32_qspi_wait_poll_status(qspi);  in stm32_qspi_send()
408 	err = stm32_qspi_tx(qspi, op);  in stm32_qspi_send()
417 	if (err || err_poll_status || qspi->fmode == CCR_FMODE_MM)  in stm32_qspi_send()
421 	err = stm32_qspi_wait_cmd(qspi);  in stm32_qspi_send()
428 	cr = readl_relaxed(qspi->io_base + QSPI_CR) | CR_ABORT;  in stm32_qspi_send()
429 	writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_send()
432 	timeout = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_CR,  in stm32_qspi_send()
436 	writel_relaxed(FCR_CTCF | FCR_CSMF, qspi->io_base + QSPI_FCR);  in stm32_qspi_send()
439 		dev_err(qspi->dev, "%s err:%d err_poll_status:%d abort timeout:%d\n",  in stm32_qspi_send()
451 	struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master);  in stm32_qspi_poll_status()  local
457 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_poll_status()
461 	mutex_lock(&qspi->lock);  in stm32_qspi_poll_status()
463 	writel_relaxed(mask, qspi->io_base + QSPI_PSMKR);  in stm32_qspi_poll_status()
464 	writel_relaxed(match, qspi->io_base + QSPI_PSMAR);  in stm32_qspi_poll_status()
465 	qspi->fmode = CCR_FMODE_APM;  in stm32_qspi_poll_status()
466 	qspi->status_timeout = timeout_ms;  in stm32_qspi_poll_status()
469 	mutex_unlock(&qspi->lock);  in stm32_qspi_poll_status()
471 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_poll_status()
472 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_poll_status()
479 	struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master);  in stm32_qspi_exec_op()  local
482 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_exec_op()
486 	mutex_lock(&qspi->lock);  in stm32_qspi_exec_op()
488 		qspi->fmode = CCR_FMODE_INDR;  in stm32_qspi_exec_op()
490 		qspi->fmode = CCR_FMODE_INDW;  in stm32_qspi_exec_op()
493 	mutex_unlock(&qspi->lock);  in stm32_qspi_exec_op()
495 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_exec_op()
496 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_exec_op()
503 	struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->master);  in stm32_qspi_dirmap_create()  local
509 	if (!qspi->mm_base && desc->info.op_tmpl.data.dir == SPI_MEM_DATA_IN)  in stm32_qspi_dirmap_create()
512 	if (!qspi->mm_size)  in stm32_qspi_dirmap_create()
521 	struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->master);  in stm32_qspi_dirmap_read()  local
526 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_dirmap_read()
530 	mutex_lock(&qspi->lock);  in stm32_qspi_dirmap_read()
536 	dev_dbg(qspi->dev, "%s len = 0x%zx offs = 0x%llx buf = 0x%p\n", __func__, len, offs, buf);  in stm32_qspi_dirmap_read()
543 	if (addr_max < qspi->mm_size && op.addr.buswidth)  in stm32_qspi_dirmap_read()
544 		qspi->fmode = CCR_FMODE_MM;  in stm32_qspi_dirmap_read()
546 		qspi->fmode = CCR_FMODE_INDR;  in stm32_qspi_dirmap_read()
549 	mutex_unlock(&qspi->lock);  in stm32_qspi_dirmap_read()
551 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_dirmap_read()
552 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_dirmap_read()
560 	struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);  in stm32_qspi_transfer_one_message()  local
569 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_transfer_one_message()
573 	mutex_lock(&qspi->lock);  in stm32_qspi_transfer_one_message()
582 		dev_dbg(qspi->dev, "tx_buf:%p tx_nbits:%d rx_buf:%p rx_nbits:%d len:%d dummy_data:%d\n",  in stm32_qspi_transfer_one_message()
609 			qspi->fmode = CCR_FMODE_INDR;  in stm32_qspi_transfer_one_message()
614 			qspi->fmode = CCR_FMODE_INDW;  in stm32_qspi_transfer_one_message()
630 	mutex_unlock(&qspi->lock);  in stm32_qspi_transfer_one_message()
635 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_transfer_one_message()
636 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_transfer_one_message()
644 	struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);  in stm32_qspi_setup()  local
658 	    gpiod_count(qspi->dev, "cs") == -ENOENT)) {  in stm32_qspi_setup()
659 		dev_err(qspi->dev, "spi-rx-bus-width\\/spi-tx-bus-width\\/cs-gpios\n");  in stm32_qspi_setup()
660 		dev_err(qspi->dev, "configuration not supported\n");  in stm32_qspi_setup()
665 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_setup()
669 	presc = DIV_ROUND_UP(qspi->clk_rate, spi->max_speed_hz) - 1;  in stm32_qspi_setup()
671 	flash = &qspi->flash[spi_get_chipselect(spi, 0)];  in stm32_qspi_setup()
675 	mutex_lock(&qspi->lock);  in stm32_qspi_setup()
676 	qspi->cr_reg = CR_APMS | 3 << CR_FTHRES_SHIFT | CR_SSHIFT | CR_EN;  in stm32_qspi_setup()
683 		qspi->cr_reg |= CR_DFM;  in stm32_qspi_setup()
684 		dev_dbg(qspi->dev, "Dual flash mode enable");  in stm32_qspi_setup()
687 	writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);  in stm32_qspi_setup()
690 	qspi->dcr_reg = DCR_FSIZE_MASK;  in stm32_qspi_setup()
691 	writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR);  in stm32_qspi_setup()
692 	mutex_unlock(&qspi->lock);  in stm32_qspi_setup()
694 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_setup()
695 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_setup()
700 static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)  in stm32_qspi_dma_setup()  argument
703 	struct device *dev = qspi->dev;  in stm32_qspi_dma_setup()
710 	dma_cfg.src_addr = qspi->phys_base + QSPI_DR;  in stm32_qspi_dma_setup()
711 	dma_cfg.dst_addr = qspi->phys_base + QSPI_DR;  in stm32_qspi_dma_setup()
715 	qspi->dma_chrx = dma_request_chan(dev, "rx");  in stm32_qspi_dma_setup()
716 	if (IS_ERR(qspi->dma_chrx)) {  in stm32_qspi_dma_setup()
717 		ret = PTR_ERR(qspi->dma_chrx);  in stm32_qspi_dma_setup()
718 		qspi->dma_chrx = NULL;  in stm32_qspi_dma_setup()
722 		if (dmaengine_slave_config(qspi->dma_chrx, &dma_cfg)) {  in stm32_qspi_dma_setup()
724 			dma_release_channel(qspi->dma_chrx);  in stm32_qspi_dma_setup()
725 			qspi->dma_chrx = NULL;  in stm32_qspi_dma_setup()
729 	qspi->dma_chtx = dma_request_chan(dev, "tx");  in stm32_qspi_dma_setup()
730 	if (IS_ERR(qspi->dma_chtx)) {  in stm32_qspi_dma_setup()
731 		ret = PTR_ERR(qspi->dma_chtx);  in stm32_qspi_dma_setup()
732 		qspi->dma_chtx = NULL;  in stm32_qspi_dma_setup()
734 		if (dmaengine_slave_config(qspi->dma_chtx, &dma_cfg)) {  in stm32_qspi_dma_setup()
736 			dma_release_channel(qspi->dma_chtx);  in stm32_qspi_dma_setup()
737 			qspi->dma_chtx = NULL;  in stm32_qspi_dma_setup()
742 	init_completion(&qspi->dma_completion);  in stm32_qspi_dma_setup()
750 static void stm32_qspi_dma_free(struct stm32_qspi *qspi)  in stm32_qspi_dma_free()  argument
752 	if (qspi->dma_chtx)  in stm32_qspi_dma_free()
753 		dma_release_channel(qspi->dma_chtx);  in stm32_qspi_dma_free()
754 	if (qspi->dma_chrx)  in stm32_qspi_dma_free()
755 		dma_release_channel(qspi->dma_chrx);  in stm32_qspi_dma_free()
774 	struct stm32_qspi *qspi;  in stm32_qspi_probe()  local
778 	ctrl = devm_spi_alloc_master(dev, sizeof(*qspi));  in stm32_qspi_probe()
782 	qspi = spi_controller_get_devdata(ctrl);  in stm32_qspi_probe()
783 	qspi->ctrl = ctrl;  in stm32_qspi_probe()
786 	qspi->io_base = devm_ioremap_resource(dev, res);  in stm32_qspi_probe()
787 	if (IS_ERR(qspi->io_base))  in stm32_qspi_probe()
788 		return PTR_ERR(qspi->io_base);  in stm32_qspi_probe()
790 	qspi->phys_base = res->start;  in stm32_qspi_probe()
793 	qspi->mm_base = devm_ioremap_resource(dev, res);  in stm32_qspi_probe()
794 	if (IS_ERR(qspi->mm_base))  in stm32_qspi_probe()
795 		return PTR_ERR(qspi->mm_base);  in stm32_qspi_probe()
797 	qspi->mm_size = resource_size(res);  in stm32_qspi_probe()
798 	if (qspi->mm_size > STM32_QSPI_MAX_MMAP_SZ)  in stm32_qspi_probe()
806 			       dev_name(dev), qspi);  in stm32_qspi_probe()
812 	init_completion(&qspi->data_completion);  in stm32_qspi_probe()
813 	init_completion(&qspi->match_completion);  in stm32_qspi_probe()
815 	qspi->clk = devm_clk_get(dev, NULL);  in stm32_qspi_probe()
816 	if (IS_ERR(qspi->clk))  in stm32_qspi_probe()
817 		return PTR_ERR(qspi->clk);  in stm32_qspi_probe()
819 	qspi->clk_rate = clk_get_rate(qspi->clk);  in stm32_qspi_probe()
820 	if (!qspi->clk_rate)  in stm32_qspi_probe()
823 	ret = clk_prepare_enable(qspi->clk);  in stm32_qspi_probe()
840 	qspi->dev = dev;  in stm32_qspi_probe()
841 	platform_set_drvdata(pdev, qspi);  in stm32_qspi_probe()
842 	ret = stm32_qspi_dma_setup(qspi);  in stm32_qspi_probe()
846 	mutex_init(&qspi->lock);  in stm32_qspi_probe()
874 	pm_runtime_get_sync(qspi->dev);  in stm32_qspi_probe()
876 	writel_relaxed(0, qspi->io_base + QSPI_CR);  in stm32_qspi_probe()
877 	mutex_destroy(&qspi->lock);  in stm32_qspi_probe()
878 	pm_runtime_put_noidle(qspi->dev);  in stm32_qspi_probe()
879 	pm_runtime_disable(qspi->dev);  in stm32_qspi_probe()
880 	pm_runtime_set_suspended(qspi->dev);  in stm32_qspi_probe()
881 	pm_runtime_dont_use_autosuspend(qspi->dev);  in stm32_qspi_probe()
883 	stm32_qspi_dma_free(qspi);  in stm32_qspi_probe()
885 	clk_disable_unprepare(qspi->clk);  in stm32_qspi_probe()
892 	struct stm32_qspi *qspi = platform_get_drvdata(pdev);  in stm32_qspi_remove()  local
894 	pm_runtime_get_sync(qspi->dev);  in stm32_qspi_remove()
895 	spi_unregister_master(qspi->ctrl);  in stm32_qspi_remove()
897 	writel_relaxed(0, qspi->io_base + QSPI_CR);  in stm32_qspi_remove()
898 	stm32_qspi_dma_free(qspi);  in stm32_qspi_remove()
899 	mutex_destroy(&qspi->lock);  in stm32_qspi_remove()
900 	pm_runtime_put_noidle(qspi->dev);  in stm32_qspi_remove()
901 	pm_runtime_disable(qspi->dev);  in stm32_qspi_remove()
902 	pm_runtime_set_suspended(qspi->dev);  in stm32_qspi_remove()
903 	pm_runtime_dont_use_autosuspend(qspi->dev);  in stm32_qspi_remove()
904 	clk_disable_unprepare(qspi->clk);  in stm32_qspi_remove()
909 	struct stm32_qspi *qspi = dev_get_drvdata(dev);  in stm32_qspi_runtime_suspend()  local
911 	clk_disable_unprepare(qspi->clk);  in stm32_qspi_runtime_suspend()
918 	struct stm32_qspi *qspi = dev_get_drvdata(dev);  in stm32_qspi_runtime_resume()  local
920 	return clk_prepare_enable(qspi->clk);  in stm32_qspi_runtime_resume()
932 	struct stm32_qspi *qspi = dev_get_drvdata(dev);  in stm32_qspi_resume()  local
945 	writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);  in stm32_qspi_resume()
946 	writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR);  in stm32_qspi_resume()