350 	struct bam_device *bdev;  member
409 static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,  in bam_addr()  argument
412 	const struct reg_offset_data r = bdev->layout[reg];  in bam_addr()
414 	return bdev->regs + r.base_offset +  in bam_addr()
417 		r.ee_mult * bdev->ee;  in bam_addr()
424 static void bam_reset(struct bam_device *bdev)  in bam_reset()  argument
430 	val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));  in bam_reset()
432 	writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));  in bam_reset()
434 	writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));  in bam_reset()
441 	writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));  in bam_reset()
445 			bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));  in bam_reset()
448 	writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));  in bam_reset()
452 			bam_addr(bdev, 0, BAM_IRQ_EN));  in bam_reset()
455 	writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));  in bam_reset()
466 	struct bam_device *bdev = bchan->bdev;  in bam_reset_channel()  local
471 	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));  in bam_reset_channel()
472 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));  in bam_reset_channel()
491 	struct bam_device *bdev = bchan->bdev;  in bam_chan_init_hw()  local
502 			bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));  in bam_chan_init_hw()
504 			bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));  in bam_chan_init_hw()
508 			bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));  in bam_chan_init_hw()
511 	val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));  in bam_chan_init_hw()
513 	writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));  in bam_chan_init_hw()
523 	writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));  in bam_chan_init_hw()
541 	struct bam_device *bdev = bchan->bdev;  in bam_alloc_chan()  local
547 	bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,  in bam_alloc_chan()
551 		dev_err(bdev->dev, "Failed to allocate desc fifo\n");  in bam_alloc_chan()
555 	if (bdev->active_channels++ == 0 && bdev->powered_remotely)  in bam_alloc_chan()
556 		bam_reset(bdev);  in bam_alloc_chan()
571 	struct bam_device *bdev = bchan->bdev;  in bam_free_chan()  local
576 	ret = pm_runtime_get_sync(bdev->dev);  in bam_free_chan()
583 		dev_err(bchan->bdev->dev, "Cannot free busy channel\n");  in bam_free_chan()
591 	dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,  in bam_free_chan()
596 	val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));  in bam_free_chan()
598 	writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));  in bam_free_chan()
601 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));  in bam_free_chan()
603 	if (--bdev->active_channels == 0 && bdev->powered_remotely) {  in bam_free_chan()
605 		val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));  in bam_free_chan()
607 		writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));  in bam_free_chan()
611 	pm_runtime_mark_last_busy(bdev->dev);  in bam_free_chan()
612 	pm_runtime_put_autosuspend(bdev->dev);  in bam_free_chan()
653 	struct bam_device *bdev = bchan->bdev;  in bam_prep_slave_sg()  local
662 		dev_err(bdev->dev, "invalid dma direction\n");  in bam_prep_slave_sg()
775 	struct bam_device *bdev = bchan->bdev;  in bam_pause()  local
779 	ret = pm_runtime_get_sync(bdev->dev);  in bam_pause()
784 	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));  in bam_pause()
787 	pm_runtime_mark_last_busy(bdev->dev);  in bam_pause()
788 	pm_runtime_put_autosuspend(bdev->dev);  in bam_pause()
801 	struct bam_device *bdev = bchan->bdev;  in bam_resume()  local
805 	ret = pm_runtime_get_sync(bdev->dev);  in bam_resume()
810 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));  in bam_resume()
813 	pm_runtime_mark_last_busy(bdev->dev);  in bam_resume()
814 	pm_runtime_put_autosuspend(bdev->dev);  in bam_resume()
826 static u32 process_channel_irqs(struct bam_device *bdev)  in process_channel_irqs()  argument
832 	srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));  in process_channel_irqs()
838 	for (i = 0; i < bdev->num_channels; i++) {  in process_channel_irqs()
839 		struct bam_chan *bchan = &bdev->channels[i];  in process_channel_irqs()
845 		pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS));  in process_channel_irqs()
847 		writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));  in process_channel_irqs()
851 		offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &  in process_channel_irqs()
904 	struct bam_device *bdev = data;  in bam_dma_irq()  local
908 	srcs |= process_channel_irqs(bdev);  in bam_dma_irq()
912 		tasklet_schedule(&bdev->task);  in bam_dma_irq()
914 	ret = pm_runtime_get_sync(bdev->dev);  in bam_dma_irq()
919 		clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));  in bam_dma_irq()
927 		writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR));  in bam_dma_irq()
930 	pm_runtime_mark_last_busy(bdev->dev);  in bam_dma_irq()
931 	pm_runtime_put_autosuspend(bdev->dev);  in bam_dma_irq()
995 	struct bam_device *bdev = bchan->bdev;  in bam_apply_new_config()  local
998 	if (!bdev->controlled_remotely) {  in bam_apply_new_config()
1005 			       bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));  in bam_apply_new_config()
1018 	struct bam_device *bdev = bchan->bdev;  in bam_start_dma()  local
1032 	ret = pm_runtime_get_sync(bdev->dev);  in bam_start_dma()
1103 			bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));  in bam_start_dma()
1105 	pm_runtime_mark_last_busy(bdev->dev);  in bam_start_dma()
1106 	pm_runtime_put_autosuspend(bdev->dev);  in bam_start_dma()
1117 	struct bam_device *bdev = from_tasklet(bdev, t, task);  in dma_tasklet()  local
1123 	for (i = 0; i < bdev->num_channels; i++) {  in dma_tasklet()
1124 		bchan = &bdev->channels[i];  in dma_tasklet()
1170 	struct bam_device *bdev = container_of(of->of_dma_data,  in bam_dma_xlate()  local
1178 	if (request >= bdev->num_channels)  in bam_dma_xlate()
1181 	return dma_get_slave_channel(&(bdev->channels[request].vc.chan));  in bam_dma_xlate()
1190 static int bam_init(struct bam_device *bdev)  in bam_init()  argument
1195 	if (!bdev->num_ees) {  in bam_init()
1196 		val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION));  in bam_init()
1197 		bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK;  in bam_init()
1201 	if (bdev->ee >= bdev->num_ees)  in bam_init()
1204 	if (!bdev->num_channels) {  in bam_init()
1205 		val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES));  in bam_init()
1206 		bdev->num_channels = val & BAM_NUM_PIPES_MASK;  in bam_init()
1210 	if (!bdev->controlled_remotely && !bdev->powered_remotely)  in bam_init()
1211 		bam_reset(bdev);  in bam_init()
1216 static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,  in bam_channel_init()  argument
1220 	bchan->bdev = bdev;  in bam_channel_init()
1222 	vchan_init(&bchan->vc, &bdev->common);  in bam_channel_init()
1238 	struct bam_device *bdev;  in bam_dma_probe()  local
1242 	bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL);  in bam_dma_probe()
1243 	if (!bdev)  in bam_dma_probe()
1246 	bdev->dev = &pdev->dev;  in bam_dma_probe()
1254 	bdev->layout = match->data;  in bam_dma_probe()
1256 	bdev->regs = devm_platform_ioremap_resource(pdev, 0);  in bam_dma_probe()
1257 	if (IS_ERR(bdev->regs))  in bam_dma_probe()
1258 		return PTR_ERR(bdev->regs);  in bam_dma_probe()
1260 	bdev->irq = platform_get_irq(pdev, 0);  in bam_dma_probe()
1261 	if (bdev->irq < 0)  in bam_dma_probe()
1262 		return bdev->irq;  in bam_dma_probe()
1264 	ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee);  in bam_dma_probe()
1266 		dev_err(bdev->dev, "Execution environment unspecified\n");  in bam_dma_probe()
1270 	bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,  in bam_dma_probe()
1272 	bdev->powered_remotely = of_property_read_bool(pdev->dev.of_node,  in bam_dma_probe()
1275 	if (bdev->controlled_remotely || bdev->powered_remotely)  in bam_dma_probe()
1276 		bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk");  in bam_dma_probe()
1278 		bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");  in bam_dma_probe()
1280 	if (IS_ERR(bdev->bamclk))  in bam_dma_probe()
1281 		return PTR_ERR(bdev->bamclk);  in bam_dma_probe()
1283 	if (!bdev->bamclk) {  in bam_dma_probe()
1285 					   &bdev->num_channels);  in bam_dma_probe()
1287 			dev_err(bdev->dev, "num-channels unspecified in dt\n");  in bam_dma_probe()
1290 					   &bdev->num_ees);  in bam_dma_probe()
1292 			dev_err(bdev->dev, "num-ees unspecified in dt\n");  in bam_dma_probe()
1295 	ret = clk_prepare_enable(bdev->bamclk);  in bam_dma_probe()
1297 		dev_err(bdev->dev, "failed to prepare/enable clock\n");  in bam_dma_probe()
1301 	ret = bam_init(bdev);  in bam_dma_probe()
1305 	tasklet_setup(&bdev->task, dma_tasklet);  in bam_dma_probe()
1307 	bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,  in bam_dma_probe()
1308 				sizeof(*bdev->channels), GFP_KERNEL);  in bam_dma_probe()
1310 	if (!bdev->channels) {  in bam_dma_probe()
1316 	INIT_LIST_HEAD(&bdev->common.channels);  in bam_dma_probe()
1318 	for (i = 0; i < bdev->num_channels; i++)  in bam_dma_probe()
1319 		bam_channel_init(bdev, &bdev->channels[i], i);  in bam_dma_probe()
1321 	ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq,  in bam_dma_probe()
1322 			IRQF_TRIGGER_HIGH, "bam_dma", bdev);  in bam_dma_probe()
1327 	bdev->common.dev = bdev->dev;  in bam_dma_probe()
1328 	ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE);  in bam_dma_probe()
1330 		dev_err(bdev->dev, "cannot set maximum segment size\n");  in bam_dma_probe()
1334 	platform_set_drvdata(pdev, bdev);  in bam_dma_probe()
1337 	dma_cap_zero(bdev->common.cap_mask);  in bam_dma_probe()
1338 	dma_cap_set(DMA_SLAVE, bdev->common.cap_mask);  in bam_dma_probe()
1341 	bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in bam_dma_probe()
1342 	bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;  in bam_dma_probe()
1343 	bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;  in bam_dma_probe()
1344 	bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;  in bam_dma_probe()
1345 	bdev->common.device_alloc_chan_resources = bam_alloc_chan;  in bam_dma_probe()
1346 	bdev->common.device_free_chan_resources = bam_free_chan;  in bam_dma_probe()
1347 	bdev->common.device_prep_slave_sg = bam_prep_slave_sg;  in bam_dma_probe()
1348 	bdev->common.device_config = bam_slave_config;  in bam_dma_probe()
1349 	bdev->common.device_pause = bam_pause;  in bam_dma_probe()
1350 	bdev->common.device_resume = bam_resume;  in bam_dma_probe()
1351 	bdev->common.device_terminate_all = bam_dma_terminate_all;  in bam_dma_probe()
1352 	bdev->common.device_issue_pending = bam_issue_pending;  in bam_dma_probe()
1353 	bdev->common.device_tx_status = bam_tx_status;  in bam_dma_probe()
1354 	bdev->common.dev = bdev->dev;  in bam_dma_probe()
1356 	ret = dma_async_device_register(&bdev->common);  in bam_dma_probe()
1358 		dev_err(bdev->dev, "failed to register dma async device\n");  in bam_dma_probe()
1363 					&bdev->common);  in bam_dma_probe()
1377 	dma_async_device_unregister(&bdev->common);  in bam_dma_probe()
1379 	for (i = 0; i < bdev->num_channels; i++)  in bam_dma_probe()
1380 		tasklet_kill(&bdev->channels[i].vc.task);  in bam_dma_probe()
1382 	tasklet_kill(&bdev->task);  in bam_dma_probe()
1384 	clk_disable_unprepare(bdev->bamclk);  in bam_dma_probe()
1391 	struct bam_device *bdev = platform_get_drvdata(pdev);  in bam_dma_remove()  local
1397 	dma_async_device_unregister(&bdev->common);  in bam_dma_remove()
1400 	writel_relaxed(0, bam_addr(bdev, 0,  BAM_IRQ_SRCS_MSK_EE));  in bam_dma_remove()
1402 	devm_free_irq(bdev->dev, bdev->irq, bdev);  in bam_dma_remove()
1404 	for (i = 0; i < bdev->num_channels; i++) {  in bam_dma_remove()
1405 		bam_dma_terminate_all(&bdev->channels[i].vc.chan);  in bam_dma_remove()
1406 		tasklet_kill(&bdev->channels[i].vc.task);  in bam_dma_remove()
1408 		if (!bdev->channels[i].fifo_virt)  in bam_dma_remove()
1411 		dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE,  in bam_dma_remove()
1412 			    bdev->channels[i].fifo_virt,  in bam_dma_remove()
1413 			    bdev->channels[i].fifo_phys);  in bam_dma_remove()
1416 	tasklet_kill(&bdev->task);  in bam_dma_remove()
1418 	clk_disable_unprepare(bdev->bamclk);  in bam_dma_remove()
1425 	struct bam_device *bdev = dev_get_drvdata(dev);  in bam_dma_runtime_suspend()  local
1427 	clk_disable(bdev->bamclk);  in bam_dma_runtime_suspend()
1434 	struct bam_device *bdev = dev_get_drvdata(dev);  in bam_dma_runtime_resume()  local
1437 	ret = clk_enable(bdev->bamclk);  in bam_dma_runtime_resume()
1448 	struct bam_device *bdev = dev_get_drvdata(dev);  in bam_dma_suspend()  local
1451 	clk_unprepare(bdev->bamclk);  in bam_dma_suspend()
1458 	struct bam_device *bdev = dev_get_drvdata(dev);  in bam_dma_resume()  local
1461 	ret = clk_prepare(bdev->bamclk);  in bam_dma_resume()