183 	int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,  in k3_ringacc_request_ring()  argument
337 	mutex_lock(&ringacc->req_lock);  in k3_ringacc_request_ring()
342 						&ringacc->rm_gp_range->desc[0];  in k3_ringacc_request_ring()
346 		id = find_next_zero_bit(ringacc->rings_inuse, size,  in k3_ringacc_request_ring()
354 	if (test_bit(id, ringacc->rings_inuse) &&  in k3_ringacc_request_ring()
355 	    !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))  in k3_ringacc_request_ring()
357 	else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)  in k3_ringacc_request_ring()
361 		proxy_id = find_first_zero_bit(ringacc->proxy_inuse,  in k3_ringacc_request_ring()
362 					      ringacc->num_proxies);  in k3_ringacc_request_ring()
363 		if (proxy_id == ringacc->num_proxies)  in k3_ringacc_request_ring()
368 		set_bit(proxy_id, ringacc->proxy_inuse);  in k3_ringacc_request_ring()
369 		ringacc->rings[id].proxy_id = proxy_id;  in k3_ringacc_request_ring()
370 		dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,  in k3_ringacc_request_ring()
373 		dev_dbg(ringacc->dev, "Giving ring#%d\n", id);  in k3_ringacc_request_ring()
376 	set_bit(id, ringacc->rings_inuse);  in k3_ringacc_request_ring()
378 	ringacc->rings[id].use_count++;  in k3_ringacc_request_ring()
379 	mutex_unlock(&ringacc->req_lock);  in k3_ringacc_request_ring()
380 	return &ringacc->rings[id];  in k3_ringacc_request_ring()
383 	mutex_unlock(&ringacc->req_lock);  in k3_ringacc_request_ring()
388 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,  in k3_dmaring_request_dual_ring()  argument
401 	mutex_lock(&ringacc->req_lock);  in k3_dmaring_request_dual_ring()
403 	if (test_bit(fwd_id, ringacc->rings_inuse)) {  in k3_dmaring_request_dual_ring()
408 	*fwd_ring = &ringacc->rings[fwd_id];  in k3_dmaring_request_dual_ring()
409 	*compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];  in k3_dmaring_request_dual_ring()
410 	set_bit(fwd_id, ringacc->rings_inuse);  in k3_dmaring_request_dual_ring()
411 	ringacc->rings[fwd_id].use_count++;  in k3_dmaring_request_dual_ring()
412 	dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);  in k3_dmaring_request_dual_ring()
414 	mutex_unlock(&ringacc->req_lock);  in k3_dmaring_request_dual_ring()
418 	mutex_unlock(&ringacc->req_lock);  in k3_dmaring_request_dual_ring()
422 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,  in k3_ringacc_request_rings_pair()  argument
432 	if (ringacc->dma_rings)  in k3_ringacc_request_rings_pair()
433 		return k3_dmaring_request_dual_ring(ringacc, fwd_id,  in k3_ringacc_request_rings_pair()
436 	*fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);  in k3_ringacc_request_rings_pair()
440 	*compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);  in k3_ringacc_request_rings_pair()
453 	struct k3_ringacc *ringacc = ring->parent;  in k3_ringacc_ring_reset_sci()  local
456 	ring_cfg.nav_id = ringacc->tisci_dev_id;  in k3_ringacc_ring_reset_sci()
461 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);  in k3_ringacc_ring_reset_sci()
463 		dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",  in k3_ringacc_ring_reset_sci()
482 	struct k3_ringacc *ringacc = ring->parent;  in k3_ringacc_ring_reconfig_qmode_sci()  local
485 	ring_cfg.nav_id = ringacc->tisci_dev_id;  in k3_ringacc_ring_reconfig_qmode_sci()
490 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);  in k3_ringacc_ring_reconfig_qmode_sci()
492 		dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",  in k3_ringacc_ring_reconfig_qmode_sci()
558 	struct k3_ringacc *ringacc = ring->parent;  in k3_ringacc_ring_free_sci()  local
561 	ring_cfg.nav_id = ringacc->tisci_dev_id;  in k3_ringacc_ring_free_sci()
565 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);  in k3_ringacc_ring_free_sci()
567 		dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",  in k3_ringacc_ring_free_sci()
573 	struct k3_ringacc *ringacc;  in k3_ringacc_ring_free()  local
578 	ringacc = ring->parent;  in k3_ringacc_ring_free()
584 	if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))  in k3_ringacc_ring_free()
589 	if (!test_bit(ring->ring_id, ringacc->rings_inuse))  in k3_ringacc_ring_free()
592 	mutex_lock(&ringacc->req_lock);  in k3_ringacc_ring_free()
611 		clear_bit(ring->proxy_id, ringacc->proxy_inuse);  in k3_ringacc_ring_free()
617 	clear_bit(ring->ring_id, ringacc->rings_inuse);  in k3_ringacc_ring_free()
620 	mutex_unlock(&ringacc->req_lock);  in k3_ringacc_ring_free()
660 	struct k3_ringacc *ringacc = ring->parent;  in k3_ringacc_ring_cfg_sci()  local
663 	if (!ringacc->tisci)  in k3_ringacc_ring_cfg_sci()
666 	ring_cfg.nav_id = ringacc->tisci_dev_id;  in k3_ringacc_ring_cfg_sci()
676 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);  in k3_ringacc_ring_cfg_sci()
678 		dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",  in k3_ringacc_ring_cfg_sci()
686 	struct k3_ringacc *ringacc;  in k3_dmaring_cfg()  local
695 	ringacc = ring->parent;  in k3_dmaring_cfg()
701 	if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))  in k3_dmaring_cfg()
704 	if (!test_bit(ring->ring_id, ringacc->rings_inuse))  in k3_dmaring_cfg()
713 		dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",  in k3_dmaring_cfg()
715 		ring->dma_dev = ringacc->dev;  in k3_dmaring_cfg()
726 		dev_err(ringacc->dev, "Failed to alloc ring mem\n");  in k3_dmaring_cfg()
740 	reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];  in k3_dmaring_cfg()
770 	struct k3_ringacc *ringacc;  in k3_ringacc_ring_cfg()  local
776 	ringacc = ring->parent;  in k3_ringacc_ring_cfg()
778 	if (ringacc->dma_rings)  in k3_ringacc_ring_cfg()
784 	    !test_bit(ring->ring_id, ringacc->rings_inuse))  in k3_ringacc_ring_cfg()
790 		dev_err(ringacc->dev,  in k3_ringacc_ring_cfg()
813 		ring->proxy = ringacc->proxy_target_base +  in k3_ringacc_ring_cfg()
821 			ring->dma_dev = ringacc->dev;  in k3_ringacc_ring_cfg()
824 		ring->dma_dev = ringacc->dev;  in k3_ringacc_ring_cfg()
840 		dev_err(ringacc->dev, "Failed to alloc ring mem\n");  in k3_ringacc_ring_cfg()
1270 	struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);  in of_k3_ringacc_get_by_phandle()  local
1280 			ringacc = entry;  in of_k3_ringacc_get_by_phandle()
1286 	return ringacc;  in of_k3_ringacc_get_by_phandle()
1290 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)  in k3_ringacc_probe_dt()  argument
1292 	struct device_node *node = ringacc->dev->of_node;  in k3_ringacc_probe_dt()
1293 	struct device *dev = ringacc->dev;  in k3_ringacc_probe_dt()
1302 	ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);  in k3_ringacc_probe_dt()
1308 	ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");  in k3_ringacc_probe_dt()
1309 	if (IS_ERR(ringacc->tisci)) {  in k3_ringacc_probe_dt()
1310 		ret = PTR_ERR(ringacc->tisci);  in k3_ringacc_probe_dt()
1313 		ringacc->tisci = NULL;  in k3_ringacc_probe_dt()
1318 				   &ringacc->tisci_dev_id);  in k3_ringacc_probe_dt()
1324 	pdev->id = ringacc->tisci_dev_id;  in k3_ringacc_probe_dt()
1326 	ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,  in k3_ringacc_probe_dt()
1327 						ringacc->tisci_dev_id,  in k3_ringacc_probe_dt()
1329 	if (IS_ERR(ringacc->rm_gp_range)) {  in k3_ringacc_probe_dt()
1331 		return PTR_ERR(ringacc->rm_gp_range);  in k3_ringacc_probe_dt()
1334 	return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,  in k3_ringacc_probe_dt()
1335 						 ringacc->rm_gp_range);  in k3_ringacc_probe_dt()
1351 			   struct k3_ringacc *ringacc)  in k3_ringacc_init()  argument
1366 	ret = k3_ringacc_probe_dt(ringacc);  in k3_ringacc_init()
1374 		ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;  in k3_ringacc_init()
1388 	ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);  in k3_ringacc_init()
1389 	if (IS_ERR(ringacc->proxy_gcfg))  in k3_ringacc_init()
1390 		return PTR_ERR(ringacc->proxy_gcfg);  in k3_ringacc_init()
1394 	ringacc->proxy_target_base = devm_ioremap_resource(dev, res);  in k3_ringacc_init()
1395 	if (IS_ERR(ringacc->proxy_target_base))  in k3_ringacc_init()
1396 		return PTR_ERR(ringacc->proxy_target_base);  in k3_ringacc_init()
1398 	ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &  in k3_ringacc_init()
1401 	ringacc->rings = devm_kzalloc(dev,  in k3_ringacc_init()
1402 				      sizeof(*ringacc->rings) *  in k3_ringacc_init()
1403 				      ringacc->num_rings,  in k3_ringacc_init()
1405 	ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings,  in k3_ringacc_init()
1407 	ringacc->proxy_inuse = devm_bitmap_zalloc(dev, ringacc->num_proxies,  in k3_ringacc_init()
1410 	if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)  in k3_ringacc_init()
1413 	for (i = 0; i < ringacc->num_rings; i++) {  in k3_ringacc_init()
1414 		ringacc->rings[i].rt = base_rt +  in k3_ringacc_init()
1416 		ringacc->rings[i].fifos = base_fifo +  in k3_ringacc_init()
1418 		ringacc->rings[i].parent = ringacc;  in k3_ringacc_init()
1419 		ringacc->rings[i].ring_id = i;  in k3_ringacc_init()
1420 		ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;  in k3_ringacc_init()
1423 	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;  in k3_ringacc_init()
1426 		 ringacc->num_rings,  in k3_ringacc_init()
1427 		 ringacc->rm_gp_range->desc[0].start,  in k3_ringacc_init()
1428 		 ringacc->rm_gp_range->desc[0].num,  in k3_ringacc_init()
1429 		 ringacc->tisci_dev_id);  in k3_ringacc_init()
1431 		 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");  in k3_ringacc_init()
1433 		 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);  in k3_ringacc_init()
1458 	struct k3_ringacc *ringacc;  in k3_ringacc_dmarings_init()  local
1463 	ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);  in k3_ringacc_dmarings_init()
1464 	if (!ringacc)  in k3_ringacc_dmarings_init()
1467 	ringacc->dev = dev;  in k3_ringacc_dmarings_init()
1468 	ringacc->dma_rings = true;  in k3_ringacc_dmarings_init()
1469 	ringacc->num_rings = data->num_rings;  in k3_ringacc_dmarings_init()
1470 	ringacc->tisci = data->tisci;  in k3_ringacc_dmarings_init()
1471 	ringacc->tisci_dev_id = data->tisci_dev_id;  in k3_ringacc_dmarings_init()
1473 	mutex_init(&ringacc->req_lock);  in k3_ringacc_dmarings_init()
1480 	ringacc->rings = devm_kzalloc(dev,  in k3_ringacc_dmarings_init()
1481 				      sizeof(*ringacc->rings) *  in k3_ringacc_dmarings_init()
1482 				      ringacc->num_rings * 2,  in k3_ringacc_dmarings_init()
1484 	ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings,  in k3_ringacc_dmarings_init()
1487 	if (!ringacc->rings || !ringacc->rings_inuse)  in k3_ringacc_dmarings_init()
1490 	for (i = 0; i < ringacc->num_rings; i++) {  in k3_ringacc_dmarings_init()
1491 		struct k3_ring *ring = &ringacc->rings[i];  in k3_ringacc_dmarings_init()
1494 		ring->parent = ringacc;  in k3_ringacc_dmarings_init()
1498 		ring = &ringacc->rings[ringacc->num_rings + i];  in k3_ringacc_dmarings_init()
1501 		ring->parent = ringacc;  in k3_ringacc_dmarings_init()
1507 	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;  in k3_ringacc_dmarings_init()
1509 	dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);  in k3_ringacc_dmarings_init()
1511 	return ringacc;  in k3_ringacc_dmarings_init()
1519 	struct k3_ringacc *ringacc;  in k3_ringacc_probe()  local
1526 	ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);  in k3_ringacc_probe()
1527 	if (!ringacc)  in k3_ringacc_probe()
1530 	ringacc->dev = dev;  in k3_ringacc_probe()
1531 	mutex_init(&ringacc->req_lock);  in k3_ringacc_probe()
1532 	ringacc->ops = &match_data->ops;  in k3_ringacc_probe()
1534 	ret = ringacc->ops->init(pdev, ringacc);  in k3_ringacc_probe()
1538 	dev_set_drvdata(dev, ringacc);  in k3_ringacc_probe()
1541 	list_add_tail(&ringacc->list, &k3_ringacc_list);  in k3_ringacc_probe()