37 	struct tegra_bpmp *bpmp = channel->bpmp;  in channel_to_ops()  local
39 	return bpmp->soc->ops;  in channel_to_ops()
45 	struct tegra_bpmp *bpmp;  in tegra_bpmp_get()  local
54 		bpmp = ERR_PTR(-ENODEV);  in tegra_bpmp_get()
58 	bpmp = platform_get_drvdata(pdev);  in tegra_bpmp_get()
59 	if (!bpmp) {  in tegra_bpmp_get()
60 		bpmp = ERR_PTR(-EPROBE_DEFER);  in tegra_bpmp_get()
67 	return bpmp;  in tegra_bpmp_get()
71 void tegra_bpmp_put(struct tegra_bpmp *bpmp)  in tegra_bpmp_put()  argument
73 	if (bpmp)  in tegra_bpmp_put()
74 		put_device(bpmp->dev);  in tegra_bpmp_put()
81 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()  local
85 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_channel_get_thread_index()
87 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
118 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_response()
164 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_request_channel_free()
193 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)  in tegra_bpmp_ring_doorbell()  argument
195 	return bpmp->soc->ops->ring_doorbell(bpmp);  in tegra_bpmp_ring_doorbell()
218 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()  local
229 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
231 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_channel_read()
232 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
235 	up(&bpmp->threaded.lock);  in tegra_bpmp_channel_read()
254 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,  in tegra_bpmp_write_threaded()  argument
257 	unsigned long timeout = bpmp->soc->channels.thread.timeout;  in tegra_bpmp_write_threaded()
258 	unsigned int count = bpmp->soc->channels.thread.count;  in tegra_bpmp_write_threaded()
264 	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));  in tegra_bpmp_write_threaded()
268 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
270 	index = find_first_zero_bit(bpmp->threaded.allocated, count);  in tegra_bpmp_write_threaded()
276 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
283 	set_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
290 	set_bit(index, bpmp->threaded.busy);  in tegra_bpmp_write_threaded()
292 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
296 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
298 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
299 	up(&bpmp->threaded.lock);  in tegra_bpmp_write_threaded()
317 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,  in tegra_bpmp_transfer_atomic()  argument
329 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
331 	spin_lock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
336 		spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
340 	spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
342 	err = tegra_bpmp_ring_doorbell(bpmp);  in tegra_bpmp_transfer_atomic()
355 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,  in tegra_bpmp_transfer()  argument
368 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
373 	err = tegra_bpmp_ring_doorbell(bpmp);  in tegra_bpmp_transfer()
377 	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);  in tegra_bpmp_transfer()
388 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,  in tegra_bpmp_find_mrq()  argument
393 	list_for_each_entry(entry, &bpmp->mrqs, list)  in tegra_bpmp_find_mrq()
404 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()  local
430 		err = tegra_bpmp_ring_doorbell(bpmp);  in tegra_bpmp_mrq_return()
437 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,  in tegra_bpmp_handle_mrq()  argument
444 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
446 	entry = tegra_bpmp_find_mrq(bpmp, mrq);  in tegra_bpmp_handle_mrq()
448 		spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
455 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
458 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,  in tegra_bpmp_request_mrq()  argument
467 	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);  in tegra_bpmp_request_mrq()
471 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
476 	list_add(&entry->list, &bpmp->mrqs);  in tegra_bpmp_request_mrq()
478 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
484 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)  in tegra_bpmp_free_mrq()  argument
489 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
491 	entry = tegra_bpmp_find_mrq(bpmp, mrq);  in tegra_bpmp_free_mrq()
496 	devm_kfree(bpmp->dev, entry);  in tegra_bpmp_free_mrq()
499 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
503 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)  in tegra_bpmp_mrq_is_supported()  argument
520 	err = tegra_bpmp_transfer(bpmp, &msg);  in tegra_bpmp_mrq_is_supported()
543 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)  in tegra_bpmp_ping()  argument
566 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);  in tegra_bpmp_ping()
571 		dev_dbg(bpmp->dev,  in tegra_bpmp_ping()
580 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,  in tegra_bpmp_get_firmware_tag_old()  argument
593 	virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,  in tegra_bpmp_get_firmware_tag_old()
607 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);  in tegra_bpmp_get_firmware_tag_old()
613 	dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);  in tegra_bpmp_get_firmware_tag_old()
618 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,  in tegra_bpmp_get_firmware_tag()  argument
621 	if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {  in tegra_bpmp_get_firmware_tag()
635 		err = tegra_bpmp_transfer(bpmp, &msg);  in tegra_bpmp_get_firmware_tag()
646 	return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);  in tegra_bpmp_get_firmware_tag()
659 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)  in tegra_bpmp_handle_rx()  argument
665 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
666 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_handle_rx()
667 	busy = bpmp->threaded.busy;  in tegra_bpmp_handle_rx()
670 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);  in tegra_bpmp_handle_rx()
672 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_rx()
677 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
685 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_rx()
690 	struct tegra_bpmp *bpmp;  in tegra_bpmp_probe()  local
695 	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);  in tegra_bpmp_probe()
696 	if (!bpmp)  in tegra_bpmp_probe()
699 	bpmp->soc = of_device_get_match_data(&pdev->dev);  in tegra_bpmp_probe()
700 	bpmp->dev = &pdev->dev;  in tegra_bpmp_probe()
702 	INIT_LIST_HEAD(&bpmp->mrqs);  in tegra_bpmp_probe()
703 	spin_lock_init(&bpmp->lock);  in tegra_bpmp_probe()
705 	bpmp->threaded.count = bpmp->soc->channels.thread.count;  in tegra_bpmp_probe()
706 	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);  in tegra_bpmp_probe()
708 	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);  in tegra_bpmp_probe()
710 	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
711 	if (!bpmp->threaded.allocated)  in tegra_bpmp_probe()
714 	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
715 	if (!bpmp->threaded.busy)  in tegra_bpmp_probe()
718 	spin_lock_init(&bpmp->atomic_tx_lock);  in tegra_bpmp_probe()
719 	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),  in tegra_bpmp_probe()
721 	if (!bpmp->tx_channel)  in tegra_bpmp_probe()
724 	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),  in tegra_bpmp_probe()
726 	if (!bpmp->rx_channel)  in tegra_bpmp_probe()
729 	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,  in tegra_bpmp_probe()
730 					       sizeof(*bpmp->threaded_channels),  in tegra_bpmp_probe()
732 	if (!bpmp->threaded_channels)  in tegra_bpmp_probe()
735 	err = bpmp->soc->ops->init(bpmp);  in tegra_bpmp_probe()
739 	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,  in tegra_bpmp_probe()
740 				     tegra_bpmp_mrq_handle_ping, bpmp);  in tegra_bpmp_probe()
744 	err = tegra_bpmp_ping(bpmp);  in tegra_bpmp_probe()
750 	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));  in tegra_bpmp_probe()
758 	platform_set_drvdata(pdev, bpmp);  in tegra_bpmp_probe()
765 		err = tegra_bpmp_init_clocks(bpmp);  in tegra_bpmp_probe()
771 		err = tegra_bpmp_init_resets(bpmp);  in tegra_bpmp_probe()
777 		err = tegra_bpmp_init_powergates(bpmp);  in tegra_bpmp_probe()
782 	err = tegra_bpmp_init_debugfs(bpmp);  in tegra_bpmp_probe()
789 	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);  in tegra_bpmp_probe()
791 	if (bpmp->soc->ops->deinit)  in tegra_bpmp_probe()
792 		bpmp->soc->ops->deinit(bpmp);  in tegra_bpmp_probe()
799 	struct tegra_bpmp *bpmp = dev_get_drvdata(dev);  in tegra_bpmp_resume()  local
801 	if (bpmp->soc->ops->resume)  in tegra_bpmp_resume()
802 		return bpmp->soc->ops->resume(bpmp);  in tegra_bpmp_resume()