40 	struct tegra_bpmp *bpmp;  in tegra_bpmp_get()  local
49 		bpmp = ERR_PTR(-ENODEV);  in tegra_bpmp_get()
53 	bpmp = platform_get_drvdata(pdev);  in tegra_bpmp_get()
54 	if (!bpmp) {  in tegra_bpmp_get()
55 		bpmp = ERR_PTR(-EPROBE_DEFER);  in tegra_bpmp_get()
62 	return bpmp;  in tegra_bpmp_get()
66 void tegra_bpmp_put(struct tegra_bpmp *bpmp)  in tegra_bpmp_put()  argument
68 	if (bpmp)  in tegra_bpmp_put()
69 		put_device(bpmp->dev);  in tegra_bpmp_put()
76 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()  local
80 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_channel_get_thread_index()
82 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
114 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_ack()
144 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_master_free()
179 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()  local
190 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
192 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_channel_read()
193 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
196 	up(&bpmp->threaded.lock);  in tegra_bpmp_channel_read()
215 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,  in tegra_bpmp_write_threaded()  argument
218 	unsigned long timeout = bpmp->soc->channels.thread.timeout;  in tegra_bpmp_write_threaded()
219 	unsigned int count = bpmp->soc->channels.thread.count;  in tegra_bpmp_write_threaded()
225 	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));  in tegra_bpmp_write_threaded()
229 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
231 	index = find_first_zero_bit(bpmp->threaded.allocated, count);  in tegra_bpmp_write_threaded()
237 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
244 	set_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
251 	set_bit(index, bpmp->threaded.busy);  in tegra_bpmp_write_threaded()
253 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
257 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
259 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
260 	up(&bpmp->threaded.lock);  in tegra_bpmp_write_threaded()
278 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,  in tegra_bpmp_transfer_atomic()  argument
290 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
292 	spin_lock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
297 		spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
301 	spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
303 	err = mbox_send_message(bpmp->mbox.channel, NULL);  in tegra_bpmp_transfer_atomic()
307 	mbox_client_txdone(bpmp->mbox.channel, 0);  in tegra_bpmp_transfer_atomic()
318 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,  in tegra_bpmp_transfer()  argument
331 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
336 	err = mbox_send_message(bpmp->mbox.channel, NULL);  in tegra_bpmp_transfer()
340 	mbox_client_txdone(bpmp->mbox.channel, 0);  in tegra_bpmp_transfer()
342 	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);  in tegra_bpmp_transfer()
353 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,  in tegra_bpmp_find_mrq()  argument
358 	list_for_each_entry(entry, &bpmp->mrqs, list)  in tegra_bpmp_find_mrq()
369 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()  local
397 		err = mbox_send_message(bpmp->mbox.channel, NULL);  in tegra_bpmp_mrq_return()
401 		mbox_client_txdone(bpmp->mbox.channel, 0);  in tegra_bpmp_mrq_return()
406 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,  in tegra_bpmp_handle_mrq()  argument
413 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
415 	entry = tegra_bpmp_find_mrq(bpmp, mrq);  in tegra_bpmp_handle_mrq()
417 		spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
424 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
427 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,  in tegra_bpmp_request_mrq()  argument
436 	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);  in tegra_bpmp_request_mrq()
440 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
445 	list_add(&entry->list, &bpmp->mrqs);  in tegra_bpmp_request_mrq()
447 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
453 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)  in tegra_bpmp_free_mrq()  argument
458 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
460 	entry = tegra_bpmp_find_mrq(bpmp, mrq);  in tegra_bpmp_free_mrq()
465 	devm_kfree(bpmp->dev, entry);  in tegra_bpmp_free_mrq()
468 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
487 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)  in tegra_bpmp_ping()  argument
510 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);  in tegra_bpmp_ping()
515 		dev_dbg(bpmp->dev,  in tegra_bpmp_ping()
523 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,  in tegra_bpmp_get_firmware_tag()  argument
533 	virt = dma_alloc_coherent(bpmp->dev, MSG_DATA_MIN_SZ, &phys,  in tegra_bpmp_get_firmware_tag()
547 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);  in tegra_bpmp_get_firmware_tag()
553 	dma_free_coherent(bpmp->dev, MSG_DATA_MIN_SZ, virt, phys);  in tegra_bpmp_get_firmware_tag()
570 	struct tegra_bpmp *bpmp = mbox_client_to_bpmp(client);  in tegra_bpmp_handle_rx()  local
575 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
576 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_handle_rx()
577 	busy = bpmp->threaded.busy;  in tegra_bpmp_handle_rx()
580 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);  in tegra_bpmp_handle_rx()
582 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_rx()
587 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
595 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_rx()
600 	struct tegra_bpmp *bpmp = data;  in tegra_bpmp_ivc_notify()  local
603 	if (WARN_ON(bpmp->mbox.channel == NULL))  in tegra_bpmp_ivc_notify()
606 	err = mbox_send_message(bpmp->mbox.channel, NULL);  in tegra_bpmp_ivc_notify()
610 	mbox_client_txdone(bpmp->mbox.channel, 0);  in tegra_bpmp_ivc_notify()
614 				   struct tegra_bpmp *bpmp,  in tegra_bpmp_channel_init()  argument
621 	channel->ivc = devm_kzalloc(bpmp->dev, sizeof(*channel->ivc),  in tegra_bpmp_channel_init()
631 			     bpmp->rx.virt + offset, bpmp->rx.phys + offset,  in tegra_bpmp_channel_init()
632 			     bpmp->tx.virt + offset, bpmp->tx.phys + offset,  in tegra_bpmp_channel_init()
634 			     bpmp);  in tegra_bpmp_channel_init()
636 		dev_err(bpmp->dev, "failed to setup IVC for channel %u: %d\n",  in tegra_bpmp_channel_init()
642 	channel->bpmp = bpmp;  in tegra_bpmp_channel_init()
664 	struct tegra_bpmp *bpmp;  in tegra_bpmp_probe()  local
670 	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);  in tegra_bpmp_probe()
671 	if (!bpmp)  in tegra_bpmp_probe()
674 	bpmp->soc = of_device_get_match_data(&pdev->dev);  in tegra_bpmp_probe()
675 	bpmp->dev = &pdev->dev;  in tegra_bpmp_probe()
677 	bpmp->tx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 0);  in tegra_bpmp_probe()
678 	if (!bpmp->tx.pool) {  in tegra_bpmp_probe()
683 	bpmp->tx.virt = gen_pool_dma_alloc(bpmp->tx.pool, 4096, &bpmp->tx.phys);  in tegra_bpmp_probe()
684 	if (!bpmp->tx.virt) {  in tegra_bpmp_probe()
689 	bpmp->rx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 1);  in tegra_bpmp_probe()
690 	if (!bpmp->rx.pool) {  in tegra_bpmp_probe()
696 	bpmp->rx.virt = gen_pool_dma_alloc(bpmp->rx.pool, 4096, &bpmp->rx.phys);  in tegra_bpmp_probe()
697 	if (!bpmp->rx.virt) {  in tegra_bpmp_probe()
703 	INIT_LIST_HEAD(&bpmp->mrqs);  in tegra_bpmp_probe()
704 	spin_lock_init(&bpmp->lock);  in tegra_bpmp_probe()
706 	bpmp->threaded.count = bpmp->soc->channels.thread.count;  in tegra_bpmp_probe()
707 	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);  in tegra_bpmp_probe()
709 	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);  in tegra_bpmp_probe()
711 	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
712 	if (!bpmp->threaded.allocated) {  in tegra_bpmp_probe()
717 	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
718 	if (!bpmp->threaded.busy) {  in tegra_bpmp_probe()
723 	spin_lock_init(&bpmp->atomic_tx_lock);  in tegra_bpmp_probe()
724 	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),  in tegra_bpmp_probe()
726 	if (!bpmp->tx_channel) {  in tegra_bpmp_probe()
731 	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),  in tegra_bpmp_probe()
733 	if (!bpmp->rx_channel) {  in tegra_bpmp_probe()
738 	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,  in tegra_bpmp_probe()
739 					       sizeof(*bpmp->threaded_channels),  in tegra_bpmp_probe()
741 	if (!bpmp->threaded_channels) {  in tegra_bpmp_probe()
746 	err = tegra_bpmp_channel_init(bpmp->tx_channel, bpmp,  in tegra_bpmp_probe()
747 				      bpmp->soc->channels.cpu_tx.offset);  in tegra_bpmp_probe()
751 	err = tegra_bpmp_channel_init(bpmp->rx_channel, bpmp,  in tegra_bpmp_probe()
752 				      bpmp->soc->channels.cpu_rx.offset);  in tegra_bpmp_probe()
756 	for (i = 0; i < bpmp->threaded.count; i++) {  in tegra_bpmp_probe()
758 			&bpmp->threaded_channels[i], bpmp,  in tegra_bpmp_probe()
759 			bpmp->soc->channels.thread.offset + i);  in tegra_bpmp_probe()
765 	bpmp->mbox.client.dev = &pdev->dev;  in tegra_bpmp_probe()
766 	bpmp->mbox.client.rx_callback = tegra_bpmp_handle_rx;  in tegra_bpmp_probe()
767 	bpmp->mbox.client.tx_block = false;  in tegra_bpmp_probe()
768 	bpmp->mbox.client.knows_txdone = false;  in tegra_bpmp_probe()
770 	bpmp->mbox.channel = mbox_request_channel(&bpmp->mbox.client, 0);  in tegra_bpmp_probe()
771 	if (IS_ERR(bpmp->mbox.channel)) {  in tegra_bpmp_probe()
772 		err = PTR_ERR(bpmp->mbox.channel);  in tegra_bpmp_probe()
778 	tegra_bpmp_channel_reset(bpmp->tx_channel);  in tegra_bpmp_probe()
779 	tegra_bpmp_channel_reset(bpmp->rx_channel);  in tegra_bpmp_probe()
780 	for (i = 0; i < bpmp->threaded.count; i++)  in tegra_bpmp_probe()
781 		tegra_bpmp_channel_reset(&bpmp->threaded_channels[i]);  in tegra_bpmp_probe()
783 	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,  in tegra_bpmp_probe()
784 				     tegra_bpmp_mrq_handle_ping, bpmp);  in tegra_bpmp_probe()
788 	err = tegra_bpmp_ping(bpmp);  in tegra_bpmp_probe()
794 	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag) - 1);  in tegra_bpmp_probe()
802 	platform_set_drvdata(pdev, bpmp);  in tegra_bpmp_probe()
808 	err = tegra_bpmp_init_clocks(bpmp);  in tegra_bpmp_probe()
812 	err = tegra_bpmp_init_resets(bpmp);  in tegra_bpmp_probe()
816 	err = tegra_bpmp_init_powergates(bpmp);  in tegra_bpmp_probe()
820 	err = tegra_bpmp_init_debugfs(bpmp);  in tegra_bpmp_probe()
827 	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);  in tegra_bpmp_probe()
829 	mbox_free_channel(bpmp->mbox.channel);  in tegra_bpmp_probe()
831 	for (i = 0; i < bpmp->threaded.count; i++) {  in tegra_bpmp_probe()
832 		if (bpmp->threaded_channels[i].bpmp)  in tegra_bpmp_probe()
833 			tegra_bpmp_channel_cleanup(&bpmp->threaded_channels[i]);  in tegra_bpmp_probe()
836 	tegra_bpmp_channel_cleanup(bpmp->rx_channel);  in tegra_bpmp_probe()
838 	tegra_bpmp_channel_cleanup(bpmp->tx_channel);  in tegra_bpmp_probe()
840 	gen_pool_free(bpmp->rx.pool, (unsigned long)bpmp->rx.virt, 4096);  in tegra_bpmp_probe()
842 	gen_pool_free(bpmp->tx.pool, (unsigned long)bpmp->tx.virt, 4096);  in tegra_bpmp_probe()