1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (c) 2016-2018, NVIDIA CORPORATION.  All rights reserved.
18 #include <dt-bindings/mailbox/tegra186-hsp.h>
113 	return readl(hsp->regs + offset);  in tegra_hsp_readl()
119 	writel(value, hsp->regs + offset);  in tegra_hsp_writel()
125 	return readl(channel->regs + offset);  in tegra_hsp_channel_readl()
131 	writel(value, channel->regs + offset);  in tegra_hsp_channel_writel()
138 	value = tegra_hsp_channel_readl(&db->channel, HSP_DB_ENABLE);  in tegra_hsp_doorbell_can_ring()
148 	list_for_each_entry(entry, &hsp->doorbells, list)  in __tegra_hsp_doorbell_get()
149 		if (entry->master == master)  in __tegra_hsp_doorbell_get()
161 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_doorbell_get()
163 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_doorbell_get()
178 	value = tegra_hsp_channel_readl(&db->channel, HSP_DB_PENDING);  in tegra_hsp_doorbell_irq()
179 	tegra_hsp_channel_writel(&db->channel, value, HSP_DB_PENDING);  in tegra_hsp_doorbell_irq()
181 	spin_lock(&hsp->lock);  in tegra_hsp_doorbell_irq()
183 	for_each_set_bit(master, &value, hsp->mbox_db.num_chans) {  in tegra_hsp_doorbell_irq()
192 		 * In that case, db->channel.chan will still be NULL here and  in tegra_hsp_doorbell_irq()
198 		if (db && db->channel.chan)  in tegra_hsp_doorbell_irq()
199 			mbox_chan_received_data(db->channel.chan, NULL);  in tegra_hsp_doorbell_irq()
202 	spin_unlock(&hsp->lock);  in tegra_hsp_doorbell_irq()
214 	status = tegra_hsp_readl(hsp, HSP_INT_IR) & hsp->mask;  in tegra_hsp_shared_irq()
219 	for_each_set_bit(bit, &mask, hsp->num_sm) {  in tegra_hsp_shared_irq()
220 		struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit];  in tegra_hsp_shared_irq()
222 		if (mb->producer) {  in tegra_hsp_shared_irq()
225 			 * the next message. These interrupts are level-  in tegra_hsp_shared_irq()
230 			spin_lock(&hsp->lock);  in tegra_hsp_shared_irq()
232 			hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index);  in tegra_hsp_shared_irq()
233 			tegra_hsp_writel(hsp, hsp->mask,  in tegra_hsp_shared_irq()
234 					 HSP_INT_IE(hsp->shared_irq));  in tegra_hsp_shared_irq()
236 			spin_unlock(&hsp->lock);  in tegra_hsp_shared_irq()
238 			mbox_chan_txdone(mb->channel.chan, 0);  in tegra_hsp_shared_irq()
245 	for_each_set_bit(bit, &mask, hsp->num_sm) {  in tegra_hsp_shared_irq()
246 		struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit];  in tegra_hsp_shared_irq()
248 		if (!mb->producer) {  in tegra_hsp_shared_irq()
249 			value = tegra_hsp_channel_readl(&mb->channel,  in tegra_hsp_shared_irq()
253 			mbox_chan_received_data(mb->channel.chan, msg);  in tegra_hsp_shared_irq()
264 			tegra_hsp_channel_writel(&mb->channel, 0x0,  in tegra_hsp_shared_irq()
280 	db = devm_kzalloc(hsp->dev, sizeof(*db), GFP_KERNEL);  in tegra_hsp_doorbell_create()
282 		return ERR_PTR(-ENOMEM);  in tegra_hsp_doorbell_create()
284 	offset = (1 + (hsp->num_sm / 2) + hsp->num_ss + hsp->num_as) * SZ_64K;  in tegra_hsp_doorbell_create()
287 	db->channel.regs = hsp->regs + offset;  in tegra_hsp_doorbell_create()
288 	db->channel.hsp = hsp;  in tegra_hsp_doorbell_create()
290 	db->name = devm_kstrdup_const(hsp->dev, name, GFP_KERNEL);  in tegra_hsp_doorbell_create()
291 	db->master = master;  in tegra_hsp_doorbell_create()
292 	db->index = index;  in tegra_hsp_doorbell_create()
294 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_doorbell_create()
295 	list_add_tail(&db->list, &hsp->doorbells);  in tegra_hsp_doorbell_create()
296 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_doorbell_create()
298 	return &db->channel;  in tegra_hsp_doorbell_create()
303 	struct tegra_hsp_doorbell *db = chan->con_priv;  in tegra_hsp_doorbell_send_data()
305 	tegra_hsp_channel_writel(&db->channel, 1, HSP_DB_TRIGGER);  in tegra_hsp_doorbell_send_data()
312 	struct tegra_hsp_doorbell *db = chan->con_priv;  in tegra_hsp_doorbell_startup()
313 	struct tegra_hsp *hsp = db->channel.hsp;  in tegra_hsp_doorbell_startup()
318 	if (db->master >= chan->mbox->num_chans) {  in tegra_hsp_doorbell_startup()
319 		dev_err(chan->mbox->dev,  in tegra_hsp_doorbell_startup()
321 			db->master);  in tegra_hsp_doorbell_startup()
322 		return -EINVAL;  in tegra_hsp_doorbell_startup()
327 		return -ENODEV;  in tegra_hsp_doorbell_startup()
330 	 * On simulation platforms the BPMP hasn't had a chance yet to mark  in tegra_hsp_doorbell_startup()
335 		return -ENODEV;  in tegra_hsp_doorbell_startup()
337 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_doorbell_startup()
339 	value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE);  in tegra_hsp_doorbell_startup()
340 	value |= BIT(db->master);  in tegra_hsp_doorbell_startup()
341 	tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE);  in tegra_hsp_doorbell_startup()
343 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_doorbell_startup()
350 	struct tegra_hsp_doorbell *db = chan->con_priv;  in tegra_hsp_doorbell_shutdown()
351 	struct tegra_hsp *hsp = db->channel.hsp;  in tegra_hsp_doorbell_shutdown()
360 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_doorbell_shutdown()
362 	value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE);  in tegra_hsp_doorbell_shutdown()
363 	value &= ~BIT(db->master);  in tegra_hsp_doorbell_shutdown()
364 	tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE);  in tegra_hsp_doorbell_shutdown()
366 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_doorbell_shutdown()
377 	struct tegra_hsp_mailbox *mb = chan->con_priv;  in tegra_hsp_mailbox_send_data()
378 	struct tegra_hsp *hsp = mb->channel.hsp;  in tegra_hsp_mailbox_send_data()
382 	if (WARN_ON(!mb->producer))  in tegra_hsp_mailbox_send_data()
383 		return -EPERM;  in tegra_hsp_mailbox_send_data()
389 	tegra_hsp_channel_writel(&mb->channel, value, HSP_SM_SHRD_MBOX);  in tegra_hsp_mailbox_send_data()
392 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_mailbox_send_data()
394 	hsp->mask |= BIT(HSP_INT_EMPTY_SHIFT + mb->index);  in tegra_hsp_mailbox_send_data()
395 	tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq));  in tegra_hsp_mailbox_send_data()
397 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_mailbox_send_data()
405 	struct tegra_hsp_mailbox *mb = chan->con_priv;  in tegra_hsp_mailbox_flush()
406 	struct tegra_hsp_channel *ch = &mb->channel;  in tegra_hsp_mailbox_flush()
421 	return -ETIME;  in tegra_hsp_mailbox_flush()
426 	struct tegra_hsp_mailbox *mb = chan->con_priv;  in tegra_hsp_mailbox_startup()
427 	struct tegra_hsp_channel *ch = &mb->channel;  in tegra_hsp_mailbox_startup()
428 	struct tegra_hsp *hsp = mb->channel.hsp;  in tegra_hsp_mailbox_startup()
431 	chan->txdone_method = TXDONE_BY_IRQ;  in tegra_hsp_mailbox_startup()
439 	 * EMPTY interrupts are level-triggered, so keeping EMPTY interrupts  in tegra_hsp_mailbox_startup()
444 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_mailbox_startup()
446 	if (mb->producer)  in tegra_hsp_mailbox_startup()
447 		hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index);  in tegra_hsp_mailbox_startup()
449 		hsp->mask |= BIT(HSP_INT_FULL_SHIFT + mb->index);  in tegra_hsp_mailbox_startup()
451 	tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq));  in tegra_hsp_mailbox_startup()
453 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_mailbox_startup()
455 	if (hsp->soc->has_per_mb_ie) {  in tegra_hsp_mailbox_startup()
456 		if (mb->producer)  in tegra_hsp_mailbox_startup()
469 	struct tegra_hsp_mailbox *mb = chan->con_priv;  in tegra_hsp_mailbox_shutdown()
470 	struct tegra_hsp_channel *ch = &mb->channel;  in tegra_hsp_mailbox_shutdown()
471 	struct tegra_hsp *hsp = mb->channel.hsp;  in tegra_hsp_mailbox_shutdown()
474 	if (hsp->soc->has_per_mb_ie) {  in tegra_hsp_mailbox_shutdown()
475 		if (mb->producer)  in tegra_hsp_mailbox_shutdown()
483 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_mailbox_shutdown()
485 	if (mb->producer)  in tegra_hsp_mailbox_shutdown()
486 		hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index);  in tegra_hsp_mailbox_shutdown()
488 		hsp->mask &= ~BIT(HSP_INT_FULL_SHIFT + mb->index);  in tegra_hsp_mailbox_shutdown()
490 	tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq));  in tegra_hsp_mailbox_shutdown()
492 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_mailbox_shutdown()
506 	unsigned int type = args->args[0], master = args->args[1];  in tegra_hsp_db_xlate()
507 	struct tegra_hsp_channel *channel = ERR_PTR(-ENODEV);  in tegra_hsp_db_xlate()
513 	if (type != TEGRA_HSP_MBOX_TYPE_DB || !hsp->doorbell_irq)  in tegra_hsp_db_xlate()
514 		return ERR_PTR(-ENODEV);  in tegra_hsp_db_xlate()
518 		channel = &db->channel;  in tegra_hsp_db_xlate()
523 	spin_lock_irqsave(&hsp->lock, flags);  in tegra_hsp_db_xlate()
525 	for (i = 0; i < mbox->num_chans; i++) {  in tegra_hsp_db_xlate()
526 		chan = &mbox->chans[i];  in tegra_hsp_db_xlate()
527 		if (!chan->con_priv) {  in tegra_hsp_db_xlate()
528 			channel->chan = chan;  in tegra_hsp_db_xlate()
529 			chan->con_priv = db;  in tegra_hsp_db_xlate()
536 	spin_unlock_irqrestore(&hsp->lock, flags);  in tegra_hsp_db_xlate()
538 	return chan ?: ERR_PTR(-EBUSY);  in tegra_hsp_db_xlate()
545 	unsigned int type = args->args[0], index;  in tegra_hsp_sm_xlate()
548 	index = args->args[1] & TEGRA_HSP_SM_MASK;  in tegra_hsp_sm_xlate()
550 	if (type != TEGRA_HSP_MBOX_TYPE_SM || !hsp->shared_irqs ||  in tegra_hsp_sm_xlate()
551 	    index >= hsp->num_sm)  in tegra_hsp_sm_xlate()
552 		return ERR_PTR(-ENODEV);  in tegra_hsp_sm_xlate()
554 	mb = &hsp->mailboxes[index];  in tegra_hsp_sm_xlate()
556 	if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0)  in tegra_hsp_sm_xlate()
557 		mb->producer = false;  in tegra_hsp_sm_xlate()
559 		mb->producer = true;  in tegra_hsp_sm_xlate()
561 	return mb->channel.chan;  in tegra_hsp_sm_xlate()
566 	const struct tegra_hsp_db_map *map = hsp->soc->map;  in tegra_hsp_add_doorbells()
569 	while (map->name) {  in tegra_hsp_add_doorbells()
570 		channel = tegra_hsp_doorbell_create(hsp, map->name,  in tegra_hsp_add_doorbells()
571 						    map->master, map->index);  in tegra_hsp_add_doorbells()
585 	hsp->mailboxes = devm_kcalloc(dev, hsp->num_sm, sizeof(*hsp->mailboxes),  in tegra_hsp_add_mailboxes()
587 	if (!hsp->mailboxes)  in tegra_hsp_add_mailboxes()
588 		return -ENOMEM;  in tegra_hsp_add_mailboxes()
590 	for (i = 0; i < hsp->num_sm; i++) {  in tegra_hsp_add_mailboxes()
591 		struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i];  in tegra_hsp_add_mailboxes()
593 		mb->index = i;  in tegra_hsp_add_mailboxes()
595 		mb->channel.hsp = hsp;  in tegra_hsp_add_mailboxes()
596 		mb->channel.regs = hsp->regs + SZ_64K + i * SZ_32K;  in tegra_hsp_add_mailboxes()
597 		mb->channel.chan = &hsp->mbox_sm.chans[i];  in tegra_hsp_add_mailboxes()
598 		mb->channel.chan->con_priv = mb;  in tegra_hsp_add_mailboxes()
609 	for (i = 0; i < hsp->num_si; i++) {  in tegra_hsp_request_shared_irq()
610 		irq = hsp->shared_irqs[i];  in tegra_hsp_request_shared_irq()
614 		err = devm_request_irq(hsp->dev, irq, tegra_hsp_shared_irq, 0,  in tegra_hsp_request_shared_irq()
615 				       dev_name(hsp->dev), hsp);  in tegra_hsp_request_shared_irq()
617 			dev_err(hsp->dev, "failed to request interrupt: %d\n",  in tegra_hsp_request_shared_irq()
622 		hsp->shared_irq = i;  in tegra_hsp_request_shared_irq()
625 		tegra_hsp_writel(hsp, 0, HSP_INT_IE(hsp->shared_irq));  in tegra_hsp_request_shared_irq()
627 		dev_dbg(hsp->dev, "interrupt requested: %u\n", irq);  in tegra_hsp_request_shared_irq()
632 	if (i == hsp->num_si) {  in tegra_hsp_request_shared_irq()
633 		dev_err(hsp->dev, "failed to find available interrupt\n");  in tegra_hsp_request_shared_irq()
634 		return -ENOENT;  in tegra_hsp_request_shared_irq()
648 	hsp = devm_kzalloc(&pdev->dev, sizeof(*hsp), GFP_KERNEL);  in tegra_hsp_probe()
650 		return -ENOMEM;  in tegra_hsp_probe()
652 	hsp->dev = &pdev->dev;  in tegra_hsp_probe()
653 	hsp->soc = of_device_get_match_data(&pdev->dev);  in tegra_hsp_probe()
654 	INIT_LIST_HEAD(&hsp->doorbells);  in tegra_hsp_probe()
655 	spin_lock_init(&hsp->lock);  in tegra_hsp_probe()
658 	hsp->regs = devm_ioremap_resource(&pdev->dev, res);  in tegra_hsp_probe()
659 	if (IS_ERR(hsp->regs))  in tegra_hsp_probe()
660 		return PTR_ERR(hsp->regs);  in tegra_hsp_probe()
663 	hsp->num_sm = (value >> HSP_nSM_SHIFT) & HSP_nINT_MASK;  in tegra_hsp_probe()
664 	hsp->num_ss = (value >> HSP_nSS_SHIFT) & HSP_nINT_MASK;  in tegra_hsp_probe()
665 	hsp->num_as = (value >> HSP_nAS_SHIFT) & HSP_nINT_MASK;  in tegra_hsp_probe()
666 	hsp->num_db = (value >> HSP_nDB_SHIFT) & HSP_nINT_MASK;  in tegra_hsp_probe()
667 	hsp->num_si = (value >> HSP_nSI_SHIFT) & HSP_nINT_MASK;  in tegra_hsp_probe()
671 		hsp->doorbell_irq = err;  in tegra_hsp_probe()
673 	if (hsp->num_si > 0) {  in tegra_hsp_probe()
676 		hsp->shared_irqs = devm_kcalloc(&pdev->dev, hsp->num_si,  in tegra_hsp_probe()
677 						sizeof(*hsp->shared_irqs),  in tegra_hsp_probe()
679 		if (!hsp->shared_irqs)  in tegra_hsp_probe()
680 			return -ENOMEM;  in tegra_hsp_probe()
682 		for (i = 0; i < hsp->num_si; i++) {  in tegra_hsp_probe()
687 				return -ENOMEM;  in tegra_hsp_probe()
691 				hsp->shared_irqs[i] = err;  in tegra_hsp_probe()
699 			devm_kfree(&pdev->dev, hsp->shared_irqs);  in tegra_hsp_probe()
700 			hsp->shared_irqs = NULL;  in tegra_hsp_probe()
705 	hsp->mbox_db.of_xlate = tegra_hsp_db_xlate;  in tegra_hsp_probe()
706 	hsp->mbox_db.num_chans = 32;  in tegra_hsp_probe()
707 	hsp->mbox_db.dev = &pdev->dev;  in tegra_hsp_probe()
708 	hsp->mbox_db.ops = &tegra_hsp_db_ops;  in tegra_hsp_probe()
710 	hsp->mbox_db.chans = devm_kcalloc(&pdev->dev, hsp->mbox_db.num_chans,  in tegra_hsp_probe()
711 					  sizeof(*hsp->mbox_db.chans),  in tegra_hsp_probe()
713 	if (!hsp->mbox_db.chans)  in tegra_hsp_probe()
714 		return -ENOMEM;  in tegra_hsp_probe()
716 	if (hsp->doorbell_irq) {  in tegra_hsp_probe()
719 			dev_err(&pdev->dev, "failed to add doorbells: %d\n",  in tegra_hsp_probe()
725 	err = devm_mbox_controller_register(&pdev->dev, &hsp->mbox_db);  in tegra_hsp_probe()
727 		dev_err(&pdev->dev, "failed to register doorbell mailbox: %d\n",  in tegra_hsp_probe()
733 	hsp->mbox_sm.of_xlate = tegra_hsp_sm_xlate;  in tegra_hsp_probe()
734 	hsp->mbox_sm.num_chans = hsp->num_sm;  in tegra_hsp_probe()
735 	hsp->mbox_sm.dev = &pdev->dev;  in tegra_hsp_probe()
736 	hsp->mbox_sm.ops = &tegra_hsp_sm_ops;  in tegra_hsp_probe()
738 	hsp->mbox_sm.chans = devm_kcalloc(&pdev->dev, hsp->mbox_sm.num_chans,  in tegra_hsp_probe()
739 					  sizeof(*hsp->mbox_sm.chans),  in tegra_hsp_probe()
741 	if (!hsp->mbox_sm.chans)  in tegra_hsp_probe()
742 		return -ENOMEM;  in tegra_hsp_probe()
744 	if (hsp->shared_irqs) {  in tegra_hsp_probe()
745 		err = tegra_hsp_add_mailboxes(hsp, &pdev->dev);  in tegra_hsp_probe()
747 			dev_err(&pdev->dev, "failed to add mailboxes: %d\n",  in tegra_hsp_probe()
753 	err = devm_mbox_controller_register(&pdev->dev, &hsp->mbox_sm);  in tegra_hsp_probe()
755 		dev_err(&pdev->dev, "failed to register shared mailbox: %d\n",  in tegra_hsp_probe()
762 	if (hsp->doorbell_irq) {  in tegra_hsp_probe()
763 		err = devm_request_irq(&pdev->dev, hsp->doorbell_irq,  in tegra_hsp_probe()
765 				       dev_name(&pdev->dev), hsp);  in tegra_hsp_probe()
767 			dev_err(&pdev->dev,  in tegra_hsp_probe()
769 				hsp->doorbell_irq, err);  in tegra_hsp_probe()
774 	if (hsp->shared_irqs) {  in tegra_hsp_probe()
780 	lockdep_register_key(&hsp->lock_key);  in tegra_hsp_probe()
781 	lockdep_set_class(&hsp->lock, &hsp->lock_key);  in tegra_hsp_probe()
790 	lockdep_unregister_key(&hsp->lock_key);  in tegra_hsp_remove()
801 	list_for_each_entry(db, &hsp->doorbells, list) {  in tegra_hsp_resume()
802 		if (db && db->channel.chan)  in tegra_hsp_resume()
803 			tegra_hsp_doorbell_startup(db->channel.chan);  in tegra_hsp_resume()
806 	if (hsp->mailboxes) {  in tegra_hsp_resume()
807 		for (i = 0; i < hsp->num_sm; i++) {  in tegra_hsp_resume()
808 			struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i];  in tegra_hsp_resume()
810 			if (mb->channel.chan->cl)  in tegra_hsp_resume()
811 				tegra_hsp_mailbox_startup(mb->channel.chan);  in tegra_hsp_resume()
824 	{ "bpmp",   TEGRA_HSP_DB_MASTER_BPMP,   HSP_DB_BPMP,   },
839 	{ .compatible = "nvidia,tegra186-hsp", .data = &tegra186_hsp_soc },
840 	{ .compatible = "nvidia,tegra194-hsp", .data = &tegra194_hsp_soc },
846 		.name = "tegra-hsp",