1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
25 #include <soc/qcom/cmd-db.h>
27 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
29 #include "rpmh-internal.h"
32 #include "trace-rpmh.h"
92  * - The main rpmh-rsc address is the base of a register space that can
94  *   (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
96  *   specified in the device tree by "qcom,tcs-offset" and used to
98  * - TCS blocks come one after another. Type, count, and order are
99  *   specified by the device tree as "qcom,tcs-config".
100  * - Each TCS block has some registers, then space for up to 16 commands.
106  *  +---------------------------------------------------+
111  *  | +-----------------------------------------------+ |
114  *  | | IRQ                                           | |
117  *  | | +------------------------------------------+  | |
121  *  | | +------------------------------------------+  | |
122  *  | | +------------------------------------------+  | |
126  *  | | +------------------------------------------+  | |
127  *  | | +------------------------------------------+  | |
131  *  | | +------------------------------------------+  | |
133  *  | +-----------------------------------------------+ |
134  *  | +-----------------------------------------------+ |
137  *  | +-----------------------------------------------+ |
139  *  +---------------------------------------------------+
145 	return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg;  in tcs_reg_addr()
193 	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,  in write_tcs_reg_sync()
198  * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
212 	struct tcs_group *tcs = &drv->tcs[type];  in tcs_invalidate()
215 	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))  in tcs_invalidate()
218 	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++)  in tcs_invalidate()
221 	bitmap_zero(tcs->slots, MAX_TCS_SLOTS);  in tcs_invalidate()
225  * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
239  * get_tcs_for_msg() - Get the tcs_group used to send the given message.
254 	switch (msg->state) {  in get_tcs_for_msg()
265 		return ERR_PTR(-EINVAL);  in get_tcs_for_msg()
270 	 * dedicated TCS for active state use, then re-purpose a wake TCS to  in get_tcs_for_msg()
271 	 * send active votes. This is safe because we ensure any active-only  in get_tcs_for_msg()
275 	tcs = &drv->tcs[type];  in get_tcs_for_msg()
276 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)  in get_tcs_for_msg()
277 		tcs = &drv->tcs[WAKE_TCS];  in get_tcs_for_msg()
283  * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
305 		tcs = &drv->tcs[i];  in get_req_from_tcs()
306 		if (tcs->mask & BIT(tcs_id))  in get_req_from_tcs()
307 			return tcs->req[tcs_id - tcs->offset];  in get_req_from_tcs()
314  * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
320  * transfer. That will un-trigger/disable the TCS from the last transfer
323  * If we borrowed a wake TCS for an active-only transfer we'll also call
324  * this function with "trigger=false" to just do the un-trigger/disable
325  * before using the TCS for wake purposes again.
327  * Note that the AP is only in charge of triggering active-only transfers.
328  * The AP never triggers sleep/wake values using this function.
355  * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
360  * We only ever call this when we borrow a wake TCS for an active-only
361  * transfer. For active-only TCSes interrupts are always left enabled.
367 	data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE);  in enable_tcs_irq()
372 	writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE);  in enable_tcs_irq()
376  * tcs_tx_done() - TX Done interrupt handler.
377  * @irq: The IRQ number (ignored).
381  * IRQ for) when a transfer is done.
385 static irqreturn_t tcs_tx_done(int irq, void *p)  in tcs_tx_done()  argument
393 	irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS);  in tcs_tx_done()
401 		for (j = 0; j < req->num_cmds; j++) {  in tcs_tx_done()
404 			cmd = &req->cmds[j];  in tcs_tx_done()
407 			   ((req->wait_for_compl || cmd->wait) &&  in tcs_tx_done()
410 				       drv->name, cmd->addr, cmd->data);  in tcs_tx_done()
411 				err = -EIO;  in tcs_tx_done()
418 		 * If wake tcs was re-purposed for sending active  in tcs_tx_done()
422 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
427 		writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR);  in tcs_tx_done()
428 		spin_lock(&drv->lock);  in tcs_tx_done()
429 		clear_bit(i, drv->tcs_in_use);  in tcs_tx_done()
431 		 * Disable interrupt for WAKE TCS to avoid being  in tcs_tx_done()
433 		 * sends its wake votes.  in tcs_tx_done()
435 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
437 		spin_unlock(&drv->lock);  in tcs_tx_done()
438 		wake_up(&drv->tcs_wait);  in tcs_tx_done()
447  * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
454  * This is used for all types of transfers (active, sleep, and wake).
466 	cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;  in __tcs_buffer_write()
468 	for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {  in __tcs_buffer_write()
469 		cmd = &msg->cmds[i];  in __tcs_buffer_write()
473 		 * Additionally, if the cmd->wait is set, make the command  in __tcs_buffer_write()
474 		 * response reqd even if the overall request was fire-n-forget.  in __tcs_buffer_write()
476 		msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;  in __tcs_buffer_write()
479 		write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr);  in __tcs_buffer_write()
480 		write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data);  in __tcs_buffer_write()
489  * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
497  * one it'll return -EBUSY.
499  * Only for use for active-only transfers.
501  * Must be called with the drv->lock held since that protects tcs_in_use.
503  * Return: 0 if nothing in flight or -EBUSY if we should try again later.
504  *         The caller must re-enable interrupts between tries since that's
514 	int i = tcs->offset;  in check_for_req_inflight()
516 	for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {  in check_for_req_inflight()
521 			for (k = 0; k < msg->num_cmds; k++) {  in check_for_req_inflight()
522 				if (addr == msg->cmds[k].addr)  in check_for_req_inflight()
523 					return -EBUSY;  in check_for_req_inflight()
532  * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
533  * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
534  *       we borrowed it because there are zero active-only ones).
536  * Must be called with the drv->lock held since that protects tcs_in_use.
538  * Return: The first tcs that's free or -EBUSY if all in use.
542 	const struct rsc_drv *drv = tcs->drv;  in find_free_tcs()
544 	unsigned long max = tcs->offset + tcs->num_tcs;  in find_free_tcs()
546 	i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);  in find_free_tcs()
548 		return -EBUSY;  in find_free_tcs()
554  * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
562  * Context: Must be called with the drv->lock held since that protects
565  * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
575 	 * when one is already in-flight or being processed.  in claim_tcs_for_req()
585  * rpmh_rsc_send_data() - Write / trigger active-only message.
590  * - This is only used for "ACTIVE_ONLY" since the limitations of this
591  *   function don't make sense for sleep/wake cases.
592  * - To do the transfer, we will grab a whole TCS for ourselves--we don't
595  * - This function will not wait for the commands to be finished, only for
598  * - This function must be called with interrupts enabled. If the hardware
600  *   finish so that we can have the hardware, and to fully finish it needs
602  *   active CPU this can never happen if interrupts are disabled.
604  * Return: 0 on success, -EINVAL on error.
616 	spin_lock_irqsave(&drv->lock, flags);  in rpmh_rsc_send_data()
619 	wait_event_lock_irq(drv->tcs_wait,  in rpmh_rsc_send_data()
621 			    drv->lock);  in rpmh_rsc_send_data()
623 	tcs->req[tcs_id - tcs->offset] = msg;  in rpmh_rsc_send_data()
624 	set_bit(tcs_id, drv->tcs_in_use);  in rpmh_rsc_send_data()
625 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {  in rpmh_rsc_send_data()
627 		 * Clear previously programmed WAKE commands in selected  in rpmh_rsc_send_data()
628 		 * repurposed TCS to avoid triggering them. tcs->slots will be  in rpmh_rsc_send_data()
634 	spin_unlock_irqrestore(&drv->lock, flags);  in rpmh_rsc_send_data()
637 	 * These two can be done after the lock is released because:  in rpmh_rsc_send_data()
638 	 * - We marked "tcs_in_use" under lock.  in rpmh_rsc_send_data()
639 	 * - Once "tcs_in_use" has been marked nobody else could be writing  in rpmh_rsc_send_data()
641 	 * - The interrupt can't go off until we trigger w/ the last line  in rpmh_rsc_send_data()
651  * find_slots() - Find a place to write the given message.
660  * Only for use on sleep/wake TCSes since those are the only ones we maintain
661  * tcs->slots for.
663  * Return: -ENOMEM if there was no room, else 0.
671 	/* Do over, until we can fit the full payload in a single TCS */  in find_slots()
673 		slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,  in find_slots()
674 						  i, msg->num_cmds, 0);  in find_slots()
675 		if (slot >= tcs->num_tcs * tcs->ncpt)  in find_slots()
676 			return -ENOMEM;  in find_slots()
677 		i += tcs->ncpt;  in find_slots()
678 	} while (slot + msg->num_cmds - 1 >= i);  in find_slots()
680 	bitmap_set(tcs->slots, slot, msg->num_cmds);  in find_slots()
682 	offset = slot / tcs->ncpt;  in find_slots()
683 	*tcs_id = offset + tcs->offset;  in find_slots()
684 	*cmd_id = slot % tcs->ncpt;  in find_slots()
690  * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
694  * This should only be called for sleep/wake state, never active-only
700  * Return: 0 if no error; else -error.
721  * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
726  * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
729  * Context: Must be called with the drv->lock held.
732  * * False		- AMCs are idle
733  * * True		- AMCs are busy
738 	const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];  in rpmh_rsc_ctrlr_is_busy()
743 	 * dedicated TCS for active state use, then re-purposed wake TCSes  in rpmh_rsc_ctrlr_is_busy()
744 	 * should be checked for not busy, because we used wake TCSes for  in rpmh_rsc_ctrlr_is_busy()
747 	if (!tcs->num_tcs)  in rpmh_rsc_ctrlr_is_busy()
748 		tcs = &drv->tcs[WAKE_TCS];  in rpmh_rsc_ctrlr_is_busy()
750 	max = tcs->offset + tcs->num_tcs;  in rpmh_rsc_ctrlr_is_busy()
751 	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);  in rpmh_rsc_ctrlr_is_busy()
757  * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
762  * This function is given to cpu_pm_register_notifier so we can be informed
764  * transfers will be started so we write sleep/wake sets. This function gets
768  * and wake messages to TCSes. The firmware then takes care of triggering
782 		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
789 		 * AND that CPU was not idle AND that CPU was the last non-idle  in rpmh_rsc_cpu_pm_callback()
790 		 * CPU. That can't happen. CPUs would have to come out of idle  in rpmh_rsc_cpu_pm_callback()
798 		atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
805 	 * It's likely we're on the last CPU. Grab the drv->lock and write  in rpmh_rsc_cpu_pm_callback()
806 	 * out the sleep/wake commands to RPMH hardware. Grabbing the lock  in rpmh_rsc_cpu_pm_callback()
816 	if (spin_trylock(&drv->lock)) {  in rpmh_rsc_cpu_pm_callback()
817 		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))  in rpmh_rsc_cpu_pm_callback()
819 		spin_unlock(&drv->lock);  in rpmh_rsc_cpu_pm_callback()
826 		/* Double-check if we're here because someone else is up */  in rpmh_rsc_cpu_pm_callback()
831 			atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
844 	struct device_node *dn = pdev->dev.of_node;  in rpmh_probe_tcs_config()
849 	ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);  in rpmh_probe_tcs_config()
852 	drv->tcs_base = base + offset;  in rpmh_probe_tcs_config()
857 	max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
858 	max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
863 	n = of_property_count_u32_elems(dn, "qcom,tcs-config");  in rpmh_probe_tcs_config()
865 		return -EINVAL;  in rpmh_probe_tcs_config()
868 		ret = of_property_read_u32_index(dn, "qcom,tcs-config",  in rpmh_probe_tcs_config()
873 			return -EINVAL;  in rpmh_probe_tcs_config()
875 		ret = of_property_read_u32_index(dn, "qcom,tcs-config",  in rpmh_probe_tcs_config()
880 			return -EINVAL;  in rpmh_probe_tcs_config()
884 		tcs = &drv->tcs[tcs_cfg[i].type];  in rpmh_probe_tcs_config()
885 		if (tcs->drv)  in rpmh_probe_tcs_config()
886 			return -EINVAL;  in rpmh_probe_tcs_config()
887 		tcs->drv = drv;  in rpmh_probe_tcs_config()
888 		tcs->type = tcs_cfg[i].type;  in rpmh_probe_tcs_config()
889 		tcs->num_tcs = tcs_cfg[i].n;  in rpmh_probe_tcs_config()
890 		tcs->ncpt = ncpt;  in rpmh_probe_tcs_config()
892 		if (!tcs->num_tcs || tcs->type == CONTROL_TCS)  in rpmh_probe_tcs_config()
895 		if (st + tcs->num_tcs > max_tcs ||  in rpmh_probe_tcs_config()
896 		    st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))  in rpmh_probe_tcs_config()
897 			return -EINVAL;  in rpmh_probe_tcs_config()
899 		tcs->mask = ((1 << tcs->num_tcs) - 1) << st;  in rpmh_probe_tcs_config()
900 		tcs->offset = st;  in rpmh_probe_tcs_config()
901 		st += tcs->num_tcs;  in rpmh_probe_tcs_config()
904 	drv->num_tcs = st;  in rpmh_probe_tcs_config()
911 	struct device_node *dn = pdev->dev.of_node;  in rpmh_rsc_probe()
914 	int ret, irq;  in rpmh_rsc_probe()  local
919 	 * Even though RPMh doesn't directly use cmd-db, all of its children  in rpmh_rsc_probe()
924 		if (ret != -EPROBE_DEFER)  in rpmh_rsc_probe()
925 			dev_err(&pdev->dev, "Command DB not available (%d)\n",  in rpmh_rsc_probe()
930 	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);  in rpmh_rsc_probe()
932 		return -ENOMEM;  in rpmh_rsc_probe()
934 	ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);  in rpmh_rsc_probe()
938 	drv->name = of_get_property(dn, "label", NULL);  in rpmh_rsc_probe()
939 	if (!drv->name)  in rpmh_rsc_probe()
940 		drv->name = dev_name(&pdev->dev);  in rpmh_rsc_probe()
942 	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);  in rpmh_rsc_probe()
951 	spin_lock_init(&drv->lock);  in rpmh_rsc_probe()
952 	init_waitqueue_head(&drv->tcs_wait);  in rpmh_rsc_probe()
953 	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);  in rpmh_rsc_probe()
955 	irq = platform_get_irq(pdev, drv->id);  in rpmh_rsc_probe()
956 	if (irq < 0)  in rpmh_rsc_probe()
957 		return irq;  in rpmh_rsc_probe()
959 	ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,  in rpmh_rsc_probe()
961 			       drv->name, drv);  in rpmh_rsc_probe()
967 	 * 'HW solver' mode where they can be in autonomous mode executing low  in rpmh_rsc_probe()
974 		drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;  in rpmh_rsc_probe()
975 		cpu_pm_register_notifier(&drv->rsc_pm);  in rpmh_rsc_probe()
979 	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,  in rpmh_rsc_probe()
980 		       drv->tcs_base + RSC_DRV_IRQ_ENABLE);  in rpmh_rsc_probe()
982 	spin_lock_init(&drv->client.cache_lock);  in rpmh_rsc_probe()
983 	INIT_LIST_HEAD(&drv->client.cache);  in rpmh_rsc_probe()
984 	INIT_LIST_HEAD(&drv->client.batch_cache);  in rpmh_rsc_probe()
986 	dev_set_drvdata(&pdev->dev, drv);  in rpmh_rsc_probe()
988 	return devm_of_platform_populate(&pdev->dev);  in rpmh_rsc_probe()
992 	{ .compatible = "qcom,rpmh-rsc", },