37 /* DRV HW Solver Configuration Information Register */
42 /* DRV TCS Configuration Information Register */
143 tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)  in tcs_reg_addr()  argument
145 	return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg;  in tcs_reg_addr()
149 tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)  in tcs_cmd_addr()  argument
151 	return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id;  in tcs_cmd_addr()
154 static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,  in read_tcs_cmd()  argument
157 	return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));  in read_tcs_cmd()
160 static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)  in read_tcs_reg()  argument
162 	return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));  in read_tcs_reg()
165 static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,  in write_tcs_cmd()  argument
168 	writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));  in write_tcs_cmd()
171 static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,  in write_tcs_reg()  argument
174 	writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));  in write_tcs_reg()
177 static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,  in write_tcs_reg_sync()  argument
182 	writel(data, tcs_reg_addr(drv, reg, tcs_id));  in write_tcs_reg_sync()
189 		if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)  in write_tcs_reg_sync()
193 	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,  in write_tcs_reg_sync()
199  * @drv:  The RSC controller.
209 static void tcs_invalidate(struct rsc_drv *drv, int type)  in tcs_invalidate()  argument
212 	struct tcs_group *tcs = &drv->tcs[type];  in tcs_invalidate()
219 		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0);  in tcs_invalidate()
226  * @drv: The RSC controller.
232 void rpmh_rsc_invalidate(struct rsc_drv *drv)  in rpmh_rsc_invalidate()  argument
234 	tcs_invalidate(drv, SLEEP_TCS);  in rpmh_rsc_invalidate()
235 	tcs_invalidate(drv, WAKE_TCS);  in rpmh_rsc_invalidate()
240  * @drv: The RSC controller.
248 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,  in get_tcs_for_msg()  argument
275 	tcs = &drv->tcs[type];  in get_tcs_for_msg()
277 		tcs = &drv->tcs[WAKE_TCS];  in get_tcs_for_msg()
284  * @drv:    The RSC controller.
298 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,  in get_req_from_tcs()  argument
305 		tcs = &drv->tcs[i];  in get_req_from_tcs()
315  * @drv:     The controller.
330 static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)  in __tcs_set_trigger()  argument
339 	enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id);  in __tcs_set_trigger()
341 	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);  in __tcs_set_trigger()
343 	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);  in __tcs_set_trigger()
348 		write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);  in __tcs_set_trigger()
350 		write_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, enable);  in __tcs_set_trigger()
356  * @drv:     The controller.
363 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)  in enable_tcs_irq()  argument
367 	data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE);  in enable_tcs_irq()
372 	writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE);  in enable_tcs_irq()
387 	struct rsc_drv *drv = p;  in tcs_tx_done()  local
393 	irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS);  in tcs_tx_done()
396 		req = get_req_from_tcs(drv, i);  in tcs_tx_done()
405 			sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j);  in tcs_tx_done()
410 				       drv->name, cmd->addr, cmd->data);  in tcs_tx_done()
415 		trace_rpmh_tx_done(drv, i, req, err);  in tcs_tx_done()
422 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
423 			__tcs_set_trigger(drv, i, false);  in tcs_tx_done()
426 		write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0);  in tcs_tx_done()
427 		writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR);  in tcs_tx_done()
428 		spin_lock(&drv->lock);  in tcs_tx_done()
429 		clear_bit(i, drv->tcs_in_use);  in tcs_tx_done()
435 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
436 			enable_tcs_irq(drv, i, false);  in tcs_tx_done()
437 		spin_unlock(&drv->lock);  in tcs_tx_done()
438 		wake_up(&drv->tcs_wait);  in tcs_tx_done()
448  * @drv:    The controller.
456 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,  in __tcs_buffer_write()  argument
478 		write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid);  in __tcs_buffer_write()
479 		write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr);  in __tcs_buffer_write()
480 		write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data);  in __tcs_buffer_write()
481 		trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd);  in __tcs_buffer_write()
484 	cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);  in __tcs_buffer_write()
485 	write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable);  in __tcs_buffer_write()
490  * @drv: The controller.
501  * Must be called with the drv->lock held since that protects tcs_in_use.
508 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,  in check_for_req_inflight()  argument
516 	for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {  in check_for_req_inflight()
517 		curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i);  in check_for_req_inflight()
520 			addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, i, j);  in check_for_req_inflight()
536  * Must be called with the drv->lock held since that protects tcs_in_use.
542 	const struct rsc_drv *drv = tcs->drv;  in find_free_tcs()  local
546 	i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);  in find_free_tcs()
555  * @drv: The controller.
562  * Context: Must be called with the drv->lock held since that protects
568 static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,  in claim_tcs_for_req()  argument
577 	ret = check_for_req_inflight(drv, tcs, msg);  in claim_tcs_for_req()
586  * @drv: The controller.
606 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)  in rpmh_rsc_send_data()  argument
612 	tcs = get_tcs_for_msg(drv, msg);  in rpmh_rsc_send_data()
616 	spin_lock_irqsave(&drv->lock, flags);  in rpmh_rsc_send_data()
619 	wait_event_lock_irq(drv->tcs_wait,  in rpmh_rsc_send_data()
620 			    (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,  in rpmh_rsc_send_data()
621 			    drv->lock);  in rpmh_rsc_send_data()
624 	set_bit(tcs_id, drv->tcs_in_use);  in rpmh_rsc_send_data()
631 		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);  in rpmh_rsc_send_data()
632 		enable_tcs_irq(drv, tcs_id, true);  in rpmh_rsc_send_data()
634 	spin_unlock_irqrestore(&drv->lock, flags);  in rpmh_rsc_send_data()
644 	__tcs_buffer_write(drv, tcs_id, 0, msg);  in rpmh_rsc_send_data()
645 	__tcs_set_trigger(drv, tcs_id, true);  in rpmh_rsc_send_data()
691  * @drv: The controller.
702 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)  in rpmh_rsc_write_ctrl_data()  argument
708 	tcs = get_tcs_for_msg(drv, msg);  in rpmh_rsc_write_ctrl_data()
715 		__tcs_buffer_write(drv, tcs_id, cmd_id, msg);  in rpmh_rsc_write_ctrl_data()
722  * @drv: The controller
729  * Context: Must be called with the drv->lock held.
735 static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)  in rpmh_rsc_ctrlr_is_busy()  argument
738 	const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];  in rpmh_rsc_ctrlr_is_busy()
748 		tcs = &drv->tcs[WAKE_TCS];  in rpmh_rsc_ctrlr_is_busy()
751 	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);  in rpmh_rsc_ctrlr_is_busy()
776 	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);  in rpmh_rsc_cpu_pm_callback()  local
782 		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
798 		atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
805 	 * It's likely we're on the last CPU. Grab the drv->lock and write  in rpmh_rsc_cpu_pm_callback()
816 	if (spin_trylock(&drv->lock)) {  in rpmh_rsc_cpu_pm_callback()
817 		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))  in rpmh_rsc_cpu_pm_callback()
819 		spin_unlock(&drv->lock);  in rpmh_rsc_cpu_pm_callback()
831 			atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
838 				 struct rsc_drv *drv, void __iomem *base)  in rpmh_probe_tcs_config()  argument
852 	drv->tcs_base = base + offset;  in rpmh_probe_tcs_config()
857 	max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
858 	max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
884 		tcs = &drv->tcs[tcs_cfg[i].type];  in rpmh_probe_tcs_config()
885 		if (tcs->drv)  in rpmh_probe_tcs_config()
887 		tcs->drv = drv;  in rpmh_probe_tcs_config()
904 	drv->num_tcs = st;  in rpmh_probe_tcs_config()
912 	struct rsc_drv *drv;  in rpmh_rsc_probe()  local
931 	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);  in rpmh_rsc_probe()
932 	if (!drv)  in rpmh_rsc_probe()
935 	ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);  in rpmh_rsc_probe()
939 	drv->name = of_get_property(dn, "label", NULL);  in rpmh_rsc_probe()
940 	if (!drv->name)  in rpmh_rsc_probe()
941 		drv->name = dev_name(&pdev->dev);  in rpmh_rsc_probe()
943 	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);  in rpmh_rsc_probe()
949 	ret = rpmh_probe_tcs_config(pdev, drv, base);  in rpmh_rsc_probe()
953 	spin_lock_init(&drv->lock);  in rpmh_rsc_probe()
954 	init_waitqueue_head(&drv->tcs_wait);  in rpmh_rsc_probe()
955 	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);  in rpmh_rsc_probe()
957 	irq = platform_get_irq(pdev, drv->id);  in rpmh_rsc_probe()
963 			       drv->name, drv);  in rpmh_rsc_probe()
976 		drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;  in rpmh_rsc_probe()
977 		cpu_pm_register_notifier(&drv->rsc_pm);  in rpmh_rsc_probe()
981 	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,  in rpmh_rsc_probe()
982 		       drv->tcs_base + RSC_DRV_IRQ_ENABLE);  in rpmh_rsc_probe()
984 	spin_lock_init(&drv->client.cache_lock);  in rpmh_rsc_probe()
985 	INIT_LIST_HEAD(&drv->client.cache);  in rpmh_rsc_probe()
986 	INIT_LIST_HEAD(&drv->client.batch_cache);  in rpmh_rsc_probe()
988 	dev_set_drvdata(&pdev->dev, drv);  in rpmh_rsc_probe()