1 // SPDX-License-Identifier: GPL-2.0-only
3  * ARM DMC-620 memory controller PMU driver
40  * The PMU registers start at 0xA00 in the DMC-620 memory map, and these
50 		(DMC620_PMU_CLKDIV2_MAX_COUNTERS - 1)
53 		(DMC620_PMU_CLK_MAX_COUNTERS - 1)
116 	return sysfs_emit(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2);  in dmc620_pmu_event_show()
163 	.name = "events",
188 	(lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
193 #define GEN_PMU_FORMAT_ATTR(name)				\  argument
194 	PMU_FORMAT_ATTR(name,					\
195 	_GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG,		\
196 			     ATTR_CFG_FLD_##name##_LO,		\
197 			     ATTR_CFG_FLD_##name##_HI))
200 	((((attr)->cfg) >> lo) & GENMASK_ULL(hi - lo, 0))
202 #define ATTR_CFG_GET_FLD(attr, name)				\  argument
204 			  ATTR_CFG_FLD_##name##_CFG,		\
205 			  ATTR_CFG_FLD_##name##_LO,		\
206 			  ATTR_CFG_FLD_##name##_HI)
212 GEN_PMU_FORMAT_ATTR(event);
226 	.name	= "format",
240 	return readl(dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);  in dmc620_pmu_creg_read()
247 	writel(val, dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);  in dmc620_pmu_creg_write()
251 unsigned int dmc620_event_to_counter_control(struct perf_event *event)  in dmc620_event_to_counter_control()  argument
253 	struct perf_event_attr *attr = &event->attr;  in dmc620_event_to_counter_control()
259 			ATTR_CFG_GET_FLD(attr, event));  in dmc620_event_to_counter_control()
266 static int dmc620_get_event_idx(struct perf_event *event)  in dmc620_get_event_idx()  argument
268 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_get_event_idx()
271 	if (ATTR_CFG_GET_FLD(&event->attr, clkdiv2)) {  in dmc620_get_event_idx()
280 		if (!test_and_set_bit(idx, dmc620_pmu->used_mask))  in dmc620_get_event_idx()
285 	return -EAGAIN;  in dmc620_get_event_idx()
289 u64 dmc620_pmu_read_counter(struct perf_event *event)  in dmc620_pmu_read_counter()  argument
291 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_read_counter()
294 				    event->hw.idx, DMC620_PMU_COUNTERn_VALUE);  in dmc620_pmu_read_counter()
297 static void dmc620_pmu_event_update(struct perf_event *event)  in dmc620_pmu_event_update()  argument
299 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_event_update()
304 		prev_count = local64_read(&hwc->prev_count);  in dmc620_pmu_event_update()
305 		new_count = dmc620_pmu_read_counter(event);  in dmc620_pmu_event_update()
306 	} while (local64_cmpxchg(&hwc->prev_count,  in dmc620_pmu_event_update()
308 	delta = (new_count - prev_count) & DMC620_CNT_MAX_PERIOD;  in dmc620_pmu_event_update()
309 	local64_add(delta, &event->count);  in dmc620_pmu_event_update()
312 static void dmc620_pmu_event_set_period(struct perf_event *event)  in dmc620_pmu_event_set_period()  argument
314 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_event_set_period()
316 	local64_set(&event->hw.prev_count, DMC620_CNT_INIT);  in dmc620_pmu_event_set_period()
318 			      event->hw.idx, DMC620_PMU_COUNTERn_VALUE, DMC620_CNT_INIT);  in dmc620_pmu_event_set_period()
321 static void dmc620_pmu_enable_counter(struct perf_event *event)  in dmc620_pmu_enable_counter()  argument
323 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_enable_counter()
326 	reg = dmc620_event_to_counter_control(event) | DMC620_PMU_COUNTERn_CONTROL_ENABLE;  in dmc620_pmu_enable_counter()
328 			      event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, reg);  in dmc620_pmu_enable_counter()
331 static void dmc620_pmu_disable_counter(struct perf_event *event)  in dmc620_pmu_disable_counter()  argument
333 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_disable_counter()
336 			      event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, 0);  in dmc620_pmu_disable_counter()
346 	list_for_each_entry_rcu(dmc620_pmu, &irq->pmus_node, pmus_node) {  in dmc620_pmu_handle_irq()
348 		struct perf_event *event;  in dmc620_pmu_handle_irq()  local
357 			event = dmc620_pmu->events[idx];  in dmc620_pmu_handle_irq()
358 			if (!event)  in dmc620_pmu_handle_irq()
360 			dmc620_pmu_disable_counter(event);  in dmc620_pmu_handle_irq()
363 		status = readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);  in dmc620_pmu_handle_irq()
364 		status |= (readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK) <<  in dmc620_pmu_handle_irq()
369 				event = dmc620_pmu->events[idx];  in dmc620_pmu_handle_irq()
370 				if (WARN_ON_ONCE(!event))  in dmc620_pmu_handle_irq()
372 				dmc620_pmu_event_update(event);  in dmc620_pmu_handle_irq()
373 				dmc620_pmu_event_set_period(event);  in dmc620_pmu_handle_irq()
377 				writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);  in dmc620_pmu_handle_irq()
381 				writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);  in dmc620_pmu_handle_irq()
385 			event = dmc620_pmu->events[idx];  in dmc620_pmu_handle_irq()
386 			if (!event)  in dmc620_pmu_handle_irq()
388 			if (!(event->hw.state & PERF_HES_STOPPED))  in dmc620_pmu_handle_irq()
389 				dmc620_pmu_enable_counter(event);  in dmc620_pmu_handle_irq()
405 		if (irq->irq_num == irq_num && refcount_inc_not_zero(&irq->refcount))  in __dmc620_pmu_get_irq()
410 		return ERR_PTR(-ENOMEM);  in __dmc620_pmu_get_irq()
412 	INIT_LIST_HEAD(&irq->pmus_node);  in __dmc620_pmu_get_irq()
415 	irq->cpu = raw_smp_processor_id();  in __dmc620_pmu_get_irq()
416 	refcount_set(&irq->refcount, 1);  in __dmc620_pmu_get_irq()
420 			  "dmc620-pmu", irq);  in __dmc620_pmu_get_irq()
424 	ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu));  in __dmc620_pmu_get_irq()
428 	ret = cpuhp_state_add_instance_nocalls(cpuhp_state_num, &irq->node);  in __dmc620_pmu_get_irq()
432 	irq->irq_num = irq_num;  in __dmc620_pmu_get_irq()
433 	list_add(&irq->irqs_node, &dmc620_pmu_irqs);  in __dmc620_pmu_get_irq()
455 	dmc620_pmu->irq = irq;  in dmc620_pmu_get_irq()
457 	list_add_rcu(&dmc620_pmu->pmus_node, &irq->pmus_node);  in dmc620_pmu_get_irq()
465 	struct dmc620_pmu_irq *irq = dmc620_pmu->irq;  in dmc620_pmu_put_irq()
468 	list_del_rcu(&dmc620_pmu->pmus_node);  in dmc620_pmu_put_irq()
470 	if (!refcount_dec_and_test(&irq->refcount)) {  in dmc620_pmu_put_irq()
475 	list_del(&irq->irqs_node);  in dmc620_pmu_put_irq()
478 	free_irq(irq->irq_num, irq);  in dmc620_pmu_put_irq()
479 	cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node);  in dmc620_pmu_put_irq()
483 static int dmc620_pmu_event_init(struct perf_event *event)  in dmc620_pmu_event_init()  argument
485 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_event_init()
486 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_event_init()
489 	if (event->attr.type != event->pmu->type)  in dmc620_pmu_event_init()
490 		return -ENOENT;  in dmc620_pmu_event_init()
496 	if (is_sampling_event(event) ||  in dmc620_pmu_event_init()
497 		event->attach_state & PERF_ATTACH_TASK) {  in dmc620_pmu_event_init()
498 		dev_dbg(dmc620_pmu->pmu.dev,  in dmc620_pmu_event_init()
499 			"Can't support per-task counters\n");  in dmc620_pmu_event_init()
500 		return -EOPNOTSUPP;  in dmc620_pmu_event_init()
507 	 * but can lead to issues for off-core PMUs, where each  in dmc620_pmu_event_init()
508 	 * event could be theoretically assigned to a different CPU. To  in dmc620_pmu_event_init()
512 	event->cpu = dmc620_pmu->irq->cpu;  in dmc620_pmu_event_init()
513 	if (event->cpu < 0)  in dmc620_pmu_event_init()
514 		return -EINVAL;  in dmc620_pmu_event_init()
517 	 * We can't atomically disable all HW counters so only one event allowed,  in dmc620_pmu_event_init()
520 	if (event->group_leader != event &&  in dmc620_pmu_event_init()
521 			!is_software_event(event->group_leader))  in dmc620_pmu_event_init()
522 		return -EINVAL;  in dmc620_pmu_event_init()
524 	for_each_sibling_event(sibling, event->group_leader) {  in dmc620_pmu_event_init()
525 		if (sibling != event &&  in dmc620_pmu_event_init()
527 			return -EINVAL;  in dmc620_pmu_event_init()
530 	hwc->idx = -1;  in dmc620_pmu_event_init()
534 static void dmc620_pmu_read(struct perf_event *event)  in dmc620_pmu_read()  argument
536 	dmc620_pmu_event_update(event);  in dmc620_pmu_read()
539 static void dmc620_pmu_start(struct perf_event *event, int flags)  in dmc620_pmu_start()  argument
541 	event->hw.state = 0;  in dmc620_pmu_start()
542 	dmc620_pmu_event_set_period(event);  in dmc620_pmu_start()
543 	dmc620_pmu_enable_counter(event);  in dmc620_pmu_start()
546 static void dmc620_pmu_stop(struct perf_event *event, int flags)  in dmc620_pmu_stop()  argument
548 	if (event->hw.state & PERF_HES_STOPPED)  in dmc620_pmu_stop()
551 	dmc620_pmu_disable_counter(event);  in dmc620_pmu_stop()
552 	dmc620_pmu_event_update(event);  in dmc620_pmu_stop()
553 	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;  in dmc620_pmu_stop()
556 static int dmc620_pmu_add(struct perf_event *event, int flags)  in dmc620_pmu_add()  argument
558 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_add()
559 	struct perf_event_attr *attr = &event->attr;  in dmc620_pmu_add()
560 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_add()
564 	idx = dmc620_get_event_idx(event);  in dmc620_pmu_add()
568 	hwc->idx = idx;  in dmc620_pmu_add()
569 	dmc620_pmu->events[idx] = event;  in dmc620_pmu_add()
570 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;  in dmc620_pmu_add()
585 		dmc620_pmu_start(event, PERF_EF_RELOAD);  in dmc620_pmu_add()
587 	perf_event_update_userpage(event);  in dmc620_pmu_add()
591 static void dmc620_pmu_del(struct perf_event *event, int flags)  in dmc620_pmu_del()  argument
593 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_del()
594 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_del()
595 	int idx = hwc->idx;  in dmc620_pmu_del()
597 	dmc620_pmu_stop(event, PERF_EF_UPDATE);  in dmc620_pmu_del()
598 	dmc620_pmu->events[idx] = NULL;  in dmc620_pmu_del()
599 	clear_bit(idx, dmc620_pmu->used_mask);  in dmc620_pmu_del()
600 	perf_event_update_userpage(event);  in dmc620_pmu_del()
611 	if (cpu != irq->cpu)  in dmc620_pmu_cpu_teardown()
620 	list_for_each_entry(dmc620_pmu, &irq->pmus_node, pmus_node)  in dmc620_pmu_cpu_teardown()
621 		perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target);  in dmc620_pmu_cpu_teardown()
624 	WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target)));  in dmc620_pmu_cpu_teardown()
625 	irq->cpu = target;  in dmc620_pmu_cpu_teardown()
634 	char *name;  in dmc620_pmu_device_probe()  local
638 	dmc620_pmu = devm_kzalloc(&pdev->dev,  in dmc620_pmu_device_probe()
641 		return -ENOMEM;  in dmc620_pmu_device_probe()
645 	dmc620_pmu->pmu = (struct pmu) {  in dmc620_pmu_device_probe()
659 	dmc620_pmu->base = devm_ioremap_resource(&pdev->dev, res);  in dmc620_pmu_device_probe()
660 	if (IS_ERR(dmc620_pmu->base))  in dmc620_pmu_device_probe()
661 		return PTR_ERR(dmc620_pmu->base);  in dmc620_pmu_device_probe()
666 	writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);  in dmc620_pmu_device_probe()
667 	writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);  in dmc620_pmu_device_probe()
677 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL,  in dmc620_pmu_device_probe()
679 				  (u64)(res->start >> DMC620_PA_SHIFT));  in dmc620_pmu_device_probe()
680 	if (!name) {  in dmc620_pmu_device_probe()
681 		dev_err(&pdev->dev,  in dmc620_pmu_device_probe()
682 			  "Create name failed, PMU @%pa\n", &res->start);  in dmc620_pmu_device_probe()
683 		ret = -ENOMEM;  in dmc620_pmu_device_probe()
687 	ret = perf_pmu_register(&dmc620_pmu->pmu, name, -1);  in dmc620_pmu_device_probe()
706 	perf_pmu_unregister(&dmc620_pmu->pmu);  in dmc620_pmu_device_remove()
718 		.name		= DMC620_DRVNAME,
747 MODULE_DESCRIPTION("Perf driver for the ARM DMC-620 memory controller");