1 // SPDX-License-Identifier: GPL-2.0
66 	{ .compatible = "fsl,imx8-ddr-pmu", .data = &imx8_devtype_data},
67 	{ .compatible = "fsl,imx8m-ddr-pmu", .data = &imx8m_devtype_data},
68 	{ .compatible = "fsl,imx8mp-ddr-pmu", .data = &imx8mp_devtype_data},
95 	u32 quirks = pmu->devtype_data->quirks;  in ddr_perf_filter_cap_get()
117 	int cap = (long)ea->var;  in ddr_perf_filter_cap_show()
147 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));  in ddr_perf_cpumask_show()
169 	return sprintf(page, "event=0x%02llx\n", pmu_attr->id);  in ddr_pmu_event_show()
181 	IMX8_DDR_PMU_EVENT_ATTR(read-accesses, 0x04),
182 	IMX8_DDR_PMU_EVENT_ATTR(write-accesses, 0x05),
183 	IMX8_DDR_PMU_EVENT_ATTR(read-queue-depth, 0x08),
184 	IMX8_DDR_PMU_EVENT_ATTR(write-queue-depth, 0x09),
185 	IMX8_DDR_PMU_EVENT_ATTR(lp-read-credit-cnt, 0x10),
186 	IMX8_DDR_PMU_EVENT_ATTR(hp-read-credit-cnt, 0x11),
187 	IMX8_DDR_PMU_EVENT_ATTR(write-credit-cnt, 0x12),
188 	IMX8_DDR_PMU_EVENT_ATTR(read-command, 0x20),
189 	IMX8_DDR_PMU_EVENT_ATTR(write-command, 0x21),
190 	IMX8_DDR_PMU_EVENT_ATTR(read-modify-write-command, 0x22),
191 	IMX8_DDR_PMU_EVENT_ATTR(hp-read, 0x23),
192 	IMX8_DDR_PMU_EVENT_ATTR(hp-req-nocredit, 0x24),
193 	IMX8_DDR_PMU_EVENT_ATTR(hp-xact-credit, 0x25),
194 	IMX8_DDR_PMU_EVENT_ATTR(lp-req-nocredit, 0x26),
195 	IMX8_DDR_PMU_EVENT_ATTR(lp-xact-credit, 0x27),
196 	IMX8_DDR_PMU_EVENT_ATTR(wr-xact-credit, 0x29),
197 	IMX8_DDR_PMU_EVENT_ATTR(read-cycles, 0x2a),
198 	IMX8_DDR_PMU_EVENT_ATTR(write-cycles, 0x2b),
199 	IMX8_DDR_PMU_EVENT_ATTR(read-write-transition, 0x30),
202 	IMX8_DDR_PMU_EVENT_ATTR(load-mode, 0x33),
203 	IMX8_DDR_PMU_EVENT_ATTR(perf-mwr, 0x34),
205 	IMX8_DDR_PMU_EVENT_ATTR(read-activate, 0x36),
208 	IMX8_DDR_PMU_EVENT_ATTR(raw-hazard, 0x39),
209 	IMX8_DDR_PMU_EVENT_ATTR(axid-read, 0x41),
210 	IMX8_DDR_PMU_EVENT_ATTR(axid-write, 0x42),
219 PMU_FORMAT_ATTR(event, "config:0-7");
220 PMU_FORMAT_ATTR(axi_id, "config1:0-15");
221 PMU_FORMAT_ATTR(axi_mask, "config1:16-31");
245 	return event->attr.config == 0x41 || event->attr.config == 0x42;  in ddr_perf_is_filtered()
250 	return event->attr.config1;  in ddr_perf_filter_val()
266 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_is_enhanced_filtered()
268 	filt = pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER_ENHANCED;  in ddr_perf_is_enhanced_filtered()
283 		if (pmu->events[EVENT_CYCLES_COUNTER] == NULL)  in ddr_perf_alloc_counter()
286 			return -ENOENT;  in ddr_perf_alloc_counter()
290 		if (pmu->events[i] == NULL)  in ddr_perf_alloc_counter()
294 	return -ENOENT;  in ddr_perf_alloc_counter()
299 	pmu->events[counter] = NULL;  in ddr_perf_free_counter()
304 	struct perf_event *event = pmu->events[counter];  in ddr_perf_read_counter()
305 	void __iomem *base = pmu->base;  in ddr_perf_read_counter()
309 	 * axid-read and axid-write event if PMU core supports enhanced  in ddr_perf_read_counter()
319 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_init()
320 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_init()
323 	if (event->attr.type != event->pmu->type)  in ddr_perf_event_init()
324 		return -ENOENT;  in ddr_perf_event_init()
326 	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)  in ddr_perf_event_init()
327 		return -EOPNOTSUPP;  in ddr_perf_event_init()
329 	if (event->cpu < 0) {  in ddr_perf_event_init()
330 		dev_warn(pmu->dev, "Can't provide per-task data!\n");  in ddr_perf_event_init()
331 		return -EOPNOTSUPP;  in ddr_perf_event_init()
337 	 * periodically read when a hrtimer aka cpu-clock leader triggers).  in ddr_perf_event_init()
339 	if (event->group_leader->pmu != event->pmu &&  in ddr_perf_event_init()
340 			!is_software_event(event->group_leader))  in ddr_perf_event_init()
341 		return -EINVAL;  in ddr_perf_event_init()
343 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {  in ddr_perf_event_init()
344 		if (!ddr_perf_filters_compatible(event, event->group_leader))  in ddr_perf_event_init()
345 			return -EINVAL;  in ddr_perf_event_init()
346 		for_each_sibling_event(sibling, event->group_leader) {  in ddr_perf_event_init()
348 				return -EINVAL;  in ddr_perf_event_init()
352 	for_each_sibling_event(sibling, event->group_leader) {  in ddr_perf_event_init()
353 		if (sibling->pmu != event->pmu &&  in ddr_perf_event_init()
355 			return -EINVAL;  in ddr_perf_event_init()
358 	event->cpu = pmu->cpu;  in ddr_perf_event_init()
359 	hwc->idx = -1;  in ddr_perf_event_init()
367 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_update()
368 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_update()
370 	int counter = hwc->idx;  in ddr_perf_event_update()
373 		prev_raw_count = local64_read(&hwc->prev_count);  in ddr_perf_event_update()
375 	} while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,  in ddr_perf_event_update()
378 	delta = (new_raw_count - prev_raw_count) & 0xFFFFFFFF;  in ddr_perf_event_update()
380 	local64_add(delta, &event->count);  in ddr_perf_event_update()
396 		writel(0, pmu->base + reg);  in ddr_perf_counter_enable()
399 		writel(val, pmu->base + reg);  in ddr_perf_counter_enable()
402 		val = readl_relaxed(pmu->base + reg) & CNTL_EN_MASK;  in ddr_perf_counter_enable()
403 		writel(val, pmu->base + reg);  in ddr_perf_counter_enable()
409 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_start()
410 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_start()
411 	int counter = hwc->idx;  in ddr_perf_event_start()
413 	local64_set(&hwc->prev_count, 0);  in ddr_perf_event_start()
415 	ddr_perf_counter_enable(pmu, event->attr.config, counter, true);  in ddr_perf_event_start()
417 	hwc->state = 0;  in ddr_perf_event_start()
422 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_add()
423 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_add()
425 	int cfg = event->attr.config;  in ddr_perf_event_add()
426 	int cfg1 = event->attr.config1;  in ddr_perf_event_add()
428 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {  in ddr_perf_event_add()
432 			if (pmu->events[i] &&  in ddr_perf_event_add()
433 			    !ddr_perf_filters_compatible(event, pmu->events[i]))  in ddr_perf_event_add()
434 				return -EINVAL;  in ddr_perf_event_add()
440 			writel(cfg1, pmu->base + COUNTER_DPCR1);  in ddr_perf_event_add()
446 		dev_dbg(pmu->dev, "There are not enough counters\n");  in ddr_perf_event_add()
447 		return -EOPNOTSUPP;  in ddr_perf_event_add()
450 	pmu->events[counter] = event;  in ddr_perf_event_add()
451 	pmu->active_events++;  in ddr_perf_event_add()
452 	hwc->idx = counter;  in ddr_perf_event_add()
454 	hwc->state |= PERF_HES_STOPPED;  in ddr_perf_event_add()
464 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_stop()
465 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_stop()
466 	int counter = hwc->idx;  in ddr_perf_event_stop()
468 	ddr_perf_counter_enable(pmu, event->attr.config, counter, false);  in ddr_perf_event_stop()
471 	hwc->state |= PERF_HES_STOPPED;  in ddr_perf_event_stop()
476 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_del()
477 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_del()
478 	int counter = hwc->idx;  in ddr_perf_event_del()
483 	pmu->active_events--;  in ddr_perf_event_del()
484 	hwc->idx = -1;  in ddr_perf_event_del()
492 	if (ddr_pmu->events[EVENT_CYCLES_COUNTER] == NULL)  in ddr_perf_pmu_enable()
503 	if (ddr_pmu->events[EVENT_CYCLES_COUNTER] == NULL)  in ddr_perf_pmu_disable()
532 	pmu->id = ida_simple_get(&ddr_ida, 0, 0, GFP_KERNEL);  in ddr_perf_init()
533 	return pmu->id;  in ddr_perf_init()
559 		if (!pmu->events[i])  in ddr_perf_irq_handler()
562 		event = pmu->events[i];  in ddr_perf_irq_handler()
566 		if (event->hw.idx == EVENT_CYCLES_COUNTER)  in ddr_perf_irq_handler()
585 	if (cpu != pmu->cpu)  in ddr_perf_offline_cpu()
592 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);  in ddr_perf_offline_cpu()
593 	pmu->cpu = target;  in ddr_perf_offline_cpu()
595 	WARN_ON(irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu)));  in ddr_perf_offline_cpu()
614 	np = pdev->dev.of_node;  in ddr_perf_probe()
616 	pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);  in ddr_perf_probe()
618 		return -ENOMEM;  in ddr_perf_probe()
620 	num = ddr_perf_init(pmu, base, &pdev->dev);  in ddr_perf_probe()
624 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d",  in ddr_perf_probe()
627 		return -ENOMEM;  in ddr_perf_probe()
629 	pmu->devtype_data = of_device_get_match_data(&pdev->dev);  in ddr_perf_probe()
631 	pmu->cpu = raw_smp_processor_id();  in ddr_perf_probe()
638 		dev_err(&pdev->dev, "cpuhp_setup_state_multi failed\n");  in ddr_perf_probe()
642 	pmu->cpuhp_state = ret;  in ddr_perf_probe()
645 	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
647 		dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);  in ddr_perf_probe()
654 		dev_err(&pdev->dev, "Failed to get irq: %d", irq);  in ddr_perf_probe()
659 	ret = devm_request_irq(&pdev->dev, irq,  in ddr_perf_probe()
665 		dev_err(&pdev->dev, "Request irq failed: %d", ret);  in ddr_perf_probe()
669 	pmu->irq = irq;  in ddr_perf_probe()
670 	ret = irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu));  in ddr_perf_probe()
672 		dev_err(pmu->dev, "Failed to set interrupt affinity!\n");  in ddr_perf_probe()
676 	ret = perf_pmu_register(&pmu->pmu, name, -1);  in ddr_perf_probe()
683 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
685 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_probe()
687 	ida_simple_remove(&ddr_ida, pmu->id);  in ddr_perf_probe()
688 	dev_warn(&pdev->dev, "i.MX8 DDR Perf PMU failed (%d), disabled\n", ret);  in ddr_perf_probe()
696 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_remove()
697 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_remove()
698 	irq_set_affinity_hint(pmu->irq, NULL);  in ddr_perf_remove()
700 	perf_pmu_unregister(&pmu->pmu);  in ddr_perf_remove()
702 	ida_simple_remove(&ddr_ida, pmu->id);  in ddr_perf_remove()
708 		.name   = "imx-ddr-pmu",