1 // SPDX-License-Identifier: GPL-2.0-only
3  * Perf support for the Statistical Profiling Extension, introduced as
77 /* Convert a free-running index from perf into an SPE buffer offset */
78 #define PERF_IDX2OFF(idx, buf)	((idx) % ((buf)->nr_pages << PAGE_SHIFT))
106 		return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);  in arm_spe_pmu_cap_get()
110 		return spe_pmu->counter_sz;  in arm_spe_pmu_cap_get()
112 		return spe_pmu->min_period;  in arm_spe_pmu_cap_get()
127 	int cap = (long)ea->var;  in arm_spe_pmu_cap_show()
186 	(lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
198 	((((attr)->cfg) >> lo) & GENMASK(hi - lo, 0))
239 	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);  in cpumask_show()
262 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmscr()
269 	if (!attr->exclude_user)  in arm_spe_event_to_pmscr()
272 	if (!attr->exclude_kernel)  in arm_spe_event_to_pmscr()
283 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_event_sanitise_period()
284 	u64 period = event->hw.sample_period;  in arm_spe_event_sanitise_period()
288 	if (period < spe_pmu->min_period)  in arm_spe_event_sanitise_period()
289 		period = spe_pmu->min_period;  in arm_spe_event_sanitise_period()
295 	event->hw.sample_period = period;  in arm_spe_event_sanitise_period()
300 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsirr()
306 	reg |= event->hw.sample_period;  in arm_spe_event_to_pmsirr()
313 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsfcr()
334 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsevfr()
340 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmslatfr()
348 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_pad_buf()
350 	memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);  in arm_spe_pmu_pad_buf()
351 	if (!buf->snapshot)  in arm_spe_pmu_pad_buf()
358 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_snapshot_off()
359 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_snapshot_off()
360 	u64 limit = buf->nr_pages * PAGE_SIZE;  in arm_spe_pmu_next_snapshot_off()
363 	 * The trace format isn't parseable in reverse, so clamp  in arm_spe_pmu_next_snapshot_off()
364 	 * the limit to half of the buffer size in snapshot mode  in arm_spe_pmu_next_snapshot_off()
365 	 * so that the worst case is half a buffer of records, as  in arm_spe_pmu_next_snapshot_off()
375 	if (limit - head < spe_pmu->max_record_sz) {  in arm_spe_pmu_next_snapshot_off()
376 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_snapshot_off()
377 		handle->head = PERF_IDX2OFF(limit, buf);  in arm_spe_pmu_next_snapshot_off()
378 		limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;  in arm_spe_pmu_next_snapshot_off()
386 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in __arm_spe_pmu_next_off()
388 	const u64 bufsize = buf->nr_pages * PAGE_SIZE;  in __arm_spe_pmu_next_off()
396 	 *    a record when generating a buffer management event.  in __arm_spe_pmu_next_off()
398 	 * 2. We used perf_aux_output_skip to consume handle->size bytes  in __arm_spe_pmu_next_off()
403 	 * moving the head index. If we run out of buffer space, we'll  in __arm_spe_pmu_next_off()
404 	 * reduce handle->size to zero and end up reporting truncation.  in __arm_spe_pmu_next_off()
406 	head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
407 	if (!IS_ALIGNED(head, spe_pmu->align)) {  in __arm_spe_pmu_next_off()
408 		unsigned long delta = roundup(head, spe_pmu->align) - head;  in __arm_spe_pmu_next_off()
410 		delta = min(delta, handle->size);  in __arm_spe_pmu_next_off()
412 		head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
416 	if (!handle->size)  in __arm_spe_pmu_next_off()
420 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __arm_spe_pmu_next_off()
421 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __arm_spe_pmu_next_off()
425 	 * if we see head == tail we know that the buffer is empty. If  in __arm_spe_pmu_next_off()
441 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __arm_spe_pmu_next_off()
447 	arm_spe_pmu_pad_buf(handle, handle->size);  in __arm_spe_pmu_next_off()
457 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_off()
459 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_off()
462 	 * If the head has come too close to the end of the buffer,  in arm_spe_pmu_next_off()
465 	if (limit && (limit - head < spe_pmu->max_record_sz)) {  in arm_spe_pmu_next_off()
466 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_off()
482 		event->hw.state |= PERF_HES_STOPPED;  in arm_spe_perf_aux_output_begin()
491 	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)  in arm_spe_perf_aux_output_begin()
496 	limit += (u64)buf->base;  in arm_spe_perf_aux_output_begin()
497 	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_begin()
509 	offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;  in arm_spe_perf_aux_output_end()
510 	size = offset - PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_end()
512 	if (buf->snapshot)  in arm_spe_perf_aux_output_end()
513 		handle->head = offset;  in arm_spe_perf_aux_output_end()
528 	/* Disable the profiling buffer */  in arm_spe_pmu_disable_and_drain_local()
568 	/* We only expect buffer management events */  in arm_spe_pmu_buf_get_fault_act()
575 		err_str = "Unexpected buffer fault";  in arm_spe_pmu_buf_get_fault_act()
582 	/* Buffer management event */  in arm_spe_pmu_buf_get_fault_act()
589 		err_str = "Unknown buffer status code";  in arm_spe_pmu_buf_get_fault_act()
607 	struct perf_event *event = handle->event;  in arm_spe_pmu_irq_handler()
619 	 * profiling buffer in response to a TRUNCATION flag.  in arm_spe_pmu_irq_handler()
627 		 * buffer enabled is a recipe waiting to happen. Since  in arm_spe_pmu_irq_handler()
629 		 * that the profiling buffer is disabled explicitly before  in arm_spe_pmu_irq_handler()
636 		 * We handled the fault (the buffer was full), so resume  in arm_spe_pmu_irq_handler()
641 		if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {  in arm_spe_pmu_irq_handler()
651 	/* The buffer pointers are now sane, so resume profiling. */  in arm_spe_pmu_irq_handler()
672 	struct perf_event_attr *attr = &event->attr;  in arm_spe_pmu_event_init()
673 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_event_init()
675 	/* This is, of course, deeply driver-specific */  in arm_spe_pmu_event_init()
676 	if (attr->type != event->pmu->type)  in arm_spe_pmu_event_init()
677 		return -ENOENT;  in arm_spe_pmu_event_init()
679 	if (event->cpu >= 0 &&  in arm_spe_pmu_event_init()
680 	    !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_event_init()
681 		return -ENOENT;  in arm_spe_pmu_event_init()
683 	if (arm_spe_event_to_pmsevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver))  in arm_spe_pmu_event_init()
684 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
686 	if (attr->exclude_idle)  in arm_spe_pmu_event_init()
687 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
690 	 * Feedback-directed frequency throttling doesn't work when we  in arm_spe_pmu_event_init()
691 	 * have a buffer of samples. We'd need to manually count the  in arm_spe_pmu_event_init()
692 	 * samples in the buffer when it fills up and adjust the event  in arm_spe_pmu_event_init()
696 	if (attr->freq)  in arm_spe_pmu_event_init()
697 		return -EINVAL;  in arm_spe_pmu_event_init()
701 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))  in arm_spe_pmu_event_init()
702 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
705 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))  in arm_spe_pmu_event_init()
706 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
709 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))  in arm_spe_pmu_event_init()
710 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
717 		return -EACCES;  in arm_spe_pmu_event_init()
725 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_start()
726 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_start()
727 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_start()
729 	hwc->state = 0;  in arm_spe_pmu_start()
731 	if (hwc->state)  in arm_spe_pmu_start()
747 		reg = local64_read(&hwc->period_left);  in arm_spe_pmu_start()
758 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_stop()
759 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_stop()
760 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_stop()
763 	if (hwc->state & PERF_HES_STOPPED)  in arm_spe_pmu_stop()
766 	/* Stop all trace generation */  in arm_spe_pmu_stop()
772 		 * to this buffer, since we might be on the context-switch  in arm_spe_pmu_stop()
790 		local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));  in arm_spe_pmu_stop()
791 		hwc->state |= PERF_HES_UPTODATE;  in arm_spe_pmu_stop()
794 	hwc->state |= PERF_HES_STOPPED;  in arm_spe_pmu_stop()
800 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_add()
801 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_add()
802 	int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;  in arm_spe_pmu_add()
804 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_add()
805 		return -ENOENT;  in arm_spe_pmu_add()
807 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;  in arm_spe_pmu_add()
811 		if (hwc->state & PERF_HES_STOPPED)  in arm_spe_pmu_add()
812 			ret = -EINVAL;  in arm_spe_pmu_add()
830 	int i, cpu = event->cpu;  in arm_spe_pmu_setup_aux()
840 	 * we can effectively treat the buffer as consisting of two equal  in arm_spe_pmu_setup_aux()
847 	if (cpu == -1)  in arm_spe_pmu_setup_aux()
861 	buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);  in arm_spe_pmu_setup_aux()
862 	if (!buf->base)  in arm_spe_pmu_setup_aux()
865 	buf->nr_pages	= nr_pages;  in arm_spe_pmu_setup_aux()
866 	buf->snapshot	= snapshot;  in arm_spe_pmu_setup_aux()
882 	vunmap(buf->base);  in arm_spe_pmu_free_aux()
889 	static atomic_t pmu_idx = ATOMIC_INIT(-1);  in arm_spe_pmu_perf_init()
893 	struct device *dev = &spe_pmu->pdev->dev;  in arm_spe_pmu_perf_init()
895 	spe_pmu->pmu = (struct pmu) {  in arm_spe_pmu_perf_init()
901 		 * we can support per-task profiling (which is not possible  in arm_spe_pmu_perf_init()
904 		 * perf_event_open, since the aux buffer is not setup until  in arm_spe_pmu_perf_init()
907 		 * once the buffer has been created.  in arm_spe_pmu_perf_init()
924 		return -ENOMEM;  in arm_spe_pmu_perf_init()
927 	return perf_pmu_register(&spe_pmu->pmu, name, -1);  in arm_spe_pmu_perf_init()
932 	perf_pmu_unregister(&spe_pmu->pmu);  in arm_spe_pmu_perf_destroy()
940 	struct device *dev = &spe_pmu->pdev->dev;  in __arm_spe_pmu_dev_probe()
950 	spe_pmu->pmsver = (u16)fld;  in __arm_spe_pmu_dev_probe()
956 			"profiling buffer owned by higher exception level\n");  in __arm_spe_pmu_dev_probe()
960 	/* Minimum alignment. If it's out-of-range, then fail the probe */  in __arm_spe_pmu_dev_probe()
962 	spe_pmu->align = 1 << fld;  in __arm_spe_pmu_dev_probe()
963 	if (spe_pmu->align > SZ_2K) {  in __arm_spe_pmu_dev_probe()
972 		spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT;  in __arm_spe_pmu_dev_probe()
975 		spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP;  in __arm_spe_pmu_dev_probe()
978 		spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT;  in __arm_spe_pmu_dev_probe()
981 		spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST;  in __arm_spe_pmu_dev_probe()
984 		spe_pmu->features |= SPE_PMU_FEAT_LDS;  in __arm_spe_pmu_dev_probe()
987 		spe_pmu->features |= SPE_PMU_FEAT_ERND;  in __arm_spe_pmu_dev_probe()
989 	/* This field has a spaced out encoding, so just use a look-up */  in __arm_spe_pmu_dev_probe()
993 		spe_pmu->min_period = 256;  in __arm_spe_pmu_dev_probe()
996 		spe_pmu->min_period = 512;  in __arm_spe_pmu_dev_probe()
999 		spe_pmu->min_period = 768;  in __arm_spe_pmu_dev_probe()
1002 		spe_pmu->min_period = 1024;  in __arm_spe_pmu_dev_probe()
1005 		spe_pmu->min_period = 1536;  in __arm_spe_pmu_dev_probe()
1008 		spe_pmu->min_period = 2048;  in __arm_spe_pmu_dev_probe()
1011 		spe_pmu->min_period = 3072;  in __arm_spe_pmu_dev_probe()
1018 		spe_pmu->min_period = 4096;  in __arm_spe_pmu_dev_probe()
1021 	/* Maximum record size. If it's out-of-range, then fail the probe */  in __arm_spe_pmu_dev_probe()
1023 	spe_pmu->max_record_sz = 1 << fld;  in __arm_spe_pmu_dev_probe()
1024 	if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) {  in __arm_spe_pmu_dev_probe()
1037 		spe_pmu->counter_sz = 12;  in __arm_spe_pmu_dev_probe()
1042 		 cpumask_pr_args(&spe_pmu->supported_cpus),  in __arm_spe_pmu_dev_probe()
1043 		 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);  in __arm_spe_pmu_dev_probe()
1045 	spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;  in __arm_spe_pmu_dev_probe()
1056 	/* Reset the buffer base pointer */  in __arm_spe_pmu_reset_local()
1070 	enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);  in __arm_spe_pmu_setup_one()
1077 	disable_percpu_irq(spe_pmu->irq);  in __arm_spe_pmu_stop_one()
1086 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_startup()
1098 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_teardown()
1108 	cpumask_t *mask = &spe_pmu->supported_cpus;  in arm_spe_pmu_dev_init()
1112 	if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))  in arm_spe_pmu_dev_init()
1113 		return -ENXIO;  in arm_spe_pmu_dev_init()
1116 	ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,  in arm_spe_pmu_dev_init()
1117 				 spe_pmu->handle);  in arm_spe_pmu_dev_init()
1127 				       &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_init()
1129 		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_init()
1136 	cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_teardown()
1137 	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_teardown()
1143 	struct platform_device *pdev = spe_pmu->pdev;  in arm_spe_pmu_irq_probe()
1147 		return -ENXIO;  in arm_spe_pmu_irq_probe()
1150 		dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);  in arm_spe_pmu_irq_probe()
1151 		return -EINVAL;  in arm_spe_pmu_irq_probe()
1154 	if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {  in arm_spe_pmu_irq_probe()
1155 		dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq);  in arm_spe_pmu_irq_probe()
1156 		return -EINVAL;  in arm_spe_pmu_irq_probe()
1159 	spe_pmu->irq = irq;  in arm_spe_pmu_irq_probe()
1164 	{ .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
1179 	struct device *dev = &pdev->dev;  in arm_spe_pmu_device_probe()
1183 	 * buffer will fault and prematurely terminate the AUX session.  in arm_spe_pmu_device_probe()
1186 …dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command …  in arm_spe_pmu_device_probe()
1187 		return -EPERM;  in arm_spe_pmu_device_probe()
1192 		return -ENOMEM;  in arm_spe_pmu_device_probe()
1194 	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));  in arm_spe_pmu_device_probe()
1195 	if (!spe_pmu->handle)  in arm_spe_pmu_device_probe()
1196 		return -ENOMEM;  in arm_spe_pmu_device_probe()
1198 	spe_pmu->pdev = pdev;  in arm_spe_pmu_device_probe()
1218 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_probe()
1228 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_remove()
1270 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension");