Lines Matching +full:smmu +full:- +full:v3
1 // SPDX-License-Identifier: GPL-2.0
9 * <phys_addr_page> is the physical page address of the SMMU PMCG wrapped
15 * filter_enable - 0 = no filtering, 1 = filtering enabled
16 * filter_span - 0 = exact match, 1 = pattern match
17 * filter_stream_id - pattern to filter against
19 * To match a partial StreamID where the X most-significant bits must match
20 * but the Y least-significant bits might differ, STREAMID is programmed
22 * STREAMID[Y - 1] == 0.
23 * STREAMID[Y - 2:0] == 1 (where Y > 1).
27 * Example: perf stat -e smmuv3_pmcg_ff88840/transaction,filter_enable=1,
28 * filter_span=1,filter_stream_id=0x42/ -a netperf
31 * information is available in the SMMU documentation.
33 * SMMU events are not attributable to a CPU, so task mode and sampling
91 /* IMP-DEF ID registers */
145 event->attr._config); \
158 smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL); in smmu_pmu_enable()
159 writel(SMMU_PMCG_CR_ENABLE, smmu_pmu->reg_base + SMMU_PMCG_CR); in smmu_pmu_enable()
166 writel(0, smmu_pmu->reg_base + SMMU_PMCG_CR); in smmu_pmu_disable()
167 writel(0, smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL); in smmu_pmu_disable()
173 if (smmu_pmu->counter_mask & BIT(32)) in smmu_pmu_counter_set_value()
174 writeq(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8)); in smmu_pmu_counter_set_value()
176 writel(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4)); in smmu_pmu_counter_set_value()
183 if (smmu_pmu->counter_mask & BIT(32)) in smmu_pmu_counter_get_value()
184 value = readq(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8)); in smmu_pmu_counter_get_value()
186 value = readl(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4)); in smmu_pmu_counter_get_value()
193 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENSET0); in smmu_pmu_counter_enable()
198 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0); in smmu_pmu_counter_disable()
203 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENSET0); in smmu_pmu_interrupt_enable()
209 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0); in smmu_pmu_interrupt_disable()
215 writel(val, smmu_pmu->reg_base + SMMU_PMCG_EVTYPER(idx)); in smmu_pmu_set_evtyper()
220 writel(val, smmu_pmu->reg_base + SMMU_PMCG_SMR(idx)); in smmu_pmu_set_smr()
225 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_update()
226 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_update()
228 u32 idx = hwc->idx; in smmu_pmu_event_update()
231 prev = local64_read(&hwc->prev_count); in smmu_pmu_event_update()
233 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); in smmu_pmu_event_update()
236 delta = now - prev; in smmu_pmu_event_update()
237 delta &= smmu_pmu->counter_mask; in smmu_pmu_event_update()
239 local64_add(delta, &event->count); in smmu_pmu_event_update()
245 u32 idx = hwc->idx; in smmu_pmu_set_period()
248 if (smmu_pmu->options & SMMU_PMCG_EVCNTR_RDONLY) { in smmu_pmu_set_period()
264 new = smmu_pmu->counter_mask >> 1; in smmu_pmu_set_period()
268 local64_set(&hwc->prev_count, new); in smmu_pmu_set_period()
274 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_set_event_filter()
299 unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters; in smmu_pmu_apply_event_filter()
307 cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); in smmu_pmu_apply_event_filter()
309 * Per-counter filtering, or scheduling the first globally-filtered in smmu_pmu_apply_event_filter()
312 if (!smmu_pmu->global_filter || cur_idx == num_ctrs) { in smmu_pmu_apply_event_filter()
318 if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) { in smmu_pmu_apply_event_filter()
323 return -EAGAIN; in smmu_pmu_apply_event_filter()
330 unsigned int num_ctrs = smmu_pmu->num_counters; in smmu_pmu_get_event_idx()
332 idx = find_first_zero_bit(smmu_pmu->used_counters, num_ctrs); in smmu_pmu_get_event_idx()
335 return -EAGAIN; in smmu_pmu_get_event_idx()
341 set_bit(idx, smmu_pmu->used_counters); in smmu_pmu_get_event_idx()
349 if (new->pmu != curr->pmu) in smmu_pmu_events_compatible()
352 if (to_smmu_pmu(new->pmu)->global_filter && in smmu_pmu_events_compatible()
366 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_init()
367 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_init()
368 struct device *dev = smmu_pmu->dev; in smmu_pmu_event_init()
373 if (event->attr.type != event->pmu->type) in smmu_pmu_event_init()
374 return -ENOENT; in smmu_pmu_event_init()
376 if (hwc->sample_period) { in smmu_pmu_event_init()
378 return -EOPNOTSUPP; in smmu_pmu_event_init()
381 if (event->cpu < 0) { in smmu_pmu_event_init()
382 dev_dbg(dev, "Per-task mode not supported\n"); in smmu_pmu_event_init()
383 return -EOPNOTSUPP; in smmu_pmu_event_init()
389 (!test_bit(event_id, smmu_pmu->supported_events))) { in smmu_pmu_event_init()
391 return -EINVAL; in smmu_pmu_event_init()
395 if (!is_software_event(event->group_leader)) { in smmu_pmu_event_init()
396 if (!smmu_pmu_events_compatible(event->group_leader, event)) in smmu_pmu_event_init()
397 return -EINVAL; in smmu_pmu_event_init()
399 if (++group_num_events > smmu_pmu->num_counters) in smmu_pmu_event_init()
400 return -EINVAL; in smmu_pmu_event_init()
403 for_each_sibling_event(sibling, event->group_leader) { in smmu_pmu_event_init()
408 return -EINVAL; in smmu_pmu_event_init()
410 if (++group_num_events > smmu_pmu->num_counters) in smmu_pmu_event_init()
411 return -EINVAL; in smmu_pmu_event_init()
414 hwc->idx = -1; in smmu_pmu_event_init()
420 event->cpu = smmu_pmu->on_cpu; in smmu_pmu_event_init()
427 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_start()
428 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_start()
429 int idx = hwc->idx; in smmu_pmu_event_start()
431 hwc->state = 0; in smmu_pmu_event_start()
440 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_stop()
441 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_stop()
442 int idx = hwc->idx; in smmu_pmu_event_stop()
444 if (hwc->state & PERF_HES_STOPPED) in smmu_pmu_event_stop()
450 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; in smmu_pmu_event_stop()
455 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_add()
457 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_add()
463 hwc->idx = idx; in smmu_pmu_event_add()
464 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; in smmu_pmu_event_add()
465 smmu_pmu->events[idx] = event; in smmu_pmu_event_add()
466 local64_set(&hwc->prev_count, 0); in smmu_pmu_event_add()
481 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_del()
482 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_del()
483 int idx = hwc->idx; in smmu_pmu_event_del()
487 smmu_pmu->events[idx] = NULL; in smmu_pmu_event_del()
488 clear_bit(idx, smmu_pmu->used_counters); in smmu_pmu_event_del()
506 return cpumap_print_to_pagebuf(true, buf, cpumask_of(smmu_pmu->on_cpu)); in smmu_pmu_cpumask_show()
530 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); in smmu_pmu_event_show()
557 if (test_bit(pmu_attr->id, smmu_pmu->supported_events)) in smmu_pmu_event_is_visible()
558 return attr->mode; in smmu_pmu_event_is_visible()
575 return sysfs_emit(page, "0x%08x\n", smmu_pmu->iidr); in smmu_pmu_identifier_attr_show()
585 if (!smmu_pmu->iidr) in smmu_pmu_identifier_attr_visible()
587 return attr->mode; in smmu_pmu_identifier_attr_visible()
604 PMU_FORMAT_ATTR(event, "config:0-15");
605 PMU_FORMAT_ATTR(filter_stream_id, "config1:0-31");
640 if (cpu != smmu_pmu->on_cpu) in smmu_pmu_offline_cpu()
647 perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target); in smmu_pmu_offline_cpu()
648 smmu_pmu->on_cpu = target; in smmu_pmu_offline_cpu()
649 WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(target))); in smmu_pmu_offline_cpu()
661 ovsr = readq(smmu_pmu->reloc_base + SMMU_PMCG_OVSSET0); in smmu_pmu_handle_irq()
665 writeq(ovsr, smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0); in smmu_pmu_handle_irq()
668 for_each_set_bit(idx, ovs, smmu_pmu->num_counters) { in smmu_pmu_handle_irq()
669 struct perf_event *event = smmu_pmu->events[idx]; in smmu_pmu_handle_irq()
676 hwc = &event->hw; in smmu_pmu_handle_irq()
697 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; in smmu_pmu_write_msi_msg()
700 writeq_relaxed(doorbell, pmu->reg_base + SMMU_PMCG_IRQ_CFG0); in smmu_pmu_write_msi_msg()
701 writel_relaxed(msg->data, pmu->reg_base + SMMU_PMCG_IRQ_CFG1); in smmu_pmu_write_msi_msg()
703 pmu->reg_base + SMMU_PMCG_IRQ_CFG2); in smmu_pmu_write_msi_msg()
708 struct device *dev = pmu->dev; in smmu_pmu_setup_msi()
712 writeq_relaxed(0, pmu->reg_base + SMMU_PMCG_IRQ_CFG0); in smmu_pmu_setup_msi()
715 if (!(readl(pmu->reg_base + SMMU_PMCG_CFGR) & SMMU_PMCG_CFGR_MSI)) in smmu_pmu_setup_msi()
724 pmu->irq = msi_get_virq(dev, 0); in smmu_pmu_setup_msi()
733 int irq, ret = -ENXIO; in smmu_pmu_setup_irq()
737 irq = pmu->irq; in smmu_pmu_setup_irq()
739 ret = devm_request_irq(pmu->dev, irq, smmu_pmu_handle_irq, in smmu_pmu_setup_irq()
740 flags, "smmuv3-pmu", pmu); in smmu_pmu_setup_irq()
746 u64 counter_present_mask = GENMASK_ULL(smmu_pmu->num_counters - 1, 0); in smmu_pmu_reset()
748 smmu_pmu_disable(&smmu_pmu->pmu); in smmu_pmu_reset()
752 smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0); in smmu_pmu_reset()
754 smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0); in smmu_pmu_reset()
756 smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0); in smmu_pmu_reset()
763 model = *(u32 *)dev_get_platdata(smmu_pmu->dev); in smmu_pmu_get_acpi_options()
768 smmu_pmu->options |= SMMU_PMCG_EVCNTR_RDONLY; in smmu_pmu_get_acpi_options()
772 dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options); in smmu_pmu_get_acpi_options()
777 return of_device_is_compatible(smmu_pmu->dev->of_node, in smmu_pmu_coresight_id_regs()
778 "arm,mmu-600-pmcg"); in smmu_pmu_coresight_id_regs()
783 u32 iidr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_IIDR); in smmu_pmu_get_iidr()
786 u32 pidr0 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR0); in smmu_pmu_get_iidr()
787 u32 pidr1 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR1); in smmu_pmu_get_iidr()
788 u32 pidr2 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR2); in smmu_pmu_get_iidr()
789 u32 pidr3 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR3); in smmu_pmu_get_iidr()
790 u32 pidr4 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR4); in smmu_pmu_get_iidr()
807 smmu_pmu->iidr = iidr; in smmu_pmu_get_iidr()
818 struct device *dev = &pdev->dev; in smmu_pmu_probe()
822 return -ENOMEM; in smmu_pmu_probe()
824 smmu_pmu->dev = dev; in smmu_pmu_probe()
827 smmu_pmu->pmu = (struct pmu) { in smmu_pmu_probe()
842 smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res_0); in smmu_pmu_probe()
843 if (IS_ERR(smmu_pmu->reg_base)) in smmu_pmu_probe()
844 return PTR_ERR(smmu_pmu->reg_base); in smmu_pmu_probe()
846 cfgr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CFGR); in smmu_pmu_probe()
850 smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1); in smmu_pmu_probe()
851 if (IS_ERR(smmu_pmu->reloc_base)) in smmu_pmu_probe()
852 return PTR_ERR(smmu_pmu->reloc_base); in smmu_pmu_probe()
854 smmu_pmu->reloc_base = smmu_pmu->reg_base; in smmu_pmu_probe()
859 smmu_pmu->irq = irq; in smmu_pmu_probe()
861 ceid_64[0] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID0); in smmu_pmu_probe()
862 ceid_64[1] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID1); in smmu_pmu_probe()
863 bitmap_from_arr32(smmu_pmu->supported_events, (u32 *)ceid_64, in smmu_pmu_probe()
866 smmu_pmu->num_counters = FIELD_GET(SMMU_PMCG_CFGR_NCTR, cfgr) + 1; in smmu_pmu_probe()
868 smmu_pmu->global_filter = !!(cfgr & SMMU_PMCG_CFGR_SID_FILTER_TYPE); in smmu_pmu_probe()
871 smmu_pmu->counter_mask = GENMASK_ULL(reg_size, 0); in smmu_pmu_probe()
877 dev_err(dev, "Setup irq failed, PMU @%pa\n", &res_0->start); in smmu_pmu_probe()
883 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "smmuv3_pmcg_%llx", in smmu_pmu_probe()
884 (res_0->start) >> SMMU_PMCG_PA_SHIFT); in smmu_pmu_probe()
886 dev_err(dev, "Create name failed, PMU @%pa\n", &res_0->start); in smmu_pmu_probe()
887 return -EINVAL; in smmu_pmu_probe()
890 if (!dev->of_node) in smmu_pmu_probe()
894 smmu_pmu->on_cpu = raw_smp_processor_id(); in smmu_pmu_probe()
895 WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(smmu_pmu->on_cpu))); in smmu_pmu_probe()
898 &smmu_pmu->node); in smmu_pmu_probe()
901 err, &res_0->start); in smmu_pmu_probe()
905 err = perf_pmu_register(&smmu_pmu->pmu, name, -1); in smmu_pmu_probe()
908 err, &res_0->start); in smmu_pmu_probe()
913 &res_0->start, smmu_pmu->num_counters, in smmu_pmu_probe()
914 smmu_pmu->global_filter ? "Global(Counter0)" : in smmu_pmu_probe()
920 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); in smmu_pmu_probe()
928 perf_pmu_unregister(&smmu_pmu->pmu); in smmu_pmu_remove()
929 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); in smmu_pmu_remove()
938 smmu_pmu_disable(&smmu_pmu->pmu); in smmu_pmu_shutdown()
943 { .compatible = "arm,smmu-v3-pmcg" },
951 .name = "arm-smmu-v3-pmcg",