Lines Matching +full:smmu +full:- +full:v3
1 // SPDX-License-Identifier: GPL-2.0
9 * <phys_addr_page> is the physical page address of the SMMU PMCG wrapped
15 * filter_enable - 0 = no filtering, 1 = filtering enabled
16 * filter_span - 0 = exact match, 1 = pattern match
17 * filter_stream_id - pattern to filter against
19 * To match a partial StreamID where the X most-significant bits must match
20 * but the Y least-significant bits might differ, STREAMID is programmed
22 * STREAMID[Y - 1] == 0.
23 * STREAMID[Y - 2:0] == 1 (where Y > 1).
27 * Example: perf stat -e smmuv3_pmcg_ff88840/transaction,filter_enable=1,
28 * filter_span=1,filter_stream_id=0x42/ -a netperf
31 * information is available in the SMMU documentation.
33 * SMMU events are not attributable to a CPU, so task mode and sampling
126 event->attr._config); \
139 smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL); in smmu_pmu_enable()
140 writel(SMMU_PMCG_CR_ENABLE, smmu_pmu->reg_base + SMMU_PMCG_CR); in smmu_pmu_enable()
147 writel(0, smmu_pmu->reg_base + SMMU_PMCG_CR); in smmu_pmu_disable()
148 writel(0, smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL); in smmu_pmu_disable()
154 if (smmu_pmu->counter_mask & BIT(32)) in smmu_pmu_counter_set_value()
155 writeq(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8)); in smmu_pmu_counter_set_value()
157 writel(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4)); in smmu_pmu_counter_set_value()
164 if (smmu_pmu->counter_mask & BIT(32)) in smmu_pmu_counter_get_value()
165 value = readq(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8)); in smmu_pmu_counter_get_value()
167 value = readl(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4)); in smmu_pmu_counter_get_value()
174 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENSET0); in smmu_pmu_counter_enable()
179 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0); in smmu_pmu_counter_disable()
184 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENSET0); in smmu_pmu_interrupt_enable()
190 writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0); in smmu_pmu_interrupt_disable()
196 writel(val, smmu_pmu->reg_base + SMMU_PMCG_EVTYPER(idx)); in smmu_pmu_set_evtyper()
201 writel(val, smmu_pmu->reg_base + SMMU_PMCG_SMR(idx)); in smmu_pmu_set_smr()
206 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_update()
207 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_update()
209 u32 idx = hwc->idx; in smmu_pmu_event_update()
212 prev = local64_read(&hwc->prev_count); in smmu_pmu_event_update()
214 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); in smmu_pmu_event_update()
217 delta = now - prev; in smmu_pmu_event_update()
218 delta &= smmu_pmu->counter_mask; in smmu_pmu_event_update()
220 local64_add(delta, &event->count); in smmu_pmu_event_update()
226 u32 idx = hwc->idx; in smmu_pmu_set_period()
229 if (smmu_pmu->options & SMMU_PMCG_EVCNTR_RDONLY) { in smmu_pmu_set_period()
245 new = smmu_pmu->counter_mask >> 1; in smmu_pmu_set_period()
249 local64_set(&hwc->prev_count, new); in smmu_pmu_set_period()
255 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_set_event_filter()
280 unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters; in smmu_pmu_apply_event_filter()
288 cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); in smmu_pmu_apply_event_filter()
290 * Per-counter filtering, or scheduling the first globally-filtered in smmu_pmu_apply_event_filter()
293 if (!smmu_pmu->global_filter || cur_idx == num_ctrs) { in smmu_pmu_apply_event_filter()
299 if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) { in smmu_pmu_apply_event_filter()
304 return -EAGAIN; in smmu_pmu_apply_event_filter()
311 unsigned int num_ctrs = smmu_pmu->num_counters; in smmu_pmu_get_event_idx()
313 idx = find_first_zero_bit(smmu_pmu->used_counters, num_ctrs); in smmu_pmu_get_event_idx()
316 return -EAGAIN; in smmu_pmu_get_event_idx()
322 set_bit(idx, smmu_pmu->used_counters); in smmu_pmu_get_event_idx()
330 if (new->pmu != curr->pmu) in smmu_pmu_events_compatible()
333 if (to_smmu_pmu(new->pmu)->global_filter && in smmu_pmu_events_compatible()
347 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_init()
348 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_init()
349 struct device *dev = smmu_pmu->dev; in smmu_pmu_event_init()
354 if (event->attr.type != event->pmu->type) in smmu_pmu_event_init()
355 return -ENOENT; in smmu_pmu_event_init()
357 if (hwc->sample_period) { in smmu_pmu_event_init()
359 return -EOPNOTSUPP; in smmu_pmu_event_init()
362 if (event->cpu < 0) { in smmu_pmu_event_init()
363 dev_dbg(dev, "Per-task mode not supported\n"); in smmu_pmu_event_init()
364 return -EOPNOTSUPP; in smmu_pmu_event_init()
370 (!test_bit(event_id, smmu_pmu->supported_events))) { in smmu_pmu_event_init()
372 return -EINVAL; in smmu_pmu_event_init()
376 if (!is_software_event(event->group_leader)) { in smmu_pmu_event_init()
377 if (!smmu_pmu_events_compatible(event->group_leader, event)) in smmu_pmu_event_init()
378 return -EINVAL; in smmu_pmu_event_init()
380 if (++group_num_events > smmu_pmu->num_counters) in smmu_pmu_event_init()
381 return -EINVAL; in smmu_pmu_event_init()
384 for_each_sibling_event(sibling, event->group_leader) { in smmu_pmu_event_init()
389 return -EINVAL; in smmu_pmu_event_init()
391 if (++group_num_events > smmu_pmu->num_counters) in smmu_pmu_event_init()
392 return -EINVAL; in smmu_pmu_event_init()
395 hwc->idx = -1; in smmu_pmu_event_init()
401 event->cpu = smmu_pmu->on_cpu; in smmu_pmu_event_init()
408 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_start()
409 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_start()
410 int idx = hwc->idx; in smmu_pmu_event_start()
412 hwc->state = 0; in smmu_pmu_event_start()
421 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_stop()
422 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_stop()
423 int idx = hwc->idx; in smmu_pmu_event_stop()
425 if (hwc->state & PERF_HES_STOPPED) in smmu_pmu_event_stop()
431 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; in smmu_pmu_event_stop()
436 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_add()
438 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_add()
444 hwc->idx = idx; in smmu_pmu_event_add()
445 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; in smmu_pmu_event_add()
446 smmu_pmu->events[idx] = event; in smmu_pmu_event_add()
447 local64_set(&hwc->prev_count, 0); in smmu_pmu_event_add()
462 struct hw_perf_event *hwc = &event->hw; in smmu_pmu_event_del()
463 struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); in smmu_pmu_event_del()
464 int idx = hwc->idx; in smmu_pmu_event_del()
468 smmu_pmu->events[idx] = NULL; in smmu_pmu_event_del()
469 clear_bit(idx, smmu_pmu->used_counters); in smmu_pmu_event_del()
487 return cpumap_print_to_pagebuf(true, buf, cpumask_of(smmu_pmu->on_cpu)); in smmu_pmu_cpumask_show()
511 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); in smmu_pmu_event_show()
538 if (test_bit(pmu_attr->id, smmu_pmu->supported_events)) in smmu_pmu_event_is_visible()
539 return attr->mode; in smmu_pmu_event_is_visible()
556 return sysfs_emit(page, "0x%08x\n", smmu_pmu->iidr); in smmu_pmu_identifier_attr_show()
566 if (!smmu_pmu->iidr) in smmu_pmu_identifier_attr_visible()
568 return attr->mode; in smmu_pmu_identifier_attr_visible()
585 PMU_FORMAT_ATTR(event, "config:0-15");
586 PMU_FORMAT_ATTR(filter_stream_id, "config1:0-31");
621 if (cpu != smmu_pmu->on_cpu) in smmu_pmu_offline_cpu()
628 perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target); in smmu_pmu_offline_cpu()
629 smmu_pmu->on_cpu = target; in smmu_pmu_offline_cpu()
630 WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(target))); in smmu_pmu_offline_cpu()
641 ovsr = readq(smmu_pmu->reloc_base + SMMU_PMCG_OVSSET0); in smmu_pmu_handle_irq()
645 writeq(ovsr, smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0); in smmu_pmu_handle_irq()
647 for_each_set_bit(idx, (unsigned long *)&ovsr, smmu_pmu->num_counters) { in smmu_pmu_handle_irq()
648 struct perf_event *event = smmu_pmu->events[idx]; in smmu_pmu_handle_irq()
655 hwc = &event->hw; in smmu_pmu_handle_irq()
676 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; in smmu_pmu_write_msi_msg()
679 writeq_relaxed(doorbell, pmu->reg_base + SMMU_PMCG_IRQ_CFG0); in smmu_pmu_write_msi_msg()
680 writel_relaxed(msg->data, pmu->reg_base + SMMU_PMCG_IRQ_CFG1); in smmu_pmu_write_msi_msg()
682 pmu->reg_base + SMMU_PMCG_IRQ_CFG2); in smmu_pmu_write_msi_msg()
688 struct device *dev = pmu->dev; in smmu_pmu_setup_msi()
692 writeq_relaxed(0, pmu->reg_base + SMMU_PMCG_IRQ_CFG0); in smmu_pmu_setup_msi()
695 if (!(readl(pmu->reg_base + SMMU_PMCG_CFGR) & SMMU_PMCG_CFGR_MSI)) in smmu_pmu_setup_msi()
706 pmu->irq = desc->irq; in smmu_pmu_setup_msi()
715 int irq, ret = -ENXIO; in smmu_pmu_setup_irq()
719 irq = pmu->irq; in smmu_pmu_setup_irq()
721 ret = devm_request_irq(pmu->dev, irq, smmu_pmu_handle_irq, in smmu_pmu_setup_irq()
722 flags, "smmuv3-pmu", pmu); in smmu_pmu_setup_irq()
728 u64 counter_present_mask = GENMASK_ULL(smmu_pmu->num_counters - 1, 0); in smmu_pmu_reset()
730 smmu_pmu_disable(&smmu_pmu->pmu); in smmu_pmu_reset()
734 smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0); in smmu_pmu_reset()
736 smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0); in smmu_pmu_reset()
738 smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0); in smmu_pmu_reset()
745 model = *(u32 *)dev_get_platdata(smmu_pmu->dev); in smmu_pmu_get_acpi_options()
750 smmu_pmu->options |= SMMU_PMCG_EVCNTR_RDONLY; in smmu_pmu_get_acpi_options()
754 dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options); in smmu_pmu_get_acpi_options()
765 struct device *dev = &pdev->dev; in smmu_pmu_probe()
769 return -ENOMEM; in smmu_pmu_probe()
771 smmu_pmu->dev = dev; in smmu_pmu_probe()
774 smmu_pmu->pmu = (struct pmu) { in smmu_pmu_probe()
789 smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res_0); in smmu_pmu_probe()
790 if (IS_ERR(smmu_pmu->reg_base)) in smmu_pmu_probe()
791 return PTR_ERR(smmu_pmu->reg_base); in smmu_pmu_probe()
793 cfgr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CFGR); in smmu_pmu_probe()
797 smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1); in smmu_pmu_probe()
798 if (IS_ERR(smmu_pmu->reloc_base)) in smmu_pmu_probe()
799 return PTR_ERR(smmu_pmu->reloc_base); in smmu_pmu_probe()
801 smmu_pmu->reloc_base = smmu_pmu->reg_base; in smmu_pmu_probe()
806 smmu_pmu->irq = irq; in smmu_pmu_probe()
808 ceid_64[0] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID0); in smmu_pmu_probe()
809 ceid_64[1] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID1); in smmu_pmu_probe()
810 bitmap_from_arr32(smmu_pmu->supported_events, (u32 *)ceid_64, in smmu_pmu_probe()
813 smmu_pmu->num_counters = FIELD_GET(SMMU_PMCG_CFGR_NCTR, cfgr) + 1; in smmu_pmu_probe()
815 smmu_pmu->global_filter = !!(cfgr & SMMU_PMCG_CFGR_SID_FILTER_TYPE); in smmu_pmu_probe()
818 smmu_pmu->counter_mask = GENMASK_ULL(reg_size, 0); in smmu_pmu_probe()
824 dev_err(dev, "Setup irq failed, PMU @%pa\n", &res_0->start); in smmu_pmu_probe()
828 smmu_pmu->iidr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_IIDR); in smmu_pmu_probe()
830 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "smmuv3_pmcg_%llx", in smmu_pmu_probe()
831 (res_0->start) >> SMMU_PMCG_PA_SHIFT); in smmu_pmu_probe()
833 dev_err(dev, "Create name failed, PMU @%pa\n", &res_0->start); in smmu_pmu_probe()
834 return -EINVAL; in smmu_pmu_probe()
840 smmu_pmu->on_cpu = raw_smp_processor_id(); in smmu_pmu_probe()
841 WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(smmu_pmu->on_cpu))); in smmu_pmu_probe()
844 &smmu_pmu->node); in smmu_pmu_probe()
847 err, &res_0->start); in smmu_pmu_probe()
851 err = perf_pmu_register(&smmu_pmu->pmu, name, -1); in smmu_pmu_probe()
854 err, &res_0->start); in smmu_pmu_probe()
859 &res_0->start, smmu_pmu->num_counters, in smmu_pmu_probe()
860 smmu_pmu->global_filter ? "Global(Counter0)" : in smmu_pmu_probe()
866 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); in smmu_pmu_probe()
874 perf_pmu_unregister(&smmu_pmu->pmu); in smmu_pmu_remove()
875 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); in smmu_pmu_remove()
884 smmu_pmu_disable(&smmu_pmu->pmu); in smmu_pmu_shutdown()
889 .name = "arm-smmu-v3-pmcg",