1 // SPDX-License-Identifier: GPL-2.0
3  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
5  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
9  *  by the ARMv7 Oprofile code.
11  * Cortex-A8 has up to 4 configurable performance counters and
13  * Cortex-A9 has up to 31 configurable performance counters and
33  * Common ARMv7 event types
55  *  - all (taken) branch instructions,
56  *  - instructions that explicitly write the PC,
57  *  - exception generating instructions.
82 /* ARMv7 Cortex-A8 specific event types */
88 /* ARMv7 Cortex-A9 specific event types */
93 /* ARMv7 Cortex-A5 specific event types */
97 /* ARMv7 Cortex-A15 specific event types */
113 /* ARMv7 Cortex-A12 specific event types */
124 /* ARMv7 Krait specific event types */
136 /* ARMv7 Scorpion specific event types */
152  * Cortex-A8 HW events mapping
205  * Cortex-A9 HW events mapping
249  * Cortex-A5 HW events mapping
295  * Cortex-A15 HW events mapping
344  * Cortex-A7 HW events mapping
393  * Cortex-A12 HW events mapping
535 PMU_FORMAT_ATTR(event, "config:0-7");
657 	(ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
660 #define	ARMV7_COUNTER_MASK	(ARMV7_MAX_COUNTERS - 1)
663  * ARMv7 low level PMNC access
670 	(((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
673  * Per-CPU PMNC: config reg
680 #define ARMV7_PMNC_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
707 #define ARMV7_SDER_SUNIDEN	BIT(1) /* Permit non-invasive debug */
748 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_read_counter()
749 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_read_counter()
750 	int idx = hwc->idx;  in armv7pmu_read_counter()
768 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_write_counter()
769 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_write_counter()
770 	int idx = hwc->idx;  in armv7pmu_write_counter()
874 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_enable_event()
875 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_enable_event()
876 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in armv7pmu_enable_event()
877 	int idx = hwc->idx;  in armv7pmu_enable_event()
889 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in armv7pmu_enable_event()
901 	if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)  in armv7pmu_enable_event()
902 		armv7_pmnc_write_evtsel(idx, hwc->config_base);  in armv7pmu_enable_event()
914 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in armv7pmu_enable_event()
920 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_disable_event()
921 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_disable_event()
922 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in armv7pmu_disable_event()
923 	int idx = hwc->idx;  in armv7pmu_disable_event()
934 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in armv7pmu_disable_event()
946 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in armv7pmu_disable_event()
953 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv7pmu_handle_irq()
973 	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {  in armv7pmu_handle_irq()
974 		struct perf_event *event = cpuc->events[idx];  in armv7pmu_handle_irq()
988 		hwc = &event->hw;  in armv7pmu_handle_irq()
990 		perf_sample_data_init(&data, 0, hwc->last_period);  in armv7pmu_handle_irq()
995 			cpu_pmu->disable(event);  in armv7pmu_handle_irq()
1013 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in armv7pmu_start()
1015 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in armv7pmu_start()
1018 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in armv7pmu_start()
1024 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in armv7pmu_stop()
1026 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in armv7pmu_stop()
1029 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in armv7pmu_stop()
1036 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_get_event_idx()
1037 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_get_event_idx()
1038 	unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;  in armv7pmu_get_event_idx()
1042 		if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))  in armv7pmu_get_event_idx()
1043 			return -EAGAIN;  in armv7pmu_get_event_idx()
1052 	for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {  in armv7pmu_get_event_idx()
1053 		if (!test_and_set_bit(idx, cpuc->used_mask))  in armv7pmu_get_event_idx()
1058 	return -EAGAIN;  in armv7pmu_get_event_idx()
1064 	clear_bit(event->hw.idx, cpuc->used_mask);  in armv7pmu_clear_event_idx()
1075 	if (attr->exclude_idle)  in armv7pmu_set_event_filter()
1076 		return -EPERM;  in armv7pmu_set_event_filter()
1077 	if (attr->exclude_user)  in armv7pmu_set_event_filter()
1079 	if (attr->exclude_kernel)  in armv7pmu_set_event_filter()
1081 	if (!attr->exclude_hv)  in armv7pmu_set_event_filter()
1088 	event->config_base = config_base;  in armv7pmu_set_event_filter()
1096 	u32 idx, nb_cnt = cpu_pmu->num_events, val;  in armv7pmu_reset()
1098 	if (cpu_pmu->secure_access) {  in armv7pmu_reset()
1170 	cpu_pmu->handle_irq	= armv7pmu_handle_irq;  in armv7pmu_init()
1171 	cpu_pmu->enable		= armv7pmu_enable_event;  in armv7pmu_init()
1172 	cpu_pmu->disable	= armv7pmu_disable_event;  in armv7pmu_init()
1173 	cpu_pmu->read_counter	= armv7pmu_read_counter;  in armv7pmu_init()
1174 	cpu_pmu->write_counter	= armv7pmu_write_counter;  in armv7pmu_init()
1175 	cpu_pmu->get_event_idx	= armv7pmu_get_event_idx;  in armv7pmu_init()
1176 	cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx;  in armv7pmu_init()
1177 	cpu_pmu->start		= armv7pmu_start;  in armv7pmu_init()
1178 	cpu_pmu->stop		= armv7pmu_stop;  in armv7pmu_init()
1179 	cpu_pmu->reset		= armv7pmu_reset;  in armv7pmu_init()
1195 	return smp_call_function_any(&arm_pmu->supported_cpus,  in armv7_probe_num_events()
1197 				     &arm_pmu->num_events, 1);  in armv7_probe_num_events()
1203 	cpu_pmu->name		= "armv7_cortex_a8";  in armv7_a8_pmu_init()
1204 	cpu_pmu->map_event	= armv7_a8_map_event;  in armv7_a8_pmu_init()
1205 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a8_pmu_init()
1207 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a8_pmu_init()
1215 	cpu_pmu->name		= "armv7_cortex_a9";  in armv7_a9_pmu_init()
1216 	cpu_pmu->map_event	= armv7_a9_map_event;  in armv7_a9_pmu_init()
1217 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a9_pmu_init()
1219 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a9_pmu_init()
1227 	cpu_pmu->name		= "armv7_cortex_a5";  in armv7_a5_pmu_init()
1228 	cpu_pmu->map_event	= armv7_a5_map_event;  in armv7_a5_pmu_init()
1229 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a5_pmu_init()
1231 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a5_pmu_init()
1239 	cpu_pmu->name		= "armv7_cortex_a15";  in armv7_a15_pmu_init()
1240 	cpu_pmu->map_event	= armv7_a15_map_event;  in armv7_a15_pmu_init()
1241 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in armv7_a15_pmu_init()
1242 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a15_pmu_init()
1244 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a15_pmu_init()
1252 	cpu_pmu->name		= "armv7_cortex_a7";  in armv7_a7_pmu_init()
1253 	cpu_pmu->map_event	= armv7_a7_map_event;  in armv7_a7_pmu_init()
1254 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in armv7_a7_pmu_init()
1255 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a7_pmu_init()
1257 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a7_pmu_init()
1265 	cpu_pmu->name		= "armv7_cortex_a12";  in armv7_a12_pmu_init()
1266 	cpu_pmu->map_event	= armv7_a12_map_event;  in armv7_a12_pmu_init()
1267 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in armv7_a12_pmu_init()
1268 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a12_pmu_init()
1270 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a12_pmu_init()
1278 	cpu_pmu->name = "armv7_cortex_a17";  in armv7_a17_pmu_init()
1279 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a17_pmu_init()
1281 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a17_pmu_init()
1290  *            +--------------------------------+
1292  *            +--------------------------------+
1294  *            +--------------------------------+
1296  *            +--------------------------------+
1298  *            +--------------------------------+
1303  *      hwc->config_base = 0xNRCCG
1435 	/* Mix in mode-exclusion bits */  in krait_evt_setup()
1496 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_disable_event()
1497 	int idx = hwc->idx;  in krait_pmu_disable_event()
1498 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in krait_pmu_disable_event()
1499 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in krait_pmu_disable_event()
1502 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in krait_pmu_disable_event()
1510 	if (hwc->config_base & KRAIT_EVENT_MASK)  in krait_pmu_disable_event()
1511 		krait_clearpmu(hwc->config_base);  in krait_pmu_disable_event()
1516 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in krait_pmu_disable_event()
1522 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_enable_event()
1523 	int idx = hwc->idx;  in krait_pmu_enable_event()
1524 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in krait_pmu_enable_event()
1525 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in krait_pmu_enable_event()
1531 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in krait_pmu_enable_event()
1541 	if (hwc->config_base & KRAIT_EVENT_MASK)  in krait_pmu_enable_event()
1542 		krait_evt_setup(idx, hwc->config_base);  in krait_pmu_enable_event()
1544 		armv7_pmnc_write_evtsel(idx, hwc->config_base);  in krait_pmu_enable_event()
1552 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in krait_pmu_enable_event()
1559 	u32 idx, nb_cnt = cpu_pmu->num_events;  in krait_pmu_reset()
1584 	struct hw_perf_event *hwc = &event->hw;  in krait_event_to_bit()
1585 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in krait_event_to_bit()
1587 	if (hwc->config_base & VENUM_EVENT)  in krait_event_to_bit()
1591 	bit -= krait_get_pmresrn_event(0);  in krait_event_to_bit()
1610 	int bit = -1;  in krait_pmu_get_event_idx()
1611 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_get_event_idx()
1612 	unsigned int region = EVENT_REGION(hwc->config_base);  in krait_pmu_get_event_idx()
1613 	unsigned int code = EVENT_CODE(hwc->config_base);  in krait_pmu_get_event_idx()
1614 	unsigned int group = EVENT_GROUP(hwc->config_base);  in krait_pmu_get_event_idx()
1615 	bool venum_event = EVENT_VENUM(hwc->config_base);  in krait_pmu_get_event_idx()
1616 	bool krait_event = EVENT_CPU(hwc->config_base);  in krait_pmu_get_event_idx()
1621 			return -EINVAL;  in krait_pmu_get_event_idx()
1623 			return -EINVAL;  in krait_pmu_get_event_idx()
1626 		if (test_and_set_bit(bit, cpuc->used_mask))  in krait_pmu_get_event_idx()
1627 			return -EAGAIN;  in krait_pmu_get_event_idx()
1632 		clear_bit(bit, cpuc->used_mask);  in krait_pmu_get_event_idx()
1641 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_clear_event_idx()
1642 	unsigned int region = EVENT_REGION(hwc->config_base);  in krait_pmu_clear_event_idx()
1643 	unsigned int group = EVENT_GROUP(hwc->config_base);  in krait_pmu_clear_event_idx()
1644 	bool venum_event = EVENT_VENUM(hwc->config_base);  in krait_pmu_clear_event_idx()
1645 	bool krait_event = EVENT_CPU(hwc->config_base);  in krait_pmu_clear_event_idx()
1650 		clear_bit(bit, cpuc->used_mask);  in krait_pmu_clear_event_idx()
1657 	cpu_pmu->name		= "armv7_krait";  in krait_pmu_init()
1659 	if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,  in krait_pmu_init()
1660 				  "qcom,no-pc-write"))  in krait_pmu_init()
1661 		cpu_pmu->map_event = krait_map_event_no_branch;  in krait_pmu_init()
1663 		cpu_pmu->map_event = krait_map_event;  in krait_pmu_init()
1664 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in krait_pmu_init()
1665 	cpu_pmu->reset		= krait_pmu_reset;  in krait_pmu_init()
1666 	cpu_pmu->enable		= krait_pmu_enable_event;  in krait_pmu_init()
1667 	cpu_pmu->disable	= krait_pmu_disable_event;  in krait_pmu_init()
1668 	cpu_pmu->get_event_idx	= krait_pmu_get_event_idx;  in krait_pmu_init()
1669 	cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;  in krait_pmu_init()
1677  *            +--------------------------------+
1679  *            +--------------------------------+
1681  *            +--------------------------------+
1683  *            +--------------------------------+
1685  *            +--------------------------------+
1687  *            +--------------------------------+
1693  *      hwc->config_base = 0xNRCCG
1782 	/* Mix in mode-exclusion bits */  in scorpion_evt_setup()
1829 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_disable_event()
1830 	int idx = hwc->idx;  in scorpion_pmu_disable_event()
1831 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in scorpion_pmu_disable_event()
1832 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in scorpion_pmu_disable_event()
1835 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in scorpion_pmu_disable_event()
1843 	if (hwc->config_base & KRAIT_EVENT_MASK)  in scorpion_pmu_disable_event()
1844 		scorpion_clearpmu(hwc->config_base);  in scorpion_pmu_disable_event()
1849 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in scorpion_pmu_disable_event()
1855 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_enable_event()
1856 	int idx = hwc->idx;  in scorpion_pmu_enable_event()
1857 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in scorpion_pmu_enable_event()
1858 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);  in scorpion_pmu_enable_event()
1864 	raw_spin_lock_irqsave(&events->pmu_lock, flags);  in scorpion_pmu_enable_event()
1874 	if (hwc->config_base & KRAIT_EVENT_MASK)  in scorpion_pmu_enable_event()
1875 		scorpion_evt_setup(idx, hwc->config_base);  in scorpion_pmu_enable_event()
1877 		armv7_pmnc_write_evtsel(idx, hwc->config_base);  in scorpion_pmu_enable_event()
1885 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);  in scorpion_pmu_enable_event()
1892 	u32 idx, nb_cnt = cpu_pmu->num_events;  in scorpion_pmu_reset()
1917 	struct hw_perf_event *hwc = &event->hw;  in scorpion_event_to_bit()
1918 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in scorpion_event_to_bit()
1920 	if (hwc->config_base & VENUM_EVENT)  in scorpion_event_to_bit()
1924 	bit -= scorpion_get_pmresrn_event(0);  in scorpion_event_to_bit()
1943 	int bit = -1;  in scorpion_pmu_get_event_idx()
1944 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_get_event_idx()
1945 	unsigned int region = EVENT_REGION(hwc->config_base);  in scorpion_pmu_get_event_idx()
1946 	unsigned int group = EVENT_GROUP(hwc->config_base);  in scorpion_pmu_get_event_idx()
1947 	bool venum_event = EVENT_VENUM(hwc->config_base);  in scorpion_pmu_get_event_idx()
1948 	bool scorpion_event = EVENT_CPU(hwc->config_base);  in scorpion_pmu_get_event_idx()
1953 			return -EINVAL;  in scorpion_pmu_get_event_idx()
1956 		if (test_and_set_bit(bit, cpuc->used_mask))  in scorpion_pmu_get_event_idx()
1957 			return -EAGAIN;  in scorpion_pmu_get_event_idx()
1962 		clear_bit(bit, cpuc->used_mask);  in scorpion_pmu_get_event_idx()
1971 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_clear_event_idx()
1972 	unsigned int region = EVENT_REGION(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1973 	unsigned int group = EVENT_GROUP(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1974 	bool venum_event = EVENT_VENUM(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1975 	bool scorpion_event = EVENT_CPU(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1980 		clear_bit(bit, cpuc->used_mask);  in scorpion_pmu_clear_event_idx()
1987 	cpu_pmu->name		= "armv7_scorpion";  in scorpion_pmu_init()
1988 	cpu_pmu->map_event	= scorpion_map_event;  in scorpion_pmu_init()
1989 	cpu_pmu->reset		= scorpion_pmu_reset;  in scorpion_pmu_init()
1990 	cpu_pmu->enable		= scorpion_pmu_enable_event;  in scorpion_pmu_init()
1991 	cpu_pmu->disable	= scorpion_pmu_disable_event;  in scorpion_pmu_init()
1992 	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;  in scorpion_pmu_init()
1993 	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;  in scorpion_pmu_init()
2000 	cpu_pmu->name		= "armv7_scorpion_mp";  in scorpion_mp_pmu_init()
2001 	cpu_pmu->map_event	= scorpion_map_event;  in scorpion_mp_pmu_init()
2002 	cpu_pmu->reset		= scorpion_pmu_reset;  in scorpion_mp_pmu_init()
2003 	cpu_pmu->enable		= scorpion_pmu_enable_event;  in scorpion_mp_pmu_init()
2004 	cpu_pmu->disable	= scorpion_pmu_disable_event;  in scorpion_mp_pmu_init()
2005 	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;  in scorpion_mp_pmu_init()
2006 	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;  in scorpion_mp_pmu_init()
2011 	{.compatible = "arm,cortex-a17-pmu",	.data = armv7_a17_pmu_init},
2012 	{.compatible = "arm,cortex-a15-pmu",	.data = armv7_a15_pmu_init},
2013 	{.compatible = "arm,cortex-a12-pmu",	.data = armv7_a12_pmu_init},
2014 	{.compatible = "arm,cortex-a9-pmu",	.data = armv7_a9_pmu_init},
2015 	{.compatible = "arm,cortex-a8-pmu",	.data = armv7_a8_pmu_init},
2016 	{.compatible = "arm,cortex-a7-pmu",	.data = armv7_a7_pmu_init},
2017 	{.compatible = "arm,cortex-a5-pmu",	.data = armv7_a5_pmu_init},
2018 	{.compatible = "qcom,krait-pmu",	.data = krait_pmu_init},
2019 	{.compatible = "qcom,scorpion-pmu",	.data = scorpion_pmu_init},
2020 	{.compatible = "qcom,scorpion-mp-pmu",	.data = scorpion_mp_pmu_init},
2039 		.name	= "armv7-pmu",