270 	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);  in armv8pmu_event_attr_is_visible()  local
276 	    test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))  in armv8pmu_event_attr_is_visible()
283 		    test_bit(id, cpu_pmu->pmceid_ext_bitmap))  in armv8pmu_event_attr_is_visible()
328 	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);  in slots_show()  local
329 	u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;  in slots_show()
340 	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);  in bus_slots_show()  local
341 	u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT)  in bus_slots_show()
353 	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);  in bus_width_show()  local
354 	u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT)  in bus_width_show()
391 static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)  in armv8pmu_has_long_event()  argument
393 	return (cpu_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P5);  in armv8pmu_has_long_event()
409 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_event_is_chained()  local
413 	       !armv8pmu_has_long_event(cpu_pmu) &&  in armv8pmu_event_is_chained()
541 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_event_needs_bias()  local
548 	if (armv8pmu_has_long_event(cpu_pmu) ||  in armv8pmu_event_needs_bias()
749 static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu)  in armv8pmu_enable_user_access()  argument
752 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_enable_user_access()
755 	for_each_clear_bit(i, cpuc->used_mask, cpu_pmu->num_events) {  in armv8pmu_enable_user_access()
807 static void armv8pmu_start(struct arm_pmu *cpu_pmu)  in armv8pmu_start()  argument
810 		this_cpu_ptr(cpu_pmu->pmu.pmu_cpu_context)->task_ctx;  in armv8pmu_start()
813 		armv8pmu_enable_user_access(cpu_pmu);  in armv8pmu_start()
821 static void armv8pmu_stop(struct arm_pmu *cpu_pmu)  in armv8pmu_stop()  argument
827 static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)  in armv8pmu_handle_irq()  argument
831 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_handle_irq()
855 	armv8pmu_stop(cpu_pmu);  in armv8pmu_handle_irq()
856 	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {  in armv8pmu_handle_irq()
883 			cpu_pmu->disable(event);  in armv8pmu_handle_irq()
885 	armv8pmu_start(cpu_pmu);  in armv8pmu_handle_irq()
891 				    struct arm_pmu *cpu_pmu)  in armv8pmu_get_single_idx()  argument
895 	for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; idx++) {  in armv8pmu_get_single_idx()
903 				   struct arm_pmu *cpu_pmu)  in armv8pmu_get_chain_idx()  argument
911 	for (idx = ARMV8_IDX_COUNTER0 + 1; idx < cpu_pmu->num_events; idx += 2) {  in armv8pmu_get_chain_idx()
926 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_get_event_idx()  local
936 			   !armv8pmu_has_long_event(cpu_pmu))  in armv8pmu_get_event_idx()
944 		return	armv8pmu_get_chain_idx(cpuc, cpu_pmu);  in armv8pmu_get_event_idx()
946 		return armv8pmu_get_single_idx(cpuc, cpu_pmu);  in armv8pmu_get_event_idx()
1030 	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;  in armv8pmu_reset()  local
1047 	if (armv8pmu_has_long_event(cpu_pmu))  in armv8pmu_reset()
1140 	struct arm_pmu *cpu_pmu = probe->pmu;  in __armv8pmu_probe_pmu()  local
1152 	cpu_pmu->pmuver = pmuver;  in __armv8pmu_probe_pmu()
1156 	cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)  in __armv8pmu_probe_pmu()
1160 	cpu_pmu->num_events += 1;  in __armv8pmu_probe_pmu()
1165 	bitmap_from_arr32(cpu_pmu->pmceid_bitmap,  in __armv8pmu_probe_pmu()
1171 	bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,  in __armv8pmu_probe_pmu()
1176 		cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1);  in __armv8pmu_probe_pmu()
1178 		cpu_pmu->reg_pmmir = 0;  in __armv8pmu_probe_pmu()
1181 static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)  in armv8pmu_probe_pmu()  argument
1184 		.pmu = cpu_pmu,  in armv8pmu_probe_pmu()
1189 	ret = smp_call_function_any(&cpu_pmu->supported_cpus,  in armv8pmu_probe_pmu()
1235 static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,  in armv8_pmu_init()  argument
1241 	int ret = armv8pmu_probe_pmu(cpu_pmu);  in armv8_pmu_init()
1245 	cpu_pmu->handle_irq		= armv8pmu_handle_irq;  in armv8_pmu_init()
1246 	cpu_pmu->enable			= armv8pmu_enable_event;  in armv8_pmu_init()
1247 	cpu_pmu->disable		= armv8pmu_disable_event;  in armv8_pmu_init()
1248 	cpu_pmu->read_counter		= armv8pmu_read_counter;  in armv8_pmu_init()
1249 	cpu_pmu->write_counter		= armv8pmu_write_counter;  in armv8_pmu_init()
1250 	cpu_pmu->get_event_idx		= armv8pmu_get_event_idx;  in armv8_pmu_init()
1251 	cpu_pmu->clear_event_idx	= armv8pmu_clear_event_idx;  in armv8_pmu_init()
1252 	cpu_pmu->start			= armv8pmu_start;  in armv8_pmu_init()
1253 	cpu_pmu->stop			= armv8pmu_stop;  in armv8_pmu_init()
1254 	cpu_pmu->reset			= armv8pmu_reset;  in armv8_pmu_init()
1255 	cpu_pmu->set_event_filter	= armv8pmu_set_event_filter;  in armv8_pmu_init()
1256 	cpu_pmu->filter_match		= armv8pmu_filter_match;  in armv8_pmu_init()
1258 	cpu_pmu->pmu.event_idx		= armv8pmu_user_event_idx;  in armv8_pmu_init()
1260 	cpu_pmu->name			= name;  in armv8_pmu_init()
1261 	cpu_pmu->map_event		= map_event;  in armv8_pmu_init()
1262 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ?  in armv8_pmu_init()
1264 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?  in armv8_pmu_init()
1266 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ?  in armv8_pmu_init()
1273 static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name,  in armv8_pmu_init_nogroups()  argument
1276 	return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL);  in armv8_pmu_init_nogroups()
1280 static int name##_pmu_init(struct arm_pmu *cpu_pmu)			\
1282 	return armv8_pmu_init_nogroups(cpu_pmu, #name, armv8_pmuv3_map_event);\
1306 static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)  in PMUV3_INIT_SIMPLE()
1308 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a35",  in PMUV3_INIT_SIMPLE()
1312 static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a53_pmu_init()  argument
1314 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a53",  in armv8_a53_pmu_init()
1318 static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a57_pmu_init()  argument
1320 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57",  in armv8_a57_pmu_init()
1324 static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a72_pmu_init()  argument
1326 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72",  in armv8_a72_pmu_init()
1330 static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a73_pmu_init()  argument
1332 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a73",  in armv8_a73_pmu_init()
1336 static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_thunder_pmu_init()  argument
1338 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder",  in armv8_thunder_pmu_init()
1342 static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_vulcan_pmu_init()  argument
1344 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_brcm_vulcan",  in armv8_vulcan_pmu_init()