263 	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);  in armv8pmu_event_attr_is_visible()  local
269 	    test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))  in armv8pmu_event_attr_is_visible()
276 		    test_bit(id, cpu_pmu->pmceid_ext_bitmap))  in armv8pmu_event_attr_is_visible()
312 	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);  in slots_show()  local
313 	u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;  in slots_show()
342 static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)  in armv8pmu_has_long_event()  argument
344 	return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5);  in armv8pmu_has_long_event()
356 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_event_is_chained()  local
360 	       !armv8pmu_has_long_event(cpu_pmu) &&  in armv8pmu_event_is_chained()
489 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_event_needs_bias()  local
496 	if (armv8pmu_has_long_event(cpu_pmu) ||  in armv8pmu_event_needs_bias()
733 static void armv8pmu_start(struct arm_pmu *cpu_pmu)  in armv8pmu_start()  argument
739 static void armv8pmu_stop(struct arm_pmu *cpu_pmu)  in armv8pmu_stop()  argument
745 static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)  in armv8pmu_handle_irq()  argument
749 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_handle_irq()
773 	armv8pmu_stop(cpu_pmu);  in armv8pmu_handle_irq()
774 	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {  in armv8pmu_handle_irq()
801 			cpu_pmu->disable(event);  in armv8pmu_handle_irq()
803 	armv8pmu_start(cpu_pmu);  in armv8pmu_handle_irq()
809 				    struct arm_pmu *cpu_pmu)  in armv8pmu_get_single_idx()  argument
813 	for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; idx ++) {  in armv8pmu_get_single_idx()
821 				   struct arm_pmu *cpu_pmu)  in armv8pmu_get_chain_idx()  argument
829 	for (idx = ARMV8_IDX_COUNTER0 + 1; idx < cpu_pmu->num_events; idx += 2) {  in armv8pmu_get_chain_idx()
844 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_get_event_idx()  local
858 	    !armv8pmu_has_long_event(cpu_pmu))  in armv8pmu_get_event_idx()
859 		return	armv8pmu_get_chain_idx(cpuc, cpu_pmu);  in armv8pmu_get_event_idx()
861 		return armv8pmu_get_single_idx(cpuc, cpu_pmu);  in armv8pmu_get_event_idx()
929 	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;  in armv8pmu_reset()  local
946 	if (armv8pmu_has_long_event(cpu_pmu))  in armv8pmu_reset()
1020 	struct arm_pmu *cpu_pmu = probe->pmu;  in __armv8pmu_probe_pmu()  local
1032 	cpu_pmu->pmuver = pmuver;  in __armv8pmu_probe_pmu()
1036 	cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)  in __armv8pmu_probe_pmu()
1040 	cpu_pmu->num_events += 1;  in __armv8pmu_probe_pmu()
1045 	bitmap_from_arr32(cpu_pmu->pmceid_bitmap,  in __armv8pmu_probe_pmu()
1051 	bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,  in __armv8pmu_probe_pmu()
1056 		cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1);  in __armv8pmu_probe_pmu()
1058 		cpu_pmu->reg_pmmir = 0;  in __armv8pmu_probe_pmu()
1061 static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)  in armv8pmu_probe_pmu()  argument
1064 		.pmu = cpu_pmu,  in armv8pmu_probe_pmu()
1069 	ret = smp_call_function_any(&cpu_pmu->supported_cpus,  in armv8pmu_probe_pmu()
1078 static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,  in armv8_pmu_init()  argument
1084 	int ret = armv8pmu_probe_pmu(cpu_pmu);  in armv8_pmu_init()
1088 	cpu_pmu->handle_irq		= armv8pmu_handle_irq;  in armv8_pmu_init()
1089 	cpu_pmu->enable			= armv8pmu_enable_event;  in armv8_pmu_init()
1090 	cpu_pmu->disable		= armv8pmu_disable_event;  in armv8_pmu_init()
1091 	cpu_pmu->read_counter		= armv8pmu_read_counter;  in armv8_pmu_init()
1092 	cpu_pmu->write_counter		= armv8pmu_write_counter;  in armv8_pmu_init()
1093 	cpu_pmu->get_event_idx		= armv8pmu_get_event_idx;  in armv8_pmu_init()
1094 	cpu_pmu->clear_event_idx	= armv8pmu_clear_event_idx;  in armv8_pmu_init()
1095 	cpu_pmu->start			= armv8pmu_start;  in armv8_pmu_init()
1096 	cpu_pmu->stop			= armv8pmu_stop;  in armv8_pmu_init()
1097 	cpu_pmu->reset			= armv8pmu_reset;  in armv8_pmu_init()
1098 	cpu_pmu->set_event_filter	= armv8pmu_set_event_filter;  in armv8_pmu_init()
1099 	cpu_pmu->filter_match		= armv8pmu_filter_match;  in armv8_pmu_init()
1101 	cpu_pmu->name			= name;  in armv8_pmu_init()
1102 	cpu_pmu->map_event		= map_event;  in armv8_pmu_init()
1103 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ?  in armv8_pmu_init()
1105 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?  in armv8_pmu_init()
1107 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ?  in armv8_pmu_init()
1113 static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name,  in armv8_pmu_init_nogroups()  argument
1116 	return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL);  in armv8_pmu_init_nogroups()
1119 static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)  in armv8_pmuv3_init()  argument
1121 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_pmuv3",  in armv8_pmuv3_init()
1125 static int armv8_a34_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a34_pmu_init()  argument
1127 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a34",  in armv8_a34_pmu_init()
1131 static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a35_pmu_init()  argument
1133 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a35",  in armv8_a35_pmu_init()
1137 static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a53_pmu_init()  argument
1139 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a53",  in armv8_a53_pmu_init()
1143 static int armv8_a55_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a55_pmu_init()  argument
1145 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a55",  in armv8_a55_pmu_init()
1149 static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a57_pmu_init()  argument
1151 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57",  in armv8_a57_pmu_init()
1155 static int armv8_a65_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a65_pmu_init()  argument
1157 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a65",  in armv8_a65_pmu_init()
1161 static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a72_pmu_init()  argument
1163 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72",  in armv8_a72_pmu_init()
1167 static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a73_pmu_init()  argument
1169 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a73",  in armv8_a73_pmu_init()
1173 static int armv8_a75_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a75_pmu_init()  argument
1175 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a75",  in armv8_a75_pmu_init()
1179 static int armv8_a76_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a76_pmu_init()  argument
1181 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a76",  in armv8_a76_pmu_init()
1185 static int armv8_a77_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_a77_pmu_init()  argument
1187 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a77",  in armv8_a77_pmu_init()
1191 static int armv8_e1_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_e1_pmu_init()  argument
1193 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_neoverse_e1",  in armv8_e1_pmu_init()
1197 static int armv8_n1_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_n1_pmu_init()  argument
1199 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_neoverse_n1",  in armv8_n1_pmu_init()
1203 static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_thunder_pmu_init()  argument
1205 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder",  in armv8_thunder_pmu_init()
1209 static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)  in armv8_vulcan_pmu_init()  argument
1211 	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_brcm_vulcan",  in armv8_vulcan_pmu_init()