83 struct perf_ibs {  struct
180 static struct perf_ibs perf_ibs_fetch;
181 static struct perf_ibs perf_ibs_op;
183 static struct perf_ibs *get_ibs_pmu(int type)  in get_ibs_pmu()
274 	struct perf_ibs *perf_ibs;  in perf_ibs_init()  local
278 	perf_ibs = get_ibs_pmu(event->attr.type);  in perf_ibs_init()
279 	if (!perf_ibs)  in perf_ibs_init()
284 	if (event->pmu != &perf_ibs->pmu)  in perf_ibs_init()
287 	if (config & ~perf_ibs->config_mask)  in perf_ibs_init()
295 		if (config & perf_ibs->cnt_mask)  in perf_ibs_init()
309 		max_cnt = config & perf_ibs->cnt_mask;  in perf_ibs_init()
310 		config &= ~perf_ibs->cnt_mask;  in perf_ibs_init()
325 	hwc->config_base = perf_ibs->msr;  in perf_ibs_init()
331 static int perf_ibs_set_period(struct perf_ibs *perf_ibs,  in perf_ibs_set_period()  argument
337 	overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period);  in perf_ibs_set_period()
372 perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event,  in perf_ibs_event_update()  argument
375 	u64 count = perf_ibs->get_count(*config);  in perf_ibs_event_update()
384 		count = perf_ibs->get_count(*config);  in perf_ibs_event_update()
388 static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,  in perf_ibs_enable_event()  argument
393 	if (perf_ibs->fetch_count_reset_broken)  in perf_ibs_enable_event()
394 		wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask);  in perf_ibs_enable_event()
396 	wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask);  in perf_ibs_enable_event()
406 static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,  in perf_ibs_disable_event()  argument
409 	config &= ~perf_ibs->cnt_mask;  in perf_ibs_disable_event()
412 	config &= ~perf_ibs->enable_mask;  in perf_ibs_disable_event()
425 	struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);  in perf_ibs_start()  local
426 	struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);  in perf_ibs_start()
435 	perf_ibs_set_period(perf_ibs, hwc, &period);  in perf_ibs_start()
436 	if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) {  in perf_ibs_start()
448 	perf_ibs_enable_event(perf_ibs, hwc, config);  in perf_ibs_start()
456 	struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);  in perf_ibs_stop()  local
457 	struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);  in perf_ibs_stop()
479 		perf_ibs_disable_event(perf_ibs, hwc, config);  in perf_ibs_stop()
501 	config &= ~perf_ibs->valid_mask;  in perf_ibs_stop()
503 	perf_ibs_event_update(perf_ibs, event, &config);  in perf_ibs_stop()
509 	struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);  in perf_ibs_add()  local
510 	struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);  in perf_ibs_add()
527 	struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);  in perf_ibs_del()  local
528 	struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);  in perf_ibs_del()
657 static struct perf_ibs perf_ibs_fetch = {
681 static struct perf_ibs perf_ibs_op = {
1004 static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type,  in perf_ibs_get_offset_max()  argument
1008 	    (perf_ibs == &perf_ibs_op &&  in perf_ibs_get_offset_max()
1013 		return perf_ibs->offset_max;  in perf_ibs_get_offset_max()
1019 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)  in perf_ibs_handle_irq()  argument
1021 	struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);  in perf_ibs_handle_irq()
1053 	if (!(*buf++ & perf_ibs->valid_mask))  in perf_ibs_handle_irq()
1057 	perf_ibs_event_update(perf_ibs, event, config);  in perf_ibs_handle_irq()
1059 	if (!perf_ibs_set_period(perf_ibs, hwc, &period))  in perf_ibs_handle_irq()
1065 	check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));  in perf_ibs_handle_irq()
1067 	offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip);  in perf_ibs_handle_irq()
1072 		offset = find_next_bit(perf_ibs->offset_mask,  in perf_ibs_handle_irq()
1073 				       perf_ibs->offset_max,  in perf_ibs_handle_irq()
1082 		if (perf_ibs == &perf_ibs_op) {  in perf_ibs_handle_irq()
1092 		if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) {  in perf_ibs_handle_irq()
1104 		if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1]))  in perf_ibs_handle_irq()
1121 	if (perf_ibs == &perf_ibs_op)  in perf_ibs_handle_irq()
1137 		if (perf_ibs == &perf_ibs_op) {  in perf_ibs_handle_irq()
1147 		perf_ibs_enable_event(perf_ibs, hwc, new_config);  in perf_ibs_handle_irq()
1173 static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)  in perf_ibs_pmu_init()  argument
1182 	perf_ibs->pcpu = pcpu;  in perf_ibs_pmu_init()
1184 	ret = perf_pmu_register(&perf_ibs->pmu, name, -1);  in perf_ibs_pmu_init()
1186 		perf_ibs->pcpu = NULL;  in perf_ibs_pmu_init()