1 // SPDX-License-Identifier: GPL-2.0
65 #define LEVEL(x) P(LVLNUM, x)  macro
71 	P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA),/* 0x00:ukn L3 */
72 	OP_LH | P(LVL, L1)  | LEVEL(L1) | P(SNOOP, NONE),  /* 0x01: L1 local */
73 	OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE), /* 0x02: LFB hit */
74 	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, NONE),  /* 0x03: L2 hit */
75 	OP_LH | P(LVL, L3)  | LEVEL(L3) | P(SNOOP, NONE),  /* 0x04: L3 hit */
76 	OP_LH | P(LVL, L3)  | LEVEL(L3) | P(SNOOP, MISS),  /* 0x05: L3 hit, snoop miss */
77 	OP_LH | P(LVL, L3)  | LEVEL(L3) | P(SNOOP, HIT),   /* 0x06: L3 hit, snoop hit */
78 	OP_LH | P(LVL, L3)  | LEVEL(L3) | P(SNOOP, HITM),  /* 0x07: L3 hit, snoop hitm */
79 	OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HIT),  /* 0x08: L3 miss snoop hit */
80 	OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HITM), /* 0x09: L3 miss snoop hitm*/
81 	OP_LH | P(LVL, LOC_RAM)  | LEVEL(RAM) | P(SNOOP, HIT),       /* 0x0a: L3 miss, shared */
82 	OP_LH | P(LVL, REM_RAM1) | REM | LEVEL(L3) | P(SNOOP, HIT),  /* 0x0b: L3 miss, shared */
83 	OP_LH | P(LVL, LOC_RAM)  | LEVEL(RAM) | SNOOP_NONE_MISS,     /* 0x0c: L3 miss, excl */
84 	OP_LH | P(LVL, REM_RAM1) | LEVEL(RAM) | REM | SNOOP_NONE_MISS, /* 0x0d: L3 miss, excl */
85 	OP_LH | P(LVL, IO)  | LEVEL(NA) | P(SNOOP, NONE), /* 0x0e: I/O */
86 	OP_LH | P(LVL, UNC) | LEVEL(NA) | P(SNOOP, NONE), /* 0x0f: uncached */
92 	pebs_data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT);  in intel_pmu_pebs_data_source_nhm()
93 	pebs_data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);  in intel_pmu_pebs_data_source_nhm()
94 	pebs_data_source[0x07] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);  in intel_pmu_pebs_data_source_nhm()
99 	u64 pmem_or_l4 = pmem ? LEVEL(PMEM) : LEVEL(L4);  in intel_pmu_pebs_data_source_skl()
103 	pebs_data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE);  in intel_pmu_pebs_data_source_skl()
104 	pebs_data_source[0x0c] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOPX, FWD);  in intel_pmu_pebs_data_source_skl()
105 	pebs_data_source[0x0d] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOP, HITM);  in intel_pmu_pebs_data_source_skl()
117 	 * 1 = stored missed 2nd level TLB  in precise_store_data()
120 	 * otherwise hit 2nd level TLB  in precise_store_data()
152 	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)  in precise_datala_hsw()
154 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)  in precise_datala_hsw()
165 	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {  in precise_datala_hsw()
182 	 * use the mapping table for bit 0-3  in load_latency_data()
195 	 * 0 = did not miss 2nd level TLB  in load_latency_data()
196 	 * 1 = missed 2nd level TLB  in load_latency_data()
244 	 * use the mapping table for bit 0-3  in store_latency_data()
250 	 * 0 = did not miss 2nd level TLB  in store_latency_data()
251 	 * 1 = missed 2nd level TLB  in store_latency_data()
364 	 * This is a cross-CPU update of the cpu_entry_area, we must shoot down  in ds_update_cea()
403 	struct debug_store *ds = hwev->ds;  in alloc_pebs_buffer()
413 		return -ENOMEM;  in alloc_pebs_buffer()
423 			return -ENOMEM;  in alloc_pebs_buffer()
427 	hwev->ds_pebs_vaddr = buffer;  in alloc_pebs_buffer()
429 	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;  in alloc_pebs_buffer()
430 	ds->pebs_buffer_base = (unsigned long) cea;  in alloc_pebs_buffer()
432 	ds->pebs_index = ds->pebs_buffer_base;  in alloc_pebs_buffer()
434 	ds->pebs_absolute_maximum = ds->pebs_buffer_base + max;  in alloc_pebs_buffer()
450 	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;  in release_pebs_buffer()
452 	dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size);  in release_pebs_buffer()
453 	hwev->ds_pebs_vaddr = NULL;  in release_pebs_buffer()
459 	struct debug_store *ds = hwev->ds;  in alloc_bts_buffer()
469 		return -ENOMEM;  in alloc_bts_buffer()
471 	hwev->ds_bts_vaddr = buffer;  in alloc_bts_buffer()
473 	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;  in alloc_bts_buffer()
474 	ds->bts_buffer_base = (unsigned long) cea;  in alloc_bts_buffer()
476 	ds->bts_index = ds->bts_buffer_base;  in alloc_bts_buffer()
478 	ds->bts_absolute_maximum = ds->bts_buffer_base +  in alloc_bts_buffer()
480 	ds->bts_interrupt_threshold = ds->bts_absolute_maximum -  in alloc_bts_buffer()
494 	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;  in release_bts_buffer()
496 	dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE);  in release_bts_buffer()
497 	hwev->ds_bts_vaddr = NULL;  in release_bts_buffer()
502 	struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store;  in alloc_ds_buffer()
634 	if (!cpuc->ds)  in intel_pmu_disable_bts()
649 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_bts_buffer()
655 	struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];  in intel_pmu_drain_bts_buffer()
656 	struct bts_record *at, *base, *top;  in intel_pmu_drain_bts_buffer()  local
669 	base = (struct bts_record *)(unsigned long)ds->bts_buffer_base;  in intel_pmu_drain_bts_buffer()
670 	top  = (struct bts_record *)(unsigned long)ds->bts_index;  in intel_pmu_drain_bts_buffer()
672 	if (top <= base)  in intel_pmu_drain_bts_buffer()
677 	ds->bts_index = ds->bts_buffer_base;  in intel_pmu_drain_bts_buffer()
679 	perf_sample_data_init(&data, 0, event->hw.last_period);  in intel_pmu_drain_bts_buffer()
691 	for (at = base; at < top; at++) {  in intel_pmu_drain_bts_buffer()
697 		if (event->attr.exclude_kernel &&  in intel_pmu_drain_bts_buffer()
698 		    (kernel_ip(at->from) || kernel_ip(at->to)))  in intel_pmu_drain_bts_buffer()
711 			      header.size * (top - base - skip)))  in intel_pmu_drain_bts_buffer()
714 	for (at = base; at < top; at++) {  in intel_pmu_drain_bts_buffer()
716 		if (event->attr.exclude_kernel &&  in intel_pmu_drain_bts_buffer()
717 		    (kernel_ip(at->from) || kernel_ip(at->to)))  in intel_pmu_drain_bts_buffer()
720 		data.ip		= at->from;  in intel_pmu_drain_bts_buffer()
721 		data.addr	= at->to;  in intel_pmu_drain_bts_buffer()
729 	event->hw.interrupts++;  in intel_pmu_drain_bts_buffer()
730 	event->pending_kill = POLL_IN;  in intel_pmu_drain_bts_buffer()
969 	struct event_constraint *pebs_constraints = hybrid(event->pmu, pebs_constraints);  in intel_pebs_constraints()
972 	if (!event->attr.precise_ip)  in intel_pebs_constraints()
977 			if (constraint_match(c, event->hw.config)) {  in intel_pebs_constraints()
978 				event->hw.flags |= c->flags;  in intel_pebs_constraints()
995  * We need the sched_task callback even for per-cpu events when we use
1001 	if (cpuc->n_pebs == cpuc->n_pebs_via_pt)  in pebs_needs_sched_cb()
1004 	return cpuc->n_pebs && (cpuc->n_pebs == cpuc->n_large_pebs);  in pebs_needs_sched_cb()
1017 	struct debug_store *ds = cpuc->ds;  in pebs_update_threshold()
1018 	int max_pebs_events = hybrid(cpuc->pmu, max_pebs_events);  in pebs_update_threshold()
1019 	int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);  in pebs_update_threshold()
1023 	if (cpuc->n_pebs_via_pt)  in pebs_update_threshold()
1031 	if (cpuc->n_pebs == cpuc->n_large_pebs) {  in pebs_update_threshold()
1032 		threshold = ds->pebs_absolute_maximum -  in pebs_update_threshold()
1033 			reserved * cpuc->pebs_record_size;  in pebs_update_threshold()
1035 		threshold = ds->pebs_buffer_base + cpuc->pebs_record_size;  in pebs_update_threshold()
1038 	ds->pebs_interrupt_threshold = threshold;  in pebs_update_threshold()
1044 	u64 pebs_data_cfg = cpuc->pebs_data_cfg;  in adaptive_pebs_record_size_update()
1056 	cpuc->pebs_record_size = sz;  in adaptive_pebs_record_size_update()
1067 	struct perf_event_attr *attr = &event->attr;  in pebs_update_adaptive_cfg()
1068 	u64 sample_type = attr->sample_type;  in pebs_update_adaptive_cfg()
1073 	    attr->precise_ip > 1)  in pebs_update_adaptive_cfg()
1086 	       (attr->sample_regs_intr & PEBS_GP_REGS);  in pebs_update_adaptive_cfg()
1089 		     ((attr->config & INTEL_ARCH_EVENT_MASK) ==  in pebs_update_adaptive_cfg()
1092 	if (gprs || (attr->precise_ip < 2) || tsx_weight)  in pebs_update_adaptive_cfg()
1096 	    (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK))  in pebs_update_adaptive_cfg()
1105 			((x86_pmu.lbr_nr-1) << PEBS_DATACFG_LBR_SHIFT);  in pebs_update_adaptive_cfg()
1115 	struct pmu *pmu = event->ctx->pmu;  in pebs_update_state()
1121 	bool update = cpuc->n_pebs == 1;  in pebs_update_state()
1140 		if (cpuc->n_pebs == 1) {  in pebs_update_state()
1141 			cpuc->pebs_data_cfg = 0;  in pebs_update_state()
1142 			cpuc->pebs_record_size = sizeof(struct pebs_basic);  in pebs_update_state()
1148 		if (pebs_data_cfg & ~cpuc->pebs_data_cfg) {  in pebs_update_state()
1149 			cpuc->pebs_data_cfg |= pebs_data_cfg;  in pebs_update_state()
1162 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_add()
1165 	cpuc->n_pebs++;  in intel_pmu_pebs_add()
1166 	if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)  in intel_pmu_pebs_add()
1167 		cpuc->n_large_pebs++;  in intel_pmu_pebs_add()
1168 	if (hwc->flags & PERF_X86_EVENT_PEBS_VIA_PT)  in intel_pmu_pebs_add()
1169 		cpuc->n_pebs_via_pt++;  in intel_pmu_pebs_add()
1181 	if (!(cpuc->pebs_enabled & ~PEBS_VIA_PT_MASK))  in intel_pmu_pebs_via_pt_disable()
1182 		cpuc->pebs_enabled &= ~PEBS_VIA_PT_MASK;  in intel_pmu_pebs_via_pt_disable()
1188 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_via_pt_enable()
1189 	struct debug_store *ds = cpuc->ds;  in intel_pmu_pebs_via_pt_enable()
1190 	u64 value = ds->pebs_event_reset[hwc->idx];  in intel_pmu_pebs_via_pt_enable()
1192 	unsigned int idx = hwc->idx;  in intel_pmu_pebs_via_pt_enable()
1197 	if (!(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))  in intel_pmu_pebs_via_pt_enable()
1198 		cpuc->pebs_enabled |= PEBS_PMI_AFTER_EACH_RECORD;  in intel_pmu_pebs_via_pt_enable()
1200 	cpuc->pebs_enabled |= PEBS_OUTPUT_PT;  in intel_pmu_pebs_via_pt_enable()
1202 	if (hwc->idx >= INTEL_PMC_IDX_FIXED) {  in intel_pmu_pebs_via_pt_enable()
1204 		idx = hwc->idx - INTEL_PMC_IDX_FIXED;  in intel_pmu_pebs_via_pt_enable()
1205 		value = ds->pebs_event_reset[MAX_PEBS_EVENTS + idx];  in intel_pmu_pebs_via_pt_enable()
1213 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_enable()
1214 	struct debug_store *ds = cpuc->ds;  in intel_pmu_pebs_enable()
1215 	unsigned int idx = hwc->idx;  in intel_pmu_pebs_enable()
1217 	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;  in intel_pmu_pebs_enable()
1219 	cpuc->pebs_enabled |= 1ULL << hwc->idx;  in intel_pmu_pebs_enable()
1221 	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))  in intel_pmu_pebs_enable()
1222 		cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);  in intel_pmu_pebs_enable()
1223 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)  in intel_pmu_pebs_enable()
1224 		cpuc->pebs_enabled |= 1ULL << 63;  in intel_pmu_pebs_enable()
1227 		hwc->config |= ICL_EVENTSEL_ADAPTIVE;  in intel_pmu_pebs_enable()
1228 		if (cpuc->pebs_data_cfg != cpuc->active_pebs_data_cfg) {  in intel_pmu_pebs_enable()
1229 			wrmsrl(MSR_PEBS_DATA_CFG, cpuc->pebs_data_cfg);  in intel_pmu_pebs_enable()
1230 			cpuc->active_pebs_data_cfg = cpuc->pebs_data_cfg;  in intel_pmu_pebs_enable()
1235 		idx = MAX_PEBS_EVENTS + (idx - INTEL_PMC_IDX_FIXED);  in intel_pmu_pebs_enable()
1238 	 * Use auto-reload if possible to save a MSR write in the PMI.  in intel_pmu_pebs_enable()
1241 	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {  in intel_pmu_pebs_enable()
1242 		ds->pebs_event_reset[idx] =  in intel_pmu_pebs_enable()
1243 			(u64)(-hwc->sample_period) & x86_pmu.cntval_mask;  in intel_pmu_pebs_enable()
1245 		ds->pebs_event_reset[idx] = 0;  in intel_pmu_pebs_enable()
1254 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_del()
1257 	cpuc->n_pebs--;  in intel_pmu_pebs_del()
1258 	if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)  in intel_pmu_pebs_del()
1259 		cpuc->n_large_pebs--;  in intel_pmu_pebs_del()
1260 	if (hwc->flags & PERF_X86_EVENT_PEBS_VIA_PT)  in intel_pmu_pebs_del()
1261 		cpuc->n_pebs_via_pt--;  in intel_pmu_pebs_del()
1269 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_disable()
1271 	if (cpuc->n_pebs == cpuc->n_large_pebs &&  in intel_pmu_pebs_disable()
1272 	    cpuc->n_pebs != cpuc->n_pebs_via_pt)  in intel_pmu_pebs_disable()
1275 	cpuc->pebs_enabled &= ~(1ULL << hwc->idx);  in intel_pmu_pebs_disable()
1277 	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&  in intel_pmu_pebs_disable()
1279 		cpuc->pebs_enabled &= ~(1ULL << (hwc->idx + 32));  in intel_pmu_pebs_disable()
1280 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)  in intel_pmu_pebs_disable()
1281 		cpuc->pebs_enabled &= ~(1ULL << 63);  in intel_pmu_pebs_disable()
1285 	if (cpuc->enabled)  in intel_pmu_pebs_disable()
1286 		wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);  in intel_pmu_pebs_disable()
1288 	hwc->config |= ARCH_PERFMON_EVENTSEL_INT;  in intel_pmu_pebs_disable()
1295 	if (cpuc->pebs_enabled)  in intel_pmu_pebs_enable_all()
1296 		wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);  in intel_pmu_pebs_enable_all()
1303 	if (cpuc->pebs_enabled)  in intel_pmu_pebs_disable_all()
1310 	unsigned long from = cpuc->lbr_entries[0].from;  in intel_pmu_pebs_fixup_ip()
1311 	unsigned long old_to, to = cpuc->lbr_entries[0].to;  in intel_pmu_pebs_fixup_ip()
1312 	unsigned long ip = regs->ip;  in intel_pmu_pebs_fixup_ip()
1326 	if (!cpuc->lbr_stack.nr || !from || !to)  in intel_pmu_pebs_fixup_ip()
1339 	if ((ip - to) > PEBS_FIXUP_SIZE)  in intel_pmu_pebs_fixup_ip()
1350 	size = ip - to;  in intel_pmu_pebs_fixup_ip()
1385 		size -= insn.length;  in intel_pmu_pebs_fixup_ip()
1422 		return ((struct pebs_record_nhm *)n)->status;  in get_pebs_status()
1423 	return ((struct pebs_basic *)n)->applicable_counters;  in get_pebs_status()
1434 	int fl = event->hw.flags;  in get_data_src()
1469 	sample_type = event->attr.sample_type;  in setup_pebs_fixed_sample_data()
1470 	fll = event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT;  in setup_pebs_fixed_sample_data()
1472 	perf_sample_data_init(data, 0, event->hw.last_period);  in setup_pebs_fixed_sample_data()
1474 	data->period = event->hw.last_period;  in setup_pebs_fixed_sample_data()
1477 	 * Use latency for weight (only avail with PEBS-LL)  in setup_pebs_fixed_sample_data()
1480 		data->weight.full = pebs->lat;  in setup_pebs_fixed_sample_data()
1486 		data->data_src.val = get_data_src(event, pebs->dse);  in setup_pebs_fixed_sample_data()
1495 		data->callchain = perf_callchain(event, iregs);  in setup_pebs_fixed_sample_data()
1511 	regs->flags = pebs->flags & ~PERF_EFLAGS_EXACT;  in setup_pebs_fixed_sample_data()
1514 		regs->ax = pebs->ax;  in setup_pebs_fixed_sample_data()
1515 		regs->bx = pebs->bx;  in setup_pebs_fixed_sample_data()
1516 		regs->cx = pebs->cx;  in setup_pebs_fixed_sample_data()
1517 		regs->dx = pebs->dx;  in setup_pebs_fixed_sample_data()
1518 		regs->si = pebs->si;  in setup_pebs_fixed_sample_data()
1519 		regs->di = pebs->di;  in setup_pebs_fixed_sample_data()
1521 		regs->bp = pebs->bp;  in setup_pebs_fixed_sample_data()
1522 		regs->sp = pebs->sp;  in setup_pebs_fixed_sample_data()
1525 		regs->r8 = pebs->r8;  in setup_pebs_fixed_sample_data()
1526 		regs->r9 = pebs->r9;  in setup_pebs_fixed_sample_data()
1527 		regs->r10 = pebs->r10;  in setup_pebs_fixed_sample_data()
1528 		regs->r11 = pebs->r11;  in setup_pebs_fixed_sample_data()
1529 		regs->r12 = pebs->r12;  in setup_pebs_fixed_sample_data()
1530 		regs->r13 = pebs->r13;  in setup_pebs_fixed_sample_data()
1531 		regs->r14 = pebs->r14;  in setup_pebs_fixed_sample_data()
1532 		regs->r15 = pebs->r15;  in setup_pebs_fixed_sample_data()
1536 	if (event->attr.precise_ip > 1) {  in setup_pebs_fixed_sample_data()
1539 		 * (real IP) which fixes the off-by-1 skid in hardware.  in setup_pebs_fixed_sample_data()
1543 			set_linear_ip(regs, pebs->real_ip);  in setup_pebs_fixed_sample_data()
1544 			regs->flags |= PERF_EFLAGS_EXACT;  in setup_pebs_fixed_sample_data()
1546 			/* Otherwise, use PEBS off-by-1 IP: */  in setup_pebs_fixed_sample_data()
1547 			set_linear_ip(regs, pebs->ip);  in setup_pebs_fixed_sample_data()
1550 			 * With precise_ip >= 2, try to fix up the off-by-1 IP  in setup_pebs_fixed_sample_data()
1552 			 * corrects regs->ip and calls set_linear_ip() on regs:  in setup_pebs_fixed_sample_data()
1555 				regs->flags |= PERF_EFLAGS_EXACT;  in setup_pebs_fixed_sample_data()
1559 		 * When precise_ip == 1, return the PEBS off-by-1 IP,  in setup_pebs_fixed_sample_data()
1562 		set_linear_ip(regs, pebs->ip);  in setup_pebs_fixed_sample_data()
1568 		data->addr = pebs->dla;  in setup_pebs_fixed_sample_data()
1573 			data->weight.full = intel_get_tsx_weight(pebs->tsx_tuning);  in setup_pebs_fixed_sample_data()
1576 			data->txn = intel_get_tsx_transaction(pebs->tsx_tuning,  in setup_pebs_fixed_sample_data()
1577 							      pebs->ax);  in setup_pebs_fixed_sample_data()
1587 		event->attr.use_clockid == 0)  in setup_pebs_fixed_sample_data()
1588 		data->time = native_sched_clock_from_tsc(pebs->tsc);  in setup_pebs_fixed_sample_data()
1591 		data->br_stack = &cpuc->lbr_stack;  in setup_pebs_fixed_sample_data()
1597 	regs->ax = gprs->ax;  in adaptive_pebs_save_regs()
1598 	regs->bx = gprs->bx;  in adaptive_pebs_save_regs()
1599 	regs->cx = gprs->cx;  in adaptive_pebs_save_regs()
1600 	regs->dx = gprs->dx;  in adaptive_pebs_save_regs()
1601 	regs->si = gprs->si;  in adaptive_pebs_save_regs()
1602 	regs->di = gprs->di;  in adaptive_pebs_save_regs()
1603 	regs->bp = gprs->bp;  in adaptive_pebs_save_regs()
1604 	regs->sp = gprs->sp;  in adaptive_pebs_save_regs()
1606 	regs->r8 = gprs->r8;  in adaptive_pebs_save_regs()
1607 	regs->r9 = gprs->r9;  in adaptive_pebs_save_regs()
1608 	regs->r10 = gprs->r10;  in adaptive_pebs_save_regs()
1609 	regs->r11 = gprs->r11;  in adaptive_pebs_save_regs()
1610 	regs->r12 = gprs->r12;  in adaptive_pebs_save_regs()
1611 	regs->r13 = gprs->r13;  in adaptive_pebs_save_regs()
1612 	regs->r14 = gprs->r14;  in adaptive_pebs_save_regs()
1613 	regs->r15 = gprs->r15;  in adaptive_pebs_save_regs()
1642 	perf_regs->xmm_regs = NULL;  in setup_pebs_adaptive_sample_data()
1644 	sample_type = event->attr.sample_type;  in setup_pebs_adaptive_sample_data()
1645 	format_size = basic->format_size;  in setup_pebs_adaptive_sample_data()
1646 	perf_sample_data_init(data, 0, event->hw.last_period);  in setup_pebs_adaptive_sample_data()
1647 	data->period = event->hw.last_period;  in setup_pebs_adaptive_sample_data()
1649 	if (event->attr.use_clockid == 0)  in setup_pebs_adaptive_sample_data()
1650 		data->time = native_sched_clock_from_tsc(basic->tsc);  in setup_pebs_adaptive_sample_data()
1659 		data->callchain = perf_callchain(event, iregs);  in setup_pebs_adaptive_sample_data()
1663 	set_linear_ip(regs, basic->ip);  in setup_pebs_adaptive_sample_data()
1664 	regs->flags = PERF_EFLAGS_EXACT;  in setup_pebs_adaptive_sample_data()
1668 	 * But PERF_SAMPLE_TRANSACTION needs gprs->ax.  in setup_pebs_adaptive_sample_data()
1680 		if (event->attr.precise_ip < 2) {  in setup_pebs_adaptive_sample_data()
1681 			set_linear_ip(regs, gprs->ip);  in setup_pebs_adaptive_sample_data()
1682 			regs->flags &= ~PERF_EFLAGS_EXACT;  in setup_pebs_adaptive_sample_data()
1691 			u64 weight = meminfo->latency;  in setup_pebs_adaptive_sample_data()
1694 				data->weight.var2_w = weight & PEBS_LATENCY_MASK;  in setup_pebs_adaptive_sample_data()
1704 				data->weight.full = weight ?:  in setup_pebs_adaptive_sample_data()
1705 					intel_get_tsx_weight(meminfo->tsx_tuning);  in setup_pebs_adaptive_sample_data()
1707 				data->weight.var1_dw = (u32)(weight & PEBS_LATENCY_MASK) ?:  in setup_pebs_adaptive_sample_data()
1708 					intel_get_tsx_weight(meminfo->tsx_tuning);  in setup_pebs_adaptive_sample_data()
1713 			data->data_src.val = get_data_src(event, meminfo->aux);  in setup_pebs_adaptive_sample_data()
1716 			data->addr = meminfo->address;  in setup_pebs_adaptive_sample_data()
1719 			data->txn = intel_get_tsx_transaction(meminfo->tsx_tuning,  in setup_pebs_adaptive_sample_data()
1720 							  gprs ? gprs->ax : 0);  in setup_pebs_adaptive_sample_data()
1727 		perf_regs->xmm_regs = xmm->xmm;  in setup_pebs_adaptive_sample_data()
1738 			data->br_stack = &cpuc->lbr_stack;  in setup_pebs_adaptive_sample_data()
1745 			(u64)(next_record - __pebs),  in setup_pebs_adaptive_sample_data()
1746 			basic->format_size);  in setup_pebs_adaptive_sample_data()
1750 get_next_pebs_record_by_bit(void *base, void *top, int bit)  in get_next_pebs_record_by_bit()  argument
1766 	for (at = base; at < top; at += cpuc->pebs_record_size) {  in get_next_pebs_record_by_bit()
1777 			/* clear non-PEBS bit and re-check */  in get_next_pebs_record_by_bit()
1778 			pebs_status = status & cpuc->pebs_enabled;  in get_next_pebs_record_by_bit()
1789 	WARN_ON(!(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD));  in intel_pmu_auto_reload_read()
1791 	perf_pmu_disable(event->pmu);  in intel_pmu_auto_reload_read()
1793 	perf_pmu_enable(event->pmu);  in intel_pmu_auto_reload_read()
1797  * Special variant of intel_pmu_save_and_restart() for auto-reload.
1802 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_save_and_restart_reload()
1803 	int shift = 64 - x86_pmu.cntval_bits;  in intel_pmu_save_and_restart_reload()
1804 	u64 period = hwc->sample_period;  in intel_pmu_save_and_restart_reload()
1815 	prev_raw_count = local64_read(&hwc->prev_count);  in intel_pmu_save_and_restart_reload()
1816 	rdpmcl(hwc->event_base_rdpmc, new_raw_count);  in intel_pmu_save_and_restart_reload()
1817 	local64_set(&hwc->prev_count, new_raw_count);  in intel_pmu_save_and_restart_reload()
1823 	 *   [-period, 0]  in intel_pmu_save_and_restart_reload()
1827 	 *   A) value2 - value1;  in intel_pmu_save_and_restart_reload()
1830 	 *   B) (0 - value1) + (value2 - (-period));  in intel_pmu_save_and_restart_reload()
1833 	 *   C) (0 - value1) + (n - 1) * (period) + (value2 - (-period));  in intel_pmu_save_and_restart_reload()
1837 	 * discrete interval, where the first term is to the top of the  in intel_pmu_save_and_restart_reload()
1844 	 *   value2 - value1 + n * period  in intel_pmu_save_and_restart_reload()
1848 	local64_add(new - old + count * period, &event->count);  in intel_pmu_save_and_restart_reload()
1850 	local64_set(&hwc->period_left, -new);  in intel_pmu_save_and_restart_reload()
1861 		       void *base, void *top,  in __intel_pmu_pebs_event()  argument
1870 	struct hw_perf_event *hwc = &event->hw;  in __intel_pmu_pebs_event()
1873 	void *at = get_next_pebs_record_by_bit(base, top, bit);  in __intel_pmu_pebs_event()
1876 	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {  in __intel_pmu_pebs_event()
1878 		 * Now, auto-reload is only enabled in fixed period mode.  in __intel_pmu_pebs_event()
1879 		 * The reload value is always hwc->sample_period.  in __intel_pmu_pebs_event()
1880 		 * May need to change it, if auto-reload is enabled in  in __intel_pmu_pebs_event()
1893 		at += cpuc->pebs_record_size;  in __intel_pmu_pebs_event()
1894 		at = get_next_pebs_record_by_bit(at, top, bit);  in __intel_pmu_pebs_event()
1895 		count--;  in __intel_pmu_pebs_event()
1901 		 * The PEBS records may be drained in the non-overflow context,  in __intel_pmu_pebs_event()
1920 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_pebs_core()
1921 	struct perf_event *event = cpuc->events[0]; /* PMC0 only */  in intel_pmu_drain_pebs_core()
1922 	struct pebs_record_core *at, *top;  in intel_pmu_drain_pebs_core()  local
1928 	at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;  in intel_pmu_drain_pebs_core()
1929 	top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;  in intel_pmu_drain_pebs_core()
1934 	ds->pebs_index = ds->pebs_buffer_base;  in intel_pmu_drain_pebs_core()
1936 	if (!test_bit(0, cpuc->active_mask))  in intel_pmu_drain_pebs_core()
1941 	if (!event->attr.precise_ip)  in intel_pmu_drain_pebs_core()
1944 	n = top - at;  in intel_pmu_drain_pebs_core()
1946 		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)  in intel_pmu_drain_pebs_core()
1951 	__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,  in intel_pmu_drain_pebs_core()
1962 	 * for auto-reload event in pmu::read(). There are no  in intel_pmu_pebs_event_update_no_drain()
1965 	 * update the event->count for this case.  in intel_pmu_pebs_event_update_no_drain()
1967 	for_each_set_bit(bit, (unsigned long *)&cpuc->pebs_enabled, size) {  in intel_pmu_pebs_event_update_no_drain()
1968 		event = cpuc->events[bit];  in intel_pmu_pebs_event_update_no_drain()
1969 		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)  in intel_pmu_pebs_event_update_no_drain()
1977 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_pebs_nhm()
1979 	void *base, *at, *top;  in intel_pmu_drain_pebs_nhm()  local
1988 	base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;  in intel_pmu_drain_pebs_nhm()
1989 	top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;  in intel_pmu_drain_pebs_nhm()
1991 	ds->pebs_index = ds->pebs_buffer_base;  in intel_pmu_drain_pebs_nhm()
1993 	mask = (1ULL << x86_pmu.max_pebs_events) - 1;  in intel_pmu_drain_pebs_nhm()
1996 		mask |= ((1ULL << x86_pmu.num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED;  in intel_pmu_drain_pebs_nhm()
2000 	if (unlikely(base >= top)) {  in intel_pmu_drain_pebs_nhm()
2005 	for (at = base; at < top; at += x86_pmu.pebs_record_size) {  in intel_pmu_drain_pebs_nhm()
2009 		pebs_status = p->status & cpuc->pebs_enabled;  in intel_pmu_drain_pebs_nhm()
2028 		if (!pebs_status && cpuc->pebs_enabled &&  in intel_pmu_drain_pebs_nhm()
2029 			!(cpuc->pebs_enabled & (cpuc->pebs_enabled-1)))  in intel_pmu_drain_pebs_nhm()
2030 			pebs_status = p->status = cpuc->pebs_enabled;  in intel_pmu_drain_pebs_nhm()
2042 		 * If these events include one PEBS and multiple non-PEBS  in intel_pmu_drain_pebs_nhm()
2065 		event = cpuc->events[bit];  in intel_pmu_drain_pebs_nhm()
2069 		if (WARN_ON_ONCE(!event->attr.precise_ip))  in intel_pmu_drain_pebs_nhm()
2082 					       top, bit, counts[bit],  in intel_pmu_drain_pebs_nhm()
2092 	int max_pebs_events = hybrid(cpuc->pmu, max_pebs_events);  in intel_pmu_drain_pebs_icl()
2093 	int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);  in intel_pmu_drain_pebs_icl()
2094 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_pebs_icl()
2096 	void *base, *at, *top;  in intel_pmu_drain_pebs_icl()  local
2103 	base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base;  in intel_pmu_drain_pebs_icl()
2104 	top = (struct pebs_basic *)(unsigned long)ds->pebs_index;  in intel_pmu_drain_pebs_icl()
2106 	ds->pebs_index = ds->pebs_buffer_base;  in intel_pmu_drain_pebs_icl()
2108 	mask = ((1ULL << max_pebs_events) - 1) |  in intel_pmu_drain_pebs_icl()
2109 	       (((1ULL << num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED);  in intel_pmu_drain_pebs_icl()
2112 	if (unlikely(base >= top)) {  in intel_pmu_drain_pebs_icl()
2117 	for (at = base; at < top; at += cpuc->pebs_record_size) {  in intel_pmu_drain_pebs_icl()
2120 		pebs_status = get_pebs_status(at) & cpuc->pebs_enabled;  in intel_pmu_drain_pebs_icl()
2131 		event = cpuc->events[bit];  in intel_pmu_drain_pebs_icl()
2135 		if (WARN_ON_ONCE(!event->attr.precise_ip))  in intel_pmu_drain_pebs_icl()
2139 				       top, bit, counts[bit],  in intel_pmu_drain_pebs_icl()
2163 		char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';  in intel_ds_init()
2213 				pebs_qual = "-baseline";  in intel_ds_init()
2214 				x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;  in intel_ds_init()
2228 				pr_cont("PEBS-via-PT, ");  in intel_ds_init()
2229 				x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_AUX_OUTPUT;  in intel_ds_init()