1 /*
2 * Performance events - AMD IBS
3 *
4 * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter
5 *
6 * For licencing details see kernel-base/COPYING
7 */
8
9 #include <linux/perf_event.h>
10 #include <linux/init.h>
11 #include <linux/export.h>
12 #include <linux/pci.h>
13 #include <linux/ptrace.h>
14 #include <linux/syscore_ops.h>
15 #include <linux/sched/clock.h>
16
17 #include <asm/apic.h>
18
19 #include "../perf_event.h"
20
21 static u32 ibs_caps;
22
23 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
24
25 #include <linux/kprobes.h>
26 #include <linux/hardirq.h>
27
28 #include <asm/nmi.h>
29 #include <asm/amd-ibs.h>
30
31 #define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT)
32 #define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT
33
34
35 /*
36 * IBS states:
37 *
38 * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken
39 * and any further add()s must fail.
40 *
41 * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are
42 * complicated by the fact that the IBS hardware can send late NMIs (ie. after
43 * we've cleared the EN bit).
44 *
45 * In order to consume these late NMIs we have the STOPPED state, any NMI that
46 * happens after we've cleared the EN state will clear this bit and report the
47 * NMI handled (this is fundamentally racy in the face or multiple NMI sources,
48 * someone else can consume our BIT and our NMI will go unhandled).
49 *
50 * And since we cannot set/clear this separate bit together with the EN bit,
51 * there are races; if we cleared STARTED early, an NMI could land in
52 * between clearing STARTED and clearing the EN bit (in fact multiple NMIs
53 * could happen if the period is small enough), and consume our STOPPED bit
54 * and trigger streams of unhandled NMIs.
55 *
56 * If, however, we clear STARTED late, an NMI can hit between clearing the
57 * EN bit and clearing STARTED, still see STARTED set and process the event.
58 * If this event will have the VALID bit clear, we bail properly, but this
59 * is not a given. With VALID set we can end up calling pmu::stop() again
60 * (the throttle logic) and trigger the WARNs in there.
61 *
62 * So what we do is set STOPPING before clearing EN to avoid the pmu::stop()
63 * nesting, and clear STARTED late, so that we have a well defined state over
64 * the clearing of the EN bit.
65 *
66 * XXX: we could probably be using !atomic bitops for all this.
67 */
68
69 enum ibs_states {
70 IBS_ENABLED = 0,
71 IBS_STARTED = 1,
72 IBS_STOPPING = 2,
73 IBS_STOPPED = 3,
74
75 IBS_MAX_STATES,
76 };
77
78 struct cpu_perf_ibs {
79 struct perf_event *event;
80 unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)];
81 };
82
83 struct perf_ibs {
84 struct pmu pmu;
85 unsigned int msr;
86 u64 config_mask;
87 u64 cnt_mask;
88 u64 enable_mask;
89 u64 valid_mask;
90 u64 max_period;
91 unsigned long offset_mask[1];
92 int offset_max;
93 unsigned int fetch_count_reset_broken : 1;
94 unsigned int fetch_ignore_if_zero_rip : 1;
95 struct cpu_perf_ibs __percpu *pcpu;
96
97 struct attribute **format_attrs;
98 struct attribute_group format_group;
99 const struct attribute_group *attr_groups[2];
100
101 u64 (*get_count)(u64 config);
102 };
103
104 static int
perf_event_set_period(struct hw_perf_event * hwc,u64 min,u64 max,u64 * hw_period)105 perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period)
106 {
107 s64 left = local64_read(&hwc->period_left);
108 s64 period = hwc->sample_period;
109 int overflow = 0;
110
111 /*
112 * If we are way outside a reasonable range then just skip forward:
113 */
114 if (unlikely(left <= -period)) {
115 left = period;
116 local64_set(&hwc->period_left, left);
117 hwc->last_period = period;
118 overflow = 1;
119 }
120
121 if (unlikely(left < (s64)min)) {
122 left += period;
123 local64_set(&hwc->period_left, left);
124 hwc->last_period = period;
125 overflow = 1;
126 }
127
128 /*
129 * If the hw period that triggers the sw overflow is too short
130 * we might hit the irq handler. This biases the results.
131 * Thus we shorten the next-to-last period and set the last
132 * period to the max period.
133 */
134 if (left > max) {
135 left -= max;
136 if (left > max)
137 left = max;
138 else if (left < min)
139 left = min;
140 }
141
142 *hw_period = (u64)left;
143
144 return overflow;
145 }
146
147 static int
perf_event_try_update(struct perf_event * event,u64 new_raw_count,int width)148 perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width)
149 {
150 struct hw_perf_event *hwc = &event->hw;
151 int shift = 64 - width;
152 u64 prev_raw_count;
153 u64 delta;
154
155 /*
156 * Careful: an NMI might modify the previous event value.
157 *
158 * Our tactic to handle this is to first atomically read and
159 * exchange a new raw count - then add that new-prev delta
160 * count to the generic event atomically:
161 */
162 prev_raw_count = local64_read(&hwc->prev_count);
163 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
164 new_raw_count) != prev_raw_count)
165 return 0;
166
167 /*
168 * Now we have the new raw value and have updated the prev
169 * timestamp already. We can now calculate the elapsed delta
170 * (event-)time and add that to the generic event.
171 *
172 * Careful, not all hw sign-extends above the physical width
173 * of the count.
174 */
175 delta = (new_raw_count << shift) - (prev_raw_count << shift);
176 delta >>= shift;
177
178 local64_add(delta, &event->count);
179 local64_sub(delta, &hwc->period_left);
180
181 return 1;
182 }
183
184 static struct perf_ibs perf_ibs_fetch;
185 static struct perf_ibs perf_ibs_op;
186
get_ibs_pmu(int type)187 static struct perf_ibs *get_ibs_pmu(int type)
188 {
189 if (perf_ibs_fetch.pmu.type == type)
190 return &perf_ibs_fetch;
191 if (perf_ibs_op.pmu.type == type)
192 return &perf_ibs_op;
193 return NULL;
194 }
195
196 /*
197 * Use IBS for precise event sampling:
198 *
199 * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count
200 * perf record -a -e r076:p ... # same as -e cpu-cycles:p
201 * perf record -a -e r0C1:p ... # use ibs op counting micro-ops
202 *
203 * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl,
204 * MSRC001_1033) is used to select either cycle or micro-ops counting
205 * mode.
206 *
207 * The rip of IBS samples has skid 0. Thus, IBS supports precise
208 * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the
209 * rip is invalid when IBS was not able to record the rip correctly.
210 * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then.
211 *
212 */
perf_ibs_precise_event(struct perf_event * event,u64 * config)213 static int perf_ibs_precise_event(struct perf_event *event, u64 *config)
214 {
215 switch (event->attr.precise_ip) {
216 case 0:
217 return -ENOENT;
218 case 1:
219 case 2:
220 break;
221 default:
222 return -EOPNOTSUPP;
223 }
224
225 switch (event->attr.type) {
226 case PERF_TYPE_HARDWARE:
227 switch (event->attr.config) {
228 case PERF_COUNT_HW_CPU_CYCLES:
229 *config = 0;
230 return 0;
231 }
232 break;
233 case PERF_TYPE_RAW:
234 switch (event->attr.config) {
235 case 0x0076:
236 *config = 0;
237 return 0;
238 case 0x00C1:
239 *config = IBS_OP_CNT_CTL;
240 return 0;
241 }
242 break;
243 default:
244 return -ENOENT;
245 }
246
247 return -EOPNOTSUPP;
248 }
249
perf_ibs_init(struct perf_event * event)250 static int perf_ibs_init(struct perf_event *event)
251 {
252 struct hw_perf_event *hwc = &event->hw;
253 struct perf_ibs *perf_ibs;
254 u64 max_cnt, config;
255 int ret;
256
257 perf_ibs = get_ibs_pmu(event->attr.type);
258 if (perf_ibs) {
259 config = event->attr.config;
260 } else {
261 perf_ibs = &perf_ibs_op;
262 ret = perf_ibs_precise_event(event, &config);
263 if (ret)
264 return ret;
265 }
266
267 if (event->pmu != &perf_ibs->pmu)
268 return -ENOENT;
269
270 if (config & ~perf_ibs->config_mask)
271 return -EINVAL;
272
273 if (hwc->sample_period) {
274 if (config & perf_ibs->cnt_mask)
275 /* raw max_cnt may not be set */
276 return -EINVAL;
277 if (!event->attr.sample_freq && hwc->sample_period & 0x0f)
278 /*
279 * lower 4 bits can not be set in ibs max cnt,
280 * but allowing it in case we adjust the
281 * sample period to set a frequency.
282 */
283 return -EINVAL;
284 hwc->sample_period &= ~0x0FULL;
285 if (!hwc->sample_period)
286 hwc->sample_period = 0x10;
287 } else {
288 max_cnt = config & perf_ibs->cnt_mask;
289 config &= ~perf_ibs->cnt_mask;
290 event->attr.sample_period = max_cnt << 4;
291 hwc->sample_period = event->attr.sample_period;
292 }
293
294 if (!hwc->sample_period)
295 return -EINVAL;
296
297 /*
298 * If we modify hwc->sample_period, we also need to update
299 * hwc->last_period and hwc->period_left.
300 */
301 hwc->last_period = hwc->sample_period;
302 local64_set(&hwc->period_left, hwc->sample_period);
303
304 hwc->config_base = perf_ibs->msr;
305 hwc->config = config;
306
307 return 0;
308 }
309
perf_ibs_set_period(struct perf_ibs * perf_ibs,struct hw_perf_event * hwc,u64 * period)310 static int perf_ibs_set_period(struct perf_ibs *perf_ibs,
311 struct hw_perf_event *hwc, u64 *period)
312 {
313 int overflow;
314
315 /* ignore lower 4 bits in min count: */
316 overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period);
317 local64_set(&hwc->prev_count, 0);
318
319 return overflow;
320 }
321
get_ibs_fetch_count(u64 config)322 static u64 get_ibs_fetch_count(u64 config)
323 {
324 union ibs_fetch_ctl fetch_ctl = (union ibs_fetch_ctl)config;
325
326 return fetch_ctl.fetch_cnt << 4;
327 }
328
get_ibs_op_count(u64 config)329 static u64 get_ibs_op_count(u64 config)
330 {
331 union ibs_op_ctl op_ctl = (union ibs_op_ctl)config;
332 u64 count = 0;
333
334 /*
335 * If the internal 27-bit counter rolled over, the count is MaxCnt
336 * and the lower 7 bits of CurCnt are randomized.
337 * Otherwise CurCnt has the full 27-bit current counter value.
338 */
339 if (op_ctl.op_val) {
340 count = op_ctl.opmaxcnt << 4;
341 if (ibs_caps & IBS_CAPS_OPCNTEXT)
342 count += op_ctl.opmaxcnt_ext << 20;
343 } else if (ibs_caps & IBS_CAPS_RDWROPCNT) {
344 count = op_ctl.opcurcnt;
345 }
346
347 return count;
348 }
349
350 static void
perf_ibs_event_update(struct perf_ibs * perf_ibs,struct perf_event * event,u64 * config)351 perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event,
352 u64 *config)
353 {
354 u64 count = perf_ibs->get_count(*config);
355
356 /*
357 * Set width to 64 since we do not overflow on max width but
358 * instead on max count. In perf_ibs_set_period() we clear
359 * prev count manually on overflow.
360 */
361 while (!perf_event_try_update(event, count, 64)) {
362 rdmsrl(event->hw.config_base, *config);
363 count = perf_ibs->get_count(*config);
364 }
365 }
366
perf_ibs_enable_event(struct perf_ibs * perf_ibs,struct hw_perf_event * hwc,u64 config)367 static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
368 struct hw_perf_event *hwc, u64 config)
369 {
370 u64 tmp = hwc->config | config;
371
372 if (perf_ibs->fetch_count_reset_broken)
373 wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask);
374
375 wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask);
376 }
377
378 /*
379 * Erratum #420 Instruction-Based Sampling Engine May Generate
380 * Interrupt that Cannot Be Cleared:
381 *
382 * Must clear counter mask first, then clear the enable bit. See
383 * Revision Guide for AMD Family 10h Processors, Publication #41322.
384 */
perf_ibs_disable_event(struct perf_ibs * perf_ibs,struct hw_perf_event * hwc,u64 config)385 static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,
386 struct hw_perf_event *hwc, u64 config)
387 {
388 config &= ~perf_ibs->cnt_mask;
389 if (boot_cpu_data.x86 == 0x10)
390 wrmsrl(hwc->config_base, config);
391 config &= ~perf_ibs->enable_mask;
392 wrmsrl(hwc->config_base, config);
393 }
394
395 /*
396 * We cannot restore the ibs pmu state, so we always needs to update
397 * the event while stopping it and then reset the state when starting
398 * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in
399 * perf_ibs_start()/perf_ibs_stop() and instead always do it.
400 */
perf_ibs_start(struct perf_event * event,int flags)401 static void perf_ibs_start(struct perf_event *event, int flags)
402 {
403 struct hw_perf_event *hwc = &event->hw;
404 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
405 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
406 u64 period, config = 0;
407
408 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
409 return;
410
411 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
412 hwc->state = 0;
413
414 perf_ibs_set_period(perf_ibs, hwc, &period);
415 if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) {
416 config |= period & IBS_OP_MAX_CNT_EXT_MASK;
417 period &= ~IBS_OP_MAX_CNT_EXT_MASK;
418 }
419 config |= period >> 4;
420
421 /*
422 * Set STARTED before enabling the hardware, such that a subsequent NMI
423 * must observe it.
424 */
425 set_bit(IBS_STARTED, pcpu->state);
426 clear_bit(IBS_STOPPING, pcpu->state);
427 perf_ibs_enable_event(perf_ibs, hwc, config);
428
429 perf_event_update_userpage(event);
430 }
431
perf_ibs_stop(struct perf_event * event,int flags)432 static void perf_ibs_stop(struct perf_event *event, int flags)
433 {
434 struct hw_perf_event *hwc = &event->hw;
435 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
436 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
437 u64 config;
438 int stopping;
439
440 if (test_and_set_bit(IBS_STOPPING, pcpu->state))
441 return;
442
443 stopping = test_bit(IBS_STARTED, pcpu->state);
444
445 if (!stopping && (hwc->state & PERF_HES_UPTODATE))
446 return;
447
448 rdmsrl(hwc->config_base, config);
449
450 if (stopping) {
451 /*
452 * Set STOPPED before disabling the hardware, such that it
453 * must be visible to NMIs the moment we clear the EN bit,
454 * at which point we can generate an !VALID sample which
455 * we need to consume.
456 */
457 set_bit(IBS_STOPPED, pcpu->state);
458 perf_ibs_disable_event(perf_ibs, hwc, config);
459 /*
460 * Clear STARTED after disabling the hardware; if it were
461 * cleared before an NMI hitting after the clear but before
462 * clearing the EN bit might think it a spurious NMI and not
463 * handle it.
464 *
465 * Clearing it after, however, creates the problem of the NMI
466 * handler seeing STARTED but not having a valid sample.
467 */
468 clear_bit(IBS_STARTED, pcpu->state);
469 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
470 hwc->state |= PERF_HES_STOPPED;
471 }
472
473 if (hwc->state & PERF_HES_UPTODATE)
474 return;
475
476 /*
477 * Clear valid bit to not count rollovers on update, rollovers
478 * are only updated in the irq handler.
479 */
480 config &= ~perf_ibs->valid_mask;
481
482 perf_ibs_event_update(perf_ibs, event, &config);
483 hwc->state |= PERF_HES_UPTODATE;
484 }
485
perf_ibs_add(struct perf_event * event,int flags)486 static int perf_ibs_add(struct perf_event *event, int flags)
487 {
488 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
489 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
490
491 if (test_and_set_bit(IBS_ENABLED, pcpu->state))
492 return -ENOSPC;
493
494 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
495
496 pcpu->event = event;
497
498 if (flags & PERF_EF_START)
499 perf_ibs_start(event, PERF_EF_RELOAD);
500
501 return 0;
502 }
503
perf_ibs_del(struct perf_event * event,int flags)504 static void perf_ibs_del(struct perf_event *event, int flags)
505 {
506 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
507 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
508
509 if (!test_and_clear_bit(IBS_ENABLED, pcpu->state))
510 return;
511
512 perf_ibs_stop(event, PERF_EF_UPDATE);
513
514 pcpu->event = NULL;
515
516 perf_event_update_userpage(event);
517 }
518
perf_ibs_read(struct perf_event * event)519 static void perf_ibs_read(struct perf_event *event) { }
520
521 PMU_FORMAT_ATTR(rand_en, "config:57");
522 PMU_FORMAT_ATTR(cnt_ctl, "config:19");
523
524 static struct attribute *ibs_fetch_format_attrs[] = {
525 &format_attr_rand_en.attr,
526 NULL,
527 };
528
529 static struct attribute *ibs_op_format_attrs[] = {
530 NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */
531 NULL,
532 };
533
534 static struct perf_ibs perf_ibs_fetch = {
535 .pmu = {
536 .task_ctx_nr = perf_invalid_context,
537
538 .event_init = perf_ibs_init,
539 .add = perf_ibs_add,
540 .del = perf_ibs_del,
541 .start = perf_ibs_start,
542 .stop = perf_ibs_stop,
543 .read = perf_ibs_read,
544 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
545 },
546 .msr = MSR_AMD64_IBSFETCHCTL,
547 .config_mask = IBS_FETCH_CONFIG_MASK,
548 .cnt_mask = IBS_FETCH_MAX_CNT,
549 .enable_mask = IBS_FETCH_ENABLE,
550 .valid_mask = IBS_FETCH_VAL,
551 .max_period = IBS_FETCH_MAX_CNT << 4,
552 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK },
553 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT,
554 .format_attrs = ibs_fetch_format_attrs,
555
556 .get_count = get_ibs_fetch_count,
557 };
558
559 static struct perf_ibs perf_ibs_op = {
560 .pmu = {
561 .task_ctx_nr = perf_invalid_context,
562
563 .event_init = perf_ibs_init,
564 .add = perf_ibs_add,
565 .del = perf_ibs_del,
566 .start = perf_ibs_start,
567 .stop = perf_ibs_stop,
568 .read = perf_ibs_read,
569 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
570 },
571 .msr = MSR_AMD64_IBSOPCTL,
572 .config_mask = IBS_OP_CONFIG_MASK,
573 .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT |
574 IBS_OP_CUR_CNT_RAND,
575 .enable_mask = IBS_OP_ENABLE,
576 .valid_mask = IBS_OP_VAL,
577 .max_period = IBS_OP_MAX_CNT << 4,
578 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK },
579 .offset_max = MSR_AMD64_IBSOP_REG_COUNT,
580 .format_attrs = ibs_op_format_attrs,
581
582 .get_count = get_ibs_op_count,
583 };
584
perf_ibs_handle_irq(struct perf_ibs * perf_ibs,struct pt_regs * iregs)585 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
586 {
587 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
588 struct perf_event *event = pcpu->event;
589 struct hw_perf_event *hwc;
590 struct perf_sample_data data;
591 struct perf_raw_record raw;
592 struct pt_regs regs;
593 struct perf_ibs_data ibs_data;
594 int offset, size, check_rip, offset_max, throttle = 0;
595 unsigned int msr;
596 u64 *buf, *config, period, new_config = 0;
597
598 if (!test_bit(IBS_STARTED, pcpu->state)) {
599 fail:
600 /*
601 * Catch spurious interrupts after stopping IBS: After
602 * disabling IBS there could be still incoming NMIs
603 * with samples that even have the valid bit cleared.
604 * Mark all this NMIs as handled.
605 */
606 if (test_and_clear_bit(IBS_STOPPED, pcpu->state))
607 return 1;
608
609 return 0;
610 }
611
612 if (WARN_ON_ONCE(!event))
613 goto fail;
614
615 hwc = &event->hw;
616 msr = hwc->config_base;
617 buf = ibs_data.regs;
618 rdmsrl(msr, *buf);
619 if (!(*buf++ & perf_ibs->valid_mask))
620 goto fail;
621
622 config = &ibs_data.regs[0];
623 perf_ibs_event_update(perf_ibs, event, config);
624 perf_sample_data_init(&data, 0, hwc->last_period);
625 if (!perf_ibs_set_period(perf_ibs, hwc, &period))
626 goto out; /* no sw counter overflow */
627
628 ibs_data.caps = ibs_caps;
629 size = 1;
630 offset = 1;
631 check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
632 if (event->attr.sample_type & PERF_SAMPLE_RAW)
633 offset_max = perf_ibs->offset_max;
634 else if (check_rip)
635 offset_max = 3;
636 else
637 offset_max = 1;
638 do {
639 rdmsrl(msr + offset, *buf++);
640 size++;
641 offset = find_next_bit(perf_ibs->offset_mask,
642 perf_ibs->offset_max,
643 offset + 1);
644 } while (offset < offset_max);
645 /*
646 * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately
647 * depending on their availability.
648 * Can't add to offset_max as they are staggered
649 */
650 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
651 if (perf_ibs == &perf_ibs_op) {
652 if (ibs_caps & IBS_CAPS_BRNTRGT) {
653 rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++);
654 size++;
655 }
656 if (ibs_caps & IBS_CAPS_OPDATA4) {
657 rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++);
658 size++;
659 }
660 }
661 if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) {
662 rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++);
663 size++;
664 }
665 }
666 ibs_data.size = sizeof(u64) * size;
667
668 regs = *iregs;
669 if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) {
670 regs.flags &= ~PERF_EFLAGS_EXACT;
671 } else {
672 /* Workaround for erratum #1197 */
673 if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1]))
674 goto out;
675
676 set_linear_ip(®s, ibs_data.regs[1]);
677 regs.flags |= PERF_EFLAGS_EXACT;
678 }
679
680 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
681 raw = (struct perf_raw_record){
682 .frag = {
683 .size = sizeof(u32) + ibs_data.size,
684 .data = ibs_data.data,
685 },
686 };
687 data.raw = &raw;
688 }
689
690 throttle = perf_event_overflow(event, &data, ®s);
691 out:
692 if (throttle) {
693 perf_ibs_stop(event, 0);
694 } else {
695 if (perf_ibs == &perf_ibs_op) {
696 if (ibs_caps & IBS_CAPS_OPCNTEXT) {
697 new_config = period & IBS_OP_MAX_CNT_EXT_MASK;
698 period &= ~IBS_OP_MAX_CNT_EXT_MASK;
699 }
700 if ((ibs_caps & IBS_CAPS_RDWROPCNT) && (*config & IBS_OP_CNT_CTL))
701 new_config |= *config & IBS_OP_CUR_CNT_RAND;
702 }
703 new_config |= period >> 4;
704
705 perf_ibs_enable_event(perf_ibs, hwc, new_config);
706 }
707
708 perf_event_update_userpage(event);
709
710 return 1;
711 }
712
713 static int
perf_ibs_nmi_handler(unsigned int cmd,struct pt_regs * regs)714 perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
715 {
716 u64 stamp = sched_clock();
717 int handled = 0;
718
719 handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs);
720 handled += perf_ibs_handle_irq(&perf_ibs_op, regs);
721
722 if (handled)
723 inc_irq_stat(apic_perf_irqs);
724
725 perf_sample_event_took(sched_clock() - stamp);
726
727 return handled;
728 }
729 NOKPROBE_SYMBOL(perf_ibs_nmi_handler);
730
perf_ibs_pmu_init(struct perf_ibs * perf_ibs,char * name)731 static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
732 {
733 struct cpu_perf_ibs __percpu *pcpu;
734 int ret;
735
736 pcpu = alloc_percpu(struct cpu_perf_ibs);
737 if (!pcpu)
738 return -ENOMEM;
739
740 perf_ibs->pcpu = pcpu;
741
742 /* register attributes */
743 if (perf_ibs->format_attrs[0]) {
744 memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group));
745 perf_ibs->format_group.name = "format";
746 perf_ibs->format_group.attrs = perf_ibs->format_attrs;
747
748 memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups));
749 perf_ibs->attr_groups[0] = &perf_ibs->format_group;
750 perf_ibs->pmu.attr_groups = perf_ibs->attr_groups;
751 }
752
753 ret = perf_pmu_register(&perf_ibs->pmu, name, -1);
754 if (ret) {
755 perf_ibs->pcpu = NULL;
756 free_percpu(pcpu);
757 }
758
759 return ret;
760 }
761
perf_event_ibs_init(void)762 static __init void perf_event_ibs_init(void)
763 {
764 struct attribute **attr = ibs_op_format_attrs;
765
766 /*
767 * Some chips fail to reset the fetch count when it is written; instead
768 * they need a 0-1 transition of IbsFetchEn.
769 */
770 if (boot_cpu_data.x86 >= 0x16 && boot_cpu_data.x86 <= 0x18)
771 perf_ibs_fetch.fetch_count_reset_broken = 1;
772
773 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model < 0x10)
774 perf_ibs_fetch.fetch_ignore_if_zero_rip = 1;
775
776 perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
777
778 if (ibs_caps & IBS_CAPS_OPCNT) {
779 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
780 *attr++ = &format_attr_cnt_ctl.attr;
781 }
782
783 if (ibs_caps & IBS_CAPS_OPCNTEXT) {
784 perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK;
785 perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK;
786 perf_ibs_op.cnt_mask |= IBS_OP_MAX_CNT_EXT_MASK;
787 }
788
789 perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
790
791 register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
792 pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps);
793 }
794
795 #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */
796
perf_event_ibs_init(void)797 static __init void perf_event_ibs_init(void) { }
798
799 #endif
800
801 /* IBS - apic initialization, for perf and oprofile */
802
__get_ibs_caps(void)803 static __init u32 __get_ibs_caps(void)
804 {
805 u32 caps;
806 unsigned int max_level;
807
808 if (!boot_cpu_has(X86_FEATURE_IBS))
809 return 0;
810
811 /* check IBS cpuid feature flags */
812 max_level = cpuid_eax(0x80000000);
813 if (max_level < IBS_CPUID_FEATURES)
814 return IBS_CAPS_DEFAULT;
815
816 caps = cpuid_eax(IBS_CPUID_FEATURES);
817 if (!(caps & IBS_CAPS_AVAIL))
818 /* cpuid flags not valid */
819 return IBS_CAPS_DEFAULT;
820
821 return caps;
822 }
823
get_ibs_caps(void)824 u32 get_ibs_caps(void)
825 {
826 return ibs_caps;
827 }
828
829 EXPORT_SYMBOL(get_ibs_caps);
830
get_eilvt(int offset)831 static inline int get_eilvt(int offset)
832 {
833 return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
834 }
835
put_eilvt(int offset)836 static inline int put_eilvt(int offset)
837 {
838 return !setup_APIC_eilvt(offset, 0, 0, 1);
839 }
840
841 /*
842 * Check and reserve APIC extended interrupt LVT offset for IBS if available.
843 */
ibs_eilvt_valid(void)844 static inline int ibs_eilvt_valid(void)
845 {
846 int offset;
847 u64 val;
848 int valid = 0;
849
850 preempt_disable();
851
852 rdmsrl(MSR_AMD64_IBSCTL, val);
853 offset = val & IBSCTL_LVT_OFFSET_MASK;
854
855 if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
856 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
857 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
858 goto out;
859 }
860
861 if (!get_eilvt(offset)) {
862 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
863 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
864 goto out;
865 }
866
867 valid = 1;
868 out:
869 preempt_enable();
870
871 return valid;
872 }
873
setup_ibs_ctl(int ibs_eilvt_off)874 static int setup_ibs_ctl(int ibs_eilvt_off)
875 {
876 struct pci_dev *cpu_cfg;
877 int nodes;
878 u32 value = 0;
879
880 nodes = 0;
881 cpu_cfg = NULL;
882 do {
883 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
884 PCI_DEVICE_ID_AMD_10H_NB_MISC,
885 cpu_cfg);
886 if (!cpu_cfg)
887 break;
888 ++nodes;
889 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
890 | IBSCTL_LVT_OFFSET_VALID);
891 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
892 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
893 pci_dev_put(cpu_cfg);
894 pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n",
895 value);
896 return -EINVAL;
897 }
898 } while (1);
899
900 if (!nodes) {
901 pr_debug("No CPU node configured for IBS\n");
902 return -ENODEV;
903 }
904
905 return 0;
906 }
907
908 /*
909 * This runs only on the current cpu. We try to find an LVT offset and
910 * setup the local APIC. For this we must disable preemption. On
911 * success we initialize all nodes with this offset. This updates then
912 * the offset in the IBS_CTL per-node msr. The per-core APIC setup of
913 * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that
914 * is using the new offset.
915 */
force_ibs_eilvt_setup(void)916 static void force_ibs_eilvt_setup(void)
917 {
918 int offset;
919 int ret;
920
921 preempt_disable();
922 /* find the next free available EILVT entry, skip offset 0 */
923 for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) {
924 if (get_eilvt(offset))
925 break;
926 }
927 preempt_enable();
928
929 if (offset == APIC_EILVT_NR_MAX) {
930 pr_debug("No EILVT entry available\n");
931 return;
932 }
933
934 ret = setup_ibs_ctl(offset);
935 if (ret)
936 goto out;
937
938 if (!ibs_eilvt_valid())
939 goto out;
940
941 pr_info("LVT offset %d assigned\n", offset);
942
943 return;
944 out:
945 preempt_disable();
946 put_eilvt(offset);
947 preempt_enable();
948 return;
949 }
950
ibs_eilvt_setup(void)951 static void ibs_eilvt_setup(void)
952 {
953 /*
954 * Force LVT offset assignment for family 10h: The offsets are
955 * not assigned by the BIOS for this family, so the OS is
956 * responsible for doing it. If the OS assignment fails, fall
957 * back to BIOS settings and try to setup this.
958 */
959 if (boot_cpu_data.x86 == 0x10)
960 force_ibs_eilvt_setup();
961 }
962
get_ibs_lvt_offset(void)963 static inline int get_ibs_lvt_offset(void)
964 {
965 u64 val;
966
967 rdmsrl(MSR_AMD64_IBSCTL, val);
968 if (!(val & IBSCTL_LVT_OFFSET_VALID))
969 return -EINVAL;
970
971 return val & IBSCTL_LVT_OFFSET_MASK;
972 }
973
setup_APIC_ibs(void)974 static void setup_APIC_ibs(void)
975 {
976 int offset;
977
978 offset = get_ibs_lvt_offset();
979 if (offset < 0)
980 goto failed;
981
982 if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
983 return;
984 failed:
985 pr_warn("perf: IBS APIC setup failed on cpu #%d\n",
986 smp_processor_id());
987 }
988
clear_APIC_ibs(void)989 static void clear_APIC_ibs(void)
990 {
991 int offset;
992
993 offset = get_ibs_lvt_offset();
994 if (offset >= 0)
995 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
996 }
997
x86_pmu_amd_ibs_starting_cpu(unsigned int cpu)998 static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu)
999 {
1000 setup_APIC_ibs();
1001 return 0;
1002 }
1003
1004 #ifdef CONFIG_PM
1005
perf_ibs_suspend(void)1006 static int perf_ibs_suspend(void)
1007 {
1008 clear_APIC_ibs();
1009 return 0;
1010 }
1011
perf_ibs_resume(void)1012 static void perf_ibs_resume(void)
1013 {
1014 ibs_eilvt_setup();
1015 setup_APIC_ibs();
1016 }
1017
1018 static struct syscore_ops perf_ibs_syscore_ops = {
1019 .resume = perf_ibs_resume,
1020 .suspend = perf_ibs_suspend,
1021 };
1022
perf_ibs_pm_init(void)1023 static void perf_ibs_pm_init(void)
1024 {
1025 register_syscore_ops(&perf_ibs_syscore_ops);
1026 }
1027
1028 #else
1029
perf_ibs_pm_init(void)1030 static inline void perf_ibs_pm_init(void) { }
1031
1032 #endif
1033
x86_pmu_amd_ibs_dying_cpu(unsigned int cpu)1034 static int x86_pmu_amd_ibs_dying_cpu(unsigned int cpu)
1035 {
1036 clear_APIC_ibs();
1037 return 0;
1038 }
1039
amd_ibs_init(void)1040 static __init int amd_ibs_init(void)
1041 {
1042 u32 caps;
1043
1044 caps = __get_ibs_caps();
1045 if (!caps)
1046 return -ENODEV; /* ibs not supported by the cpu */
1047
1048 ibs_eilvt_setup();
1049
1050 if (!ibs_eilvt_valid())
1051 return -EINVAL;
1052
1053 perf_ibs_pm_init();
1054
1055 ibs_caps = caps;
1056 /* make ibs_caps visible to other cpus: */
1057 smp_mb();
1058 /*
1059 * x86_pmu_amd_ibs_starting_cpu will be called from core on
1060 * all online cpus.
1061 */
1062 cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_IBS_STARTING,
1063 "perf/x86/amd/ibs:starting",
1064 x86_pmu_amd_ibs_starting_cpu,
1065 x86_pmu_amd_ibs_dying_cpu);
1066
1067 perf_event_ibs_init();
1068
1069 return 0;
1070 }
1071
1072 /* Since we need the pci subsystem to init ibs we can't do this earlier: */
1073 device_initcall(amd_ibs_init);
1074
