1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2004 Anton Blanchard <anton@au.ibm.com>, IBM
4 * Added mmcra[slot] support:
5 * Copyright (C) 2006-2007 Will Schmidt <willschm@us.ibm.com>, IBM
6 */
7
8 #include <linux/oprofile.h>
9 #include <linux/smp.h>
10 #include <asm/firmware.h>
11 #include <asm/ptrace.h>
12 #include <asm/processor.h>
13 #include <asm/cputable.h>
14 #include <asm/rtas.h>
15 #include <asm/oprofile_impl.h>
16 #include <asm/reg.h>
17
18 #define dbg(args...)
19 #define OPROFILE_PM_PMCSEL_MSK 0xffULL
20 #define OPROFILE_PM_UNIT_SHIFT 60
21 #define OPROFILE_PM_UNIT_MSK 0xfULL
22 #define OPROFILE_MAX_PMC_NUM 3
23 #define OPROFILE_PMSEL_FIELD_WIDTH 8
24 #define OPROFILE_UNIT_FIELD_WIDTH 4
25 #define MMCRA_SIAR_VALID_MASK 0x10000000ULL
26
27 static unsigned long reset_value[OP_MAX_COUNTER];
28
29 static int oprofile_running;
30 static int use_slot_nums;
31
32 /* mmcr values are set in power4_reg_setup, used in power4_cpu_setup */
33 static u32 mmcr0_val;
34 static u64 mmcr1_val;
35 static u64 mmcra_val;
36 static u32 cntr_marked_events;
37
power7_marked_instr_event(u64 mmcr1)38 static int power7_marked_instr_event(u64 mmcr1)
39 {
40 u64 psel, unit;
41 int pmc, cntr_marked_events = 0;
42
43 /* Given the MMCR1 value, look at the field for each counter to
44 * determine if it is a marked event. Code based on the function
45 * power7_marked_instr_event() in file arch/powerpc/perf/power7-pmu.c.
46 */
47 for (pmc = 0; pmc < 4; pmc++) {
48 psel = mmcr1 & (OPROFILE_PM_PMCSEL_MSK
49 << (OPROFILE_MAX_PMC_NUM - pmc)
50 * OPROFILE_PMSEL_FIELD_WIDTH);
51 psel = (psel >> ((OPROFILE_MAX_PMC_NUM - pmc)
52 * OPROFILE_PMSEL_FIELD_WIDTH)) & ~1ULL;
53 unit = mmcr1 & (OPROFILE_PM_UNIT_MSK
54 << (OPROFILE_PM_UNIT_SHIFT
55 - (pmc * OPROFILE_PMSEL_FIELD_WIDTH )));
56 unit = unit >> (OPROFILE_PM_UNIT_SHIFT
57 - (pmc * OPROFILE_PMSEL_FIELD_WIDTH));
58
59 switch (psel >> 4) {
60 case 2:
61 cntr_marked_events |= (pmc == 1 || pmc == 3) << pmc;
62 break;
63 case 3:
64 if (psel == 0x3c) {
65 cntr_marked_events |= (pmc == 0) << pmc;
66 break;
67 }
68
69 if (psel == 0x3e) {
70 cntr_marked_events |= (pmc != 1) << pmc;
71 break;
72 }
73
74 cntr_marked_events |= 1 << pmc;
75 break;
76 case 4:
77 case 5:
78 cntr_marked_events |= (unit == 0xd) << pmc;
79 break;
80 case 6:
81 if (psel == 0x64)
82 cntr_marked_events |= (pmc >= 2) << pmc;
83 break;
84 case 8:
85 cntr_marked_events |= (unit == 0xd) << pmc;
86 break;
87 }
88 }
89 return cntr_marked_events;
90 }
91
power4_reg_setup(struct op_counter_config * ctr,struct op_system_config * sys,int num_ctrs)92 static int power4_reg_setup(struct op_counter_config *ctr,
93 struct op_system_config *sys,
94 int num_ctrs)
95 {
96 int i;
97
98 /*
99 * The performance counter event settings are given in the mmcr0,
100 * mmcr1 and mmcra values passed from the user in the
101 * op_system_config structure (sys variable).
102 */
103 mmcr0_val = sys->mmcr0;
104 mmcr1_val = sys->mmcr1;
105 mmcra_val = sys->mmcra;
106
107 /* Power 7+ and newer architectures:
108 * Determine which counter events in the group (the group of events is
109 * specified by the bit settings in the MMCR1 register) are marked
110 * events for use in the interrupt handler. Do the calculation once
111 * before OProfile starts. Information is used in the interrupt
112 * handler. Starting with Power 7+ we only record the sample for
113 * marked events if the SIAR valid bit is set. For non marked events
114 * the sample is always recorded.
115 */
116 if (pvr_version_is(PVR_POWER7p))
117 cntr_marked_events = power7_marked_instr_event(mmcr1_val);
118 else
119 cntr_marked_events = 0; /* For older processors, set the bit map
120 * to zero so the sample will always be
121 * be recorded.
122 */
123
124 for (i = 0; i < cur_cpu_spec->num_pmcs; ++i)
125 reset_value[i] = 0x80000000UL - ctr[i].count;
126
127 /* setup user and kernel profiling */
128 if (sys->enable_kernel)
129 mmcr0_val &= ~MMCR0_KERNEL_DISABLE;
130 else
131 mmcr0_val |= MMCR0_KERNEL_DISABLE;
132
133 if (sys->enable_user)
134 mmcr0_val &= ~MMCR0_PROBLEM_DISABLE;
135 else
136 mmcr0_val |= MMCR0_PROBLEM_DISABLE;
137
138 if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) ||
139 pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) ||
140 pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX) ||
141 pvr_version_is(PVR_POWER5) || pvr_version_is(PVR_POWER5p))
142 use_slot_nums = 1;
143
144 return 0;
145 }
146
147 extern void ppc_enable_pmcs(void);
148
149 /*
150 * Older CPUs require the MMCRA sample bit to be always set, but newer
151 * CPUs only want it set for some groups. Eventually we will remove all
152 * knowledge of this bit in the kernel, oprofile userspace should be
153 * setting it when required.
154 *
155 * In order to keep current installations working we force the bit for
156 * those older CPUs. Once everyone has updated their oprofile userspace we
157 * can remove this hack.
158 */
mmcra_must_set_sample(void)159 static inline int mmcra_must_set_sample(void)
160 {
161 if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) ||
162 pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) ||
163 pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX))
164 return 1;
165
166 return 0;
167 }
168
power4_cpu_setup(struct op_counter_config * ctr)169 static int power4_cpu_setup(struct op_counter_config *ctr)
170 {
171 unsigned int mmcr0 = mmcr0_val;
172 unsigned long mmcra = mmcra_val;
173
174 ppc_enable_pmcs();
175
176 /* set the freeze bit */
177 mmcr0 |= MMCR0_FC;
178 mtspr(SPRN_MMCR0, mmcr0);
179
180 mmcr0 |= MMCR0_FCM1|MMCR0_PMXE|MMCR0_FCECE;
181 mmcr0 |= MMCR0_PMC1CE|MMCR0_PMCjCE;
182 mtspr(SPRN_MMCR0, mmcr0);
183
184 mtspr(SPRN_MMCR1, mmcr1_val);
185
186 if (mmcra_must_set_sample())
187 mmcra |= MMCRA_SAMPLE_ENABLE;
188 mtspr(SPRN_MMCRA, mmcra);
189
190 dbg("setup on cpu %d, mmcr0 %lx\n", smp_processor_id(),
191 mfspr(SPRN_MMCR0));
192 dbg("setup on cpu %d, mmcr1 %lx\n", smp_processor_id(),
193 mfspr(SPRN_MMCR1));
194 dbg("setup on cpu %d, mmcra %lx\n", smp_processor_id(),
195 mfspr(SPRN_MMCRA));
196
197 return 0;
198 }
199
power4_start(struct op_counter_config * ctr)200 static int power4_start(struct op_counter_config *ctr)
201 {
202 int i;
203 unsigned int mmcr0;
204
205 /* set the PMM bit (see comment below) */
206 mtmsr(mfmsr() | MSR_PMM);
207
208 for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) {
209 if (ctr[i].enabled) {
210 classic_ctr_write(i, reset_value[i]);
211 } else {
212 classic_ctr_write(i, 0);
213 }
214 }
215
216 mmcr0 = mfspr(SPRN_MMCR0);
217
218 /*
219 * We must clear the PMAO bit on some (GQ) chips. Just do it
220 * all the time
221 */
222 mmcr0 &= ~MMCR0_PMAO;
223
224 /*
225 * now clear the freeze bit, counting will not start until we
226 * rfid from this excetion, because only at that point will
227 * the PMM bit be cleared
228 */
229 mmcr0 &= ~MMCR0_FC;
230 mtspr(SPRN_MMCR0, mmcr0);
231
232 oprofile_running = 1;
233
234 dbg("start on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0);
235 return 0;
236 }
237
power4_stop(void)238 static void power4_stop(void)
239 {
240 unsigned int mmcr0;
241
242 /* freeze counters */
243 mmcr0 = mfspr(SPRN_MMCR0);
244 mmcr0 |= MMCR0_FC;
245 mtspr(SPRN_MMCR0, mmcr0);
246
247 oprofile_running = 0;
248
249 dbg("stop on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0);
250
251 mb();
252 }
253
254 /* Fake functions used by canonicalize_pc */
hypervisor_bucket(void)255 static void __used hypervisor_bucket(void)
256 {
257 }
258
rtas_bucket(void)259 static void __used rtas_bucket(void)
260 {
261 }
262
kernel_unknown_bucket(void)263 static void __used kernel_unknown_bucket(void)
264 {
265 }
266
267 /*
268 * On GQ and newer the MMCRA stores the HV and PR bits at the time
269 * the SIAR was sampled. We use that to work out if the SIAR was sampled in
270 * the hypervisor, our exception vectors or RTAS.
271 * If the MMCRA_SAMPLE_ENABLE bit is set, we can use the MMCRA[slot] bits
272 * to more accurately identify the address of the sampled instruction. The
273 * mmcra[slot] bits represent the slot number of a sampled instruction
274 * within an instruction group. The slot will contain a value between 1
275 * and 5 if MMCRA_SAMPLE_ENABLE is set, otherwise 0.
276 */
get_pc(struct pt_regs * regs)277 static unsigned long get_pc(struct pt_regs *regs)
278 {
279 unsigned long pc = mfspr(SPRN_SIAR);
280 unsigned long mmcra;
281 unsigned long slot;
282
283 /* Can't do much about it */
284 if (!cur_cpu_spec->oprofile_mmcra_sihv)
285 return pc;
286
287 mmcra = mfspr(SPRN_MMCRA);
288
289 if (use_slot_nums && (mmcra & MMCRA_SAMPLE_ENABLE)) {
290 slot = ((mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT);
291 if (slot > 1)
292 pc += 4 * (slot - 1);
293 }
294
295 /* Were we in the hypervisor? */
296 if (firmware_has_feature(FW_FEATURE_LPAR) &&
297 (mmcra & cur_cpu_spec->oprofile_mmcra_sihv))
298 /* function descriptor madness */
299 return *((unsigned long *)hypervisor_bucket);
300
301 /* We were in userspace, nothing to do */
302 if (mmcra & cur_cpu_spec->oprofile_mmcra_sipr)
303 return pc;
304
305 #ifdef CONFIG_PPC_RTAS
306 /* Were we in RTAS? */
307 if (pc >= rtas.base && pc < (rtas.base + rtas.size))
308 /* function descriptor madness */
309 return *((unsigned long *)rtas_bucket);
310 #endif
311
312 /* Were we in our exception vectors or SLB real mode miss handler? */
313 if (pc < 0x1000000UL)
314 return (unsigned long)__va(pc);
315
316 /* Not sure where we were */
317 if (!is_kernel_addr(pc))
318 /* function descriptor madness */
319 return *((unsigned long *)kernel_unknown_bucket);
320
321 return pc;
322 }
323
get_kernel(unsigned long pc,unsigned long mmcra)324 static int get_kernel(unsigned long pc, unsigned long mmcra)
325 {
326 int is_kernel;
327
328 if (!cur_cpu_spec->oprofile_mmcra_sihv) {
329 is_kernel = is_kernel_addr(pc);
330 } else {
331 is_kernel = ((mmcra & cur_cpu_spec->oprofile_mmcra_sipr) == 0);
332 }
333
334 return is_kernel;
335 }
336
pmc_overflow(unsigned long val)337 static bool pmc_overflow(unsigned long val)
338 {
339 if ((int)val < 0)
340 return true;
341
342 /*
343 * Events on POWER7 can roll back if a speculative event doesn't
344 * eventually complete. Unfortunately in some rare cases they will
345 * raise a performance monitor exception. We need to catch this to
346 * ensure we reset the PMC. In all cases the PMC will be 256 or less
347 * cycles from overflow.
348 *
349 * We only do this if the first pass fails to find any overflowing
350 * PMCs because a user might set a period of less than 256 and we
351 * don't want to mistakenly reset them.
352 */
353 if (pvr_version_is(PVR_POWER7) && ((0x80000000 - val) <= 256))
354 return true;
355
356 return false;
357 }
358
power4_handle_interrupt(struct pt_regs * regs,struct op_counter_config * ctr)359 static void power4_handle_interrupt(struct pt_regs *regs,
360 struct op_counter_config *ctr)
361 {
362 unsigned long pc;
363 int is_kernel;
364 int val;
365 int i;
366 unsigned int mmcr0;
367 unsigned long mmcra;
368 bool siar_valid = false;
369
370 mmcra = mfspr(SPRN_MMCRA);
371
372 pc = get_pc(regs);
373 is_kernel = get_kernel(pc, mmcra);
374
375 /* set the PMM bit (see comment below) */
376 mtmsr(mfmsr() | MSR_PMM);
377
378 /* Check that the SIAR valid bit in MMCRA is set to 1. */
379 if ((mmcra & MMCRA_SIAR_VALID_MASK) == MMCRA_SIAR_VALID_MASK)
380 siar_valid = true;
381
382 for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) {
383 val = classic_ctr_read(i);
384 if (pmc_overflow(val)) {
385 if (oprofile_running && ctr[i].enabled) {
386 /* Power 7+ and newer architectures:
387 * If the event is a marked event, then only
388 * save the sample if the SIAR valid bit is
389 * set. If the event is not marked, then
390 * always save the sample.
391 * Note, the Sample enable bit in the MMCRA
392 * register must be set to 1 if the group
393 * contains a marked event.
394 */
395 if ((siar_valid &&
396 (cntr_marked_events & (1 << i)))
397 || !(cntr_marked_events & (1 << i)))
398 oprofile_add_ext_sample(pc, regs, i,
399 is_kernel);
400
401 classic_ctr_write(i, reset_value[i]);
402 } else {
403 classic_ctr_write(i, 0);
404 }
405 }
406 }
407
408 mmcr0 = mfspr(SPRN_MMCR0);
409
410 /* reset the perfmon trigger */
411 mmcr0 |= MMCR0_PMXE;
412
413 /*
414 * We must clear the PMAO bit on some (GQ) chips. Just do it
415 * all the time
416 */
417 mmcr0 &= ~MMCR0_PMAO;
418
419 /* Clear the appropriate bits in the MMCRA */
420 mmcra &= ~cur_cpu_spec->oprofile_mmcra_clear;
421 mtspr(SPRN_MMCRA, mmcra);
422
423 /*
424 * now clear the freeze bit, counting will not start until we
425 * rfid from this exception, because only at that point will
426 * the PMM bit be cleared
427 */
428 mmcr0 &= ~MMCR0_FC;
429 mtspr(SPRN_MMCR0, mmcr0);
430 }
431
432 struct op_powerpc_model op_model_power4 = {
433 .reg_setup = power4_reg_setup,
434 .cpu_setup = power4_cpu_setup,
435 .start = power4_start,
436 .stop = power4_stop,
437 .handle_interrupt = power4_handle_interrupt,
438 };
439
