1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Cell Broadband Engine Performance Monitor
4 *
5 * (C) Copyright IBM Corporation 2001,2006
6 *
7 * Author:
8 * David Erb (djerb@us.ibm.com)
9 * Kevin Corry (kevcorry@us.ibm.com)
10 */
11
12 #include <linux/interrupt.h>
13 #include <linux/types.h>
14 #include <linux/export.h>
15 #include <asm/io.h>
16 #include <asm/irq_regs.h>
17 #include <asm/machdep.h>
18 #include <asm/pmc.h>
19 #include <asm/reg.h>
20 #include <asm/spu.h>
21 #include <asm/cell-regs.h>
22
23 #include "interrupt.h"
24
25 /*
26 * When writing to write-only mmio addresses, save a shadow copy. All of the
27 * registers are 32-bit, but stored in the upper-half of a 64-bit field in
28 * pmd_regs.
29 */
30
31 #define WRITE_WO_MMIO(reg, x) \
32 do { \
33 u32 _x = (x); \
34 struct cbe_pmd_regs __iomem *pmd_regs; \
35 struct cbe_pmd_shadow_regs *shadow_regs; \
36 pmd_regs = cbe_get_cpu_pmd_regs(cpu); \
37 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \
38 out_be64(&(pmd_regs->reg), (((u64)_x) << 32)); \
39 shadow_regs->reg = _x; \
40 } while (0)
41
42 #define READ_SHADOW_REG(val, reg) \
43 do { \
44 struct cbe_pmd_shadow_regs *shadow_regs; \
45 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \
46 (val) = shadow_regs->reg; \
47 } while (0)
48
49 #define READ_MMIO_UPPER32(val, reg) \
50 do { \
51 struct cbe_pmd_regs __iomem *pmd_regs; \
52 pmd_regs = cbe_get_cpu_pmd_regs(cpu); \
53 (val) = (u32)(in_be64(&pmd_regs->reg) >> 32); \
54 } while (0)
55
56 /*
57 * Physical counter registers.
58 * Each physical counter can act as one 32-bit counter or two 16-bit counters.
59 */
60
cbe_read_phys_ctr(u32 cpu,u32 phys_ctr)61 u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr)
62 {
63 u32 val_in_latch, val = 0;
64
65 if (phys_ctr < NR_PHYS_CTRS) {
66 READ_SHADOW_REG(val_in_latch, counter_value_in_latch);
67
68 /* Read the latch or the actual counter, whichever is newer. */
69 if (val_in_latch & (1 << phys_ctr)) {
70 READ_SHADOW_REG(val, pm_ctr[phys_ctr]);
71 } else {
72 READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]);
73 }
74 }
75
76 return val;
77 }
78 EXPORT_SYMBOL_GPL(cbe_read_phys_ctr);
79
cbe_write_phys_ctr(u32 cpu,u32 phys_ctr,u32 val)80 void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val)
81 {
82 struct cbe_pmd_shadow_regs *shadow_regs;
83 u32 pm_ctrl;
84
85 if (phys_ctr < NR_PHYS_CTRS) {
86 /* Writing to a counter only writes to a hardware latch.
87 * The new value is not propagated to the actual counter
88 * until the performance monitor is enabled.
89 */
90 WRITE_WO_MMIO(pm_ctr[phys_ctr], val);
91
92 pm_ctrl = cbe_read_pm(cpu, pm_control);
93 if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) {
94 /* The counters are already active, so we need to
95 * rewrite the pm_control register to "re-enable"
96 * the PMU.
97 */
98 cbe_write_pm(cpu, pm_control, pm_ctrl);
99 } else {
100 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);
101 shadow_regs->counter_value_in_latch |= (1 << phys_ctr);
102 }
103 }
104 }
105 EXPORT_SYMBOL_GPL(cbe_write_phys_ctr);
106
107 /*
108 * "Logical" counter registers.
109 * These will read/write 16-bits or 32-bits depending on the
110 * current size of the counter. Counters 4 - 7 are always 16-bit.
111 */
112
cbe_read_ctr(u32 cpu,u32 ctr)113 u32 cbe_read_ctr(u32 cpu, u32 ctr)
114 {
115 u32 val;
116 u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1);
117
118 val = cbe_read_phys_ctr(cpu, phys_ctr);
119
120 if (cbe_get_ctr_size(cpu, phys_ctr) == 16)
121 val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff);
122
123 return val;
124 }
125 EXPORT_SYMBOL_GPL(cbe_read_ctr);
126
cbe_write_ctr(u32 cpu,u32 ctr,u32 val)127 void cbe_write_ctr(u32 cpu, u32 ctr, u32 val)
128 {
129 u32 phys_ctr;
130 u32 phys_val;
131
132 phys_ctr = ctr & (NR_PHYS_CTRS - 1);
133
134 if (cbe_get_ctr_size(cpu, phys_ctr) == 16) {
135 phys_val = cbe_read_phys_ctr(cpu, phys_ctr);
136
137 if (ctr < NR_PHYS_CTRS)
138 val = (val << 16) | (phys_val & 0xffff);
139 else
140 val = (val & 0xffff) | (phys_val & 0xffff0000);
141 }
142
143 cbe_write_phys_ctr(cpu, phys_ctr, val);
144 }
145 EXPORT_SYMBOL_GPL(cbe_write_ctr);
146
147 /*
148 * Counter-control registers.
149 * Each "logical" counter has a corresponding control register.
150 */
151
cbe_read_pm07_control(u32 cpu,u32 ctr)152 u32 cbe_read_pm07_control(u32 cpu, u32 ctr)
153 {
154 u32 pm07_control = 0;
155
156 if (ctr < NR_CTRS)
157 READ_SHADOW_REG(pm07_control, pm07_control[ctr]);
158
159 return pm07_control;
160 }
161 EXPORT_SYMBOL_GPL(cbe_read_pm07_control);
162
cbe_write_pm07_control(u32 cpu,u32 ctr,u32 val)163 void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val)
164 {
165 if (ctr < NR_CTRS)
166 WRITE_WO_MMIO(pm07_control[ctr], val);
167 }
168 EXPORT_SYMBOL_GPL(cbe_write_pm07_control);
169
170 /*
171 * Other PMU control registers. Most of these are write-only.
172 */
173
cbe_read_pm(u32 cpu,enum pm_reg_name reg)174 u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg)
175 {
176 u32 val = 0;
177
178 switch (reg) {
179 case group_control:
180 READ_SHADOW_REG(val, group_control);
181 break;
182
183 case debug_bus_control:
184 READ_SHADOW_REG(val, debug_bus_control);
185 break;
186
187 case trace_address:
188 READ_MMIO_UPPER32(val, trace_address);
189 break;
190
191 case ext_tr_timer:
192 READ_SHADOW_REG(val, ext_tr_timer);
193 break;
194
195 case pm_status:
196 READ_MMIO_UPPER32(val, pm_status);
197 break;
198
199 case pm_control:
200 READ_SHADOW_REG(val, pm_control);
201 break;
202
203 case pm_interval:
204 READ_MMIO_UPPER32(val, pm_interval);
205 break;
206
207 case pm_start_stop:
208 READ_SHADOW_REG(val, pm_start_stop);
209 break;
210 }
211
212 return val;
213 }
214 EXPORT_SYMBOL_GPL(cbe_read_pm);
215
cbe_write_pm(u32 cpu,enum pm_reg_name reg,u32 val)216 void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val)
217 {
218 switch (reg) {
219 case group_control:
220 WRITE_WO_MMIO(group_control, val);
221 break;
222
223 case debug_bus_control:
224 WRITE_WO_MMIO(debug_bus_control, val);
225 break;
226
227 case trace_address:
228 WRITE_WO_MMIO(trace_address, val);
229 break;
230
231 case ext_tr_timer:
232 WRITE_WO_MMIO(ext_tr_timer, val);
233 break;
234
235 case pm_status:
236 WRITE_WO_MMIO(pm_status, val);
237 break;
238
239 case pm_control:
240 WRITE_WO_MMIO(pm_control, val);
241 break;
242
243 case pm_interval:
244 WRITE_WO_MMIO(pm_interval, val);
245 break;
246
247 case pm_start_stop:
248 WRITE_WO_MMIO(pm_start_stop, val);
249 break;
250 }
251 }
252 EXPORT_SYMBOL_GPL(cbe_write_pm);
253
254 /*
255 * Get/set the size of a physical counter to either 16 or 32 bits.
256 */
257
cbe_get_ctr_size(u32 cpu,u32 phys_ctr)258 u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr)
259 {
260 u32 pm_ctrl, size = 0;
261
262 if (phys_ctr < NR_PHYS_CTRS) {
263 pm_ctrl = cbe_read_pm(cpu, pm_control);
264 size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32;
265 }
266
267 return size;
268 }
269 EXPORT_SYMBOL_GPL(cbe_get_ctr_size);
270
cbe_set_ctr_size(u32 cpu,u32 phys_ctr,u32 ctr_size)271 void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size)
272 {
273 u32 pm_ctrl;
274
275 if (phys_ctr < NR_PHYS_CTRS) {
276 pm_ctrl = cbe_read_pm(cpu, pm_control);
277 switch (ctr_size) {
278 case 16:
279 pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr);
280 break;
281
282 case 32:
283 pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr);
284 break;
285 }
286 cbe_write_pm(cpu, pm_control, pm_ctrl);
287 }
288 }
289 EXPORT_SYMBOL_GPL(cbe_set_ctr_size);
290
291 /*
292 * Enable/disable the entire performance monitoring unit.
293 * When we enable the PMU, all pending writes to counters get committed.
294 */
295
cbe_enable_pm(u32 cpu)296 void cbe_enable_pm(u32 cpu)
297 {
298 struct cbe_pmd_shadow_regs *shadow_regs;
299 u32 pm_ctrl;
300
301 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);
302 shadow_regs->counter_value_in_latch = 0;
303
304 pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON;
305 cbe_write_pm(cpu, pm_control, pm_ctrl);
306 }
307 EXPORT_SYMBOL_GPL(cbe_enable_pm);
308
cbe_disable_pm(u32 cpu)309 void cbe_disable_pm(u32 cpu)
310 {
311 u32 pm_ctrl;
312 pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON;
313 cbe_write_pm(cpu, pm_control, pm_ctrl);
314 }
315 EXPORT_SYMBOL_GPL(cbe_disable_pm);
316
317 /*
318 * Reading from the trace_buffer.
319 * The trace buffer is two 64-bit registers. Reading from
320 * the second half automatically increments the trace_address.
321 */
322
cbe_read_trace_buffer(u32 cpu,u64 * buf)323 void cbe_read_trace_buffer(u32 cpu, u64 *buf)
324 {
325 struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu);
326
327 *buf++ = in_be64(&pmd_regs->trace_buffer_0_63);
328 *buf++ = in_be64(&pmd_regs->trace_buffer_64_127);
329 }
330 EXPORT_SYMBOL_GPL(cbe_read_trace_buffer);
331
332 /*
333 * Enabling/disabling interrupts for the entire performance monitoring unit.
334 */
335
cbe_get_and_clear_pm_interrupts(u32 cpu)336 u32 cbe_get_and_clear_pm_interrupts(u32 cpu)
337 {
338 /* Reading pm_status clears the interrupt bits. */
339 return cbe_read_pm(cpu, pm_status);
340 }
341 EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts);
342
cbe_enable_pm_interrupts(u32 cpu,u32 thread,u32 mask)343 void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask)
344 {
345 /* Set which node and thread will handle the next interrupt. */
346 iic_set_interrupt_routing(cpu, thread, 0);
347
348 /* Enable the interrupt bits in the pm_status register. */
349 if (mask)
350 cbe_write_pm(cpu, pm_status, mask);
351 }
352 EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts);
353
cbe_disable_pm_interrupts(u32 cpu)354 void cbe_disable_pm_interrupts(u32 cpu)
355 {
356 cbe_get_and_clear_pm_interrupts(cpu);
357 cbe_write_pm(cpu, pm_status, 0);
358 }
359 EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts);
360
cbe_pm_irq(int irq,void * dev_id)361 static irqreturn_t cbe_pm_irq(int irq, void *dev_id)
362 {
363 perf_irq(get_irq_regs());
364 return IRQ_HANDLED;
365 }
366
cbe_init_pm_irq(void)367 static int __init cbe_init_pm_irq(void)
368 {
369 unsigned int irq;
370 int rc, node;
371
372 for_each_online_node(node) {
373 irq = irq_create_mapping(NULL, IIC_IRQ_IOEX_PMI |
374 (node << IIC_IRQ_NODE_SHIFT));
375 if (!irq) {
376 printk("ERROR: Unable to allocate irq for node %d\n",
377 node);
378 return -EINVAL;
379 }
380
381 rc = request_irq(irq, cbe_pm_irq,
382 0, "cbe-pmu-0", NULL);
383 if (rc) {
384 printk("ERROR: Request for irq on node %d failed\n",
385 node);
386 return rc;
387 }
388 }
389
390 return 0;
391 }
392 machine_arch_initcall(cell, cbe_init_pm_irq);
393
cbe_sync_irq(int node)394 void cbe_sync_irq(int node)
395 {
396 unsigned int irq;
397
398 irq = irq_find_mapping(NULL,
399 IIC_IRQ_IOEX_PMI
400 | (node << IIC_IRQ_NODE_SHIFT));
401
402 if (!irq) {
403 printk(KERN_WARNING "ERROR, unable to get existing irq %d " \
404 "for node %d\n", irq, node);
405 return;
406 }
407
408 synchronize_irq(irq);
409 }
410 EXPORT_SYMBOL_GPL(cbe_sync_irq);
411
412
