1 /*
2  * Broadcom Brahma-B15 CPU read-ahead cache management functions
3  *
4  * Copyright (C) 2015-2016 Broadcom
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 
11 #include <linux/err.h>
12 #include <linux/spinlock.h>
13 #include <linux/io.h>
14 #include <linux/bitops.h>
15 #include <linux/of_address.h>
16 #include <linux/notifier.h>
17 #include <linux/cpu.h>
18 #include <linux/syscore_ops.h>
19 #include <linux/reboot.h>
20 
21 #include <asm/cacheflush.h>
22 #include <asm/hardware/cache-b15-rac.h>
23 
24 extern void v7_flush_kern_cache_all(void);
25 
26 /* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */
27 #define RAC_CONFIG0_REG			(0x78)
28 #define  RACENPREF_MASK			(0x3)
29 #define  RACPREFINST_SHIFT		(0)
30 #define  RACENINST_SHIFT		(2)
31 #define  RACPREFDATA_SHIFT		(4)
32 #define  RACENDATA_SHIFT		(6)
33 #define  RAC_CPU_SHIFT			(8)
34 #define  RACCFG_MASK			(0xff)
35 #define RAC_CONFIG1_REG			(0x7c)
36 /* Brahma-B15 is a quad-core only design */
37 #define B15_RAC_FLUSH_REG		(0x80)
38 /* Brahma-B53 is an octo-core design */
39 #define B53_RAC_FLUSH_REG		(0x84)
40 #define  FLUSH_RAC			(1 << 0)
41 
42 /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
43 #define RAC_DATA_INST_EN_MASK		(1 << RACPREFINST_SHIFT | \
44 					 RACENPREF_MASK << RACENINST_SHIFT | \
45 					 1 << RACPREFDATA_SHIFT | \
46 					 RACENPREF_MASK << RACENDATA_SHIFT)
47 
48 #define RAC_ENABLED			0
49 /* Special state where we want to bypass the spinlock and call directly
50  * into the v7 cache maintenance operations during suspend/resume
51  */
52 #define RAC_SUSPENDED			1
53 
54 static void __iomem *b15_rac_base;
55 static DEFINE_SPINLOCK(rac_lock);
56 
57 static u32 rac_config0_reg;
58 static u32 rac_flush_offset;
59 
60 /* Initialization flag to avoid checking for b15_rac_base, and to prevent
61  * multi-platform kernels from crashing here as well.
62  */
63 static unsigned long b15_rac_flags;
64 
__b15_rac_disable(void)65 static inline u32 __b15_rac_disable(void)
66 {
67 	u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
68 	__raw_writel(0, b15_rac_base + RAC_CONFIG0_REG);
69 	dmb();
70 	return val;
71 }
72 
__b15_rac_flush(void)73 static inline void __b15_rac_flush(void)
74 {
75 	u32 reg;
76 
77 	__raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset);
78 	do {
79 		/* This dmb() is required to force the Bus Interface Unit
80 		 * to clean oustanding writes, and forces an idle cycle
81 		 * to be inserted.
82 		 */
83 		dmb();
84 		reg = __raw_readl(b15_rac_base + rac_flush_offset);
85 	} while (reg & FLUSH_RAC);
86 }
87 
b15_rac_disable_and_flush(void)88 static inline u32 b15_rac_disable_and_flush(void)
89 {
90 	u32 reg;
91 
92 	reg = __b15_rac_disable();
93 	__b15_rac_flush();
94 	return reg;
95 }
96 
__b15_rac_enable(u32 val)97 static inline void __b15_rac_enable(u32 val)
98 {
99 	__raw_writel(val, b15_rac_base + RAC_CONFIG0_REG);
100 	/* dsb() is required here to be consistent with __flush_icache_all() */
101 	dsb();
102 }
103 
104 #define BUILD_RAC_CACHE_OP(name, bar)				\
105 void b15_flush_##name(void)					\
106 {								\
107 	unsigned int do_flush;					\
108 	u32 val = 0;						\
109 								\
110 	if (test_bit(RAC_SUSPENDED, &b15_rac_flags)) {		\
111 		v7_flush_##name();				\
112 		bar;						\
113 		return;						\
114 	}							\
115 								\
116 	spin_lock(&rac_lock);					\
117 	do_flush = test_bit(RAC_ENABLED, &b15_rac_flags);	\
118 	if (do_flush)						\
119 		val = b15_rac_disable_and_flush();		\
120 	v7_flush_##name();					\
121 	if (!do_flush)						\
122 		bar;						\
123 	else							\
124 		__b15_rac_enable(val);				\
125 	spin_unlock(&rac_lock);					\
126 }
127 
128 #define nobarrier
129 
130 /* The readahead cache present in the Brahma-B15 CPU is a special piece of
131  * hardware after the integrated L2 cache of the B15 CPU complex whose purpose
132  * is to prefetch instruction and/or data with a line size of either 64 bytes
133  * or 256 bytes. The rationale is that the data-bus of the CPU interface is
134  * optimized for 256-bytes transactions, and enabling the readahead cache
135  * provides a significant performance boost we want it enabled (typically
136  * twice the performance for a memcpy benchmark application).
137  *
138  * The readahead cache is transparent for Modified Virtual Addresses
139  * cache maintenance operations: ICIMVAU, DCIMVAC, DCCMVAC, DCCMVAU and
140  * DCCIMVAC.
141  *
142  * It is however not transparent for the following cache maintenance
143  * operations: DCISW, DCCSW, DCCISW, ICIALLUIS and ICIALLU which is precisely
144  * what we are patching here with our BUILD_RAC_CACHE_OP here.
145  */
146 BUILD_RAC_CACHE_OP(kern_cache_all, nobarrier);
147 
b15_rac_enable(void)148 static void b15_rac_enable(void)
149 {
150 	unsigned int cpu;
151 	u32 enable = 0;
152 
153 	for_each_possible_cpu(cpu)
154 		enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT));
155 
156 	b15_rac_disable_and_flush();
157 	__b15_rac_enable(enable);
158 }
159 
b15_rac_reboot_notifier(struct notifier_block * nb,unsigned long action,void * data)160 static int b15_rac_reboot_notifier(struct notifier_block *nb,
161 				   unsigned long action,
162 				   void *data)
163 {
164 	/* During kexec, we are not yet migrated on the boot CPU, so we need to
165 	 * make sure we are SMP safe here. Once the RAC is disabled, flag it as
166 	 * suspended such that the hotplug notifier returns early.
167 	 */
168 	if (action == SYS_RESTART) {
169 		spin_lock(&rac_lock);
170 		b15_rac_disable_and_flush();
171 		clear_bit(RAC_ENABLED, &b15_rac_flags);
172 		set_bit(RAC_SUSPENDED, &b15_rac_flags);
173 		spin_unlock(&rac_lock);
174 	}
175 
176 	return NOTIFY_DONE;
177 }
178 
179 static struct notifier_block b15_rac_reboot_nb = {
180 	.notifier_call	= b15_rac_reboot_notifier,
181 };
182 
183 /* The CPU hotplug case is the most interesting one, we basically need to make
184  * sure that the RAC is disabled for the entire system prior to having a CPU
185  * die, in particular prior to this dying CPU having exited the coherency
186  * domain.
187  *
188  * Once this CPU is marked dead, we can safely re-enable the RAC for the
189  * remaining CPUs in the system which are still online.
190  *
191  * Offlining a CPU is the problematic case, onlining a CPU is not much of an
192  * issue since the CPU and its cache-level hierarchy will start filling with
193  * the RAC disabled, so L1 and L2 only.
194  *
195  * In this function, we should NOT have to verify any unsafe setting/condition
196  * b15_rac_base:
197  *
198  *   It is protected by the RAC_ENABLED flag which is cleared by default, and
199  *   being cleared when initial procedure is done. b15_rac_base had been set at
200  *   that time.
201  *
202  * RAC_ENABLED:
203  *   There is a small timing windows, in b15_rac_init(), between
204  *      cpuhp_setup_state_*()
205  *      ...
206  *      set RAC_ENABLED
207  *   However, there is no hotplug activity based on the Linux booting procedure.
208  *
209  * Since we have to disable RAC for all cores, we keep RAC on as long as as
210  * possible (disable it as late as possible) to gain the cache benefit.
211  *
212  * Thus, dying/dead states are chosen here
213  *
214  * We are choosing not do disable the RAC on a per-CPU basis, here, if we did
215  * we would want to consider disabling it as early as possible to benefit the
216  * other active CPUs.
217  */
218 
219 /* Running on the dying CPU */
b15_rac_dying_cpu(unsigned int cpu)220 static int b15_rac_dying_cpu(unsigned int cpu)
221 {
222 	/* During kexec/reboot, the RAC is disabled via the reboot notifier
223 	 * return early here.
224 	 */
225 	if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
226 		return 0;
227 
228 	spin_lock(&rac_lock);
229 
230 	/* Indicate that we are starting a hotplug procedure */
231 	__clear_bit(RAC_ENABLED, &b15_rac_flags);
232 
233 	/* Disable the readahead cache and save its value to a global */
234 	rac_config0_reg = b15_rac_disable_and_flush();
235 
236 	spin_unlock(&rac_lock);
237 
238 	return 0;
239 }
240 
241 /* Running on a non-dying CPU */
b15_rac_dead_cpu(unsigned int cpu)242 static int b15_rac_dead_cpu(unsigned int cpu)
243 {
244 	/* During kexec/reboot, the RAC is disabled via the reboot notifier
245 	 * return early here.
246 	 */
247 	if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
248 		return 0;
249 
250 	spin_lock(&rac_lock);
251 
252 	/* And enable it */
253 	__b15_rac_enable(rac_config0_reg);
254 	__set_bit(RAC_ENABLED, &b15_rac_flags);
255 
256 	spin_unlock(&rac_lock);
257 
258 	return 0;
259 }
260 
b15_rac_suspend(void)261 static int b15_rac_suspend(void)
262 {
263 	/* Suspend the read-ahead cache oeprations, forcing our cache
264 	 * implementation to fallback to the regular ARMv7 calls.
265 	 *
266 	 * We are guaranteed to be running on the boot CPU at this point and
267 	 * with every other CPU quiesced, so setting RAC_SUSPENDED is not racy
268 	 * here.
269 	 */
270 	rac_config0_reg = b15_rac_disable_and_flush();
271 	set_bit(RAC_SUSPENDED, &b15_rac_flags);
272 
273 	return 0;
274 }
275 
b15_rac_resume(void)276 static void b15_rac_resume(void)
277 {
278 	/* Coming out of a S3 suspend/resume cycle, the read-ahead cache
279 	 * register RAC_CONFIG0_REG will be restored to its default value, make
280 	 * sure we re-enable it and set the enable flag, we are also guaranteed
281 	 * to run on the boot CPU, so not racy again.
282 	 */
283 	__b15_rac_enable(rac_config0_reg);
284 	clear_bit(RAC_SUSPENDED, &b15_rac_flags);
285 }
286 
287 static struct syscore_ops b15_rac_syscore_ops = {
288 	.suspend	= b15_rac_suspend,
289 	.resume		= b15_rac_resume,
290 };
291 
b15_rac_init(void)292 static int __init b15_rac_init(void)
293 {
294 	struct device_node *dn, *cpu_dn;
295 	int ret = 0, cpu;
296 	u32 reg, en_mask = 0;
297 
298 	dn = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl");
299 	if (!dn)
300 		return -ENODEV;
301 
302 	if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n"))
303 		goto out;
304 
305 	b15_rac_base = of_iomap(dn, 0);
306 	if (!b15_rac_base) {
307 		pr_err("failed to remap BIU control base\n");
308 		ret = -ENOMEM;
309 		goto out;
310 	}
311 
312 	cpu_dn = of_get_cpu_node(0, NULL);
313 	if (!cpu_dn) {
314 		ret = -ENODEV;
315 		goto out;
316 	}
317 
318 	if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15"))
319 		rac_flush_offset = B15_RAC_FLUSH_REG;
320 	else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53"))
321 		rac_flush_offset = B53_RAC_FLUSH_REG;
322 	else {
323 		pr_err("Unsupported CPU\n");
324 		of_node_put(cpu_dn);
325 		ret = -EINVAL;
326 		goto out;
327 	}
328 	of_node_put(cpu_dn);
329 
330 	ret = register_reboot_notifier(&b15_rac_reboot_nb);
331 	if (ret) {
332 		pr_err("failed to register reboot notifier\n");
333 		iounmap(b15_rac_base);
334 		goto out;
335 	}
336 
337 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
338 		ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
339 					"arm/cache-b15-rac:dead",
340 					NULL, b15_rac_dead_cpu);
341 		if (ret)
342 			goto out_unmap;
343 
344 		ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING,
345 					"arm/cache-b15-rac:dying",
346 					NULL, b15_rac_dying_cpu);
347 		if (ret)
348 			goto out_cpu_dead;
349 	}
350 
351 	if (IS_ENABLED(CONFIG_PM_SLEEP))
352 		register_syscore_ops(&b15_rac_syscore_ops);
353 
354 	spin_lock(&rac_lock);
355 	reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
356 	for_each_possible_cpu(cpu)
357 		en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT));
358 	WARN(reg & en_mask, "Read-ahead cache not previously disabled\n");
359 
360 	b15_rac_enable();
361 	set_bit(RAC_ENABLED, &b15_rac_flags);
362 	spin_unlock(&rac_lock);
363 
364 	pr_info("Broadcom Brahma-B15 readahead cache at: 0x%p\n",
365 		b15_rac_base + RAC_CONFIG0_REG);
366 
367 	goto out;
368 
369 out_cpu_dead:
370 	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING);
371 out_unmap:
372 	unregister_reboot_notifier(&b15_rac_reboot_nb);
373 	iounmap(b15_rac_base);
374 out:
375 	of_node_put(dn);
376 	return ret;
377 }
378 arch_initcall(b15_rac_init);
379