1 /*
2 * Copyright (C) 2015-2016 Socionext Inc.
3 * Author: Masahiro Yamada <yamada.masahiro@socionext.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16 #define pr_fmt(fmt) "uniphier: " fmt
17
18 #include <linux/bitops.h>
19 #include <linux/init.h>
20 #include <linux/io.h>
21 #include <linux/log2.h>
22 #include <linux/of_address.h>
23 #include <linux/slab.h>
24 #include <asm/hardware/cache-uniphier.h>
25 #include <asm/outercache.h>
26
27 /* control registers */
28 #define UNIPHIER_SSCC 0x0 /* Control Register */
29 #define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */
30 #define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */
31 #define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */
32 #define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */
33 #define UNIPHIER_SSCC_ON BIT(0) /* enable cache */
34 #define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */
35 #define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */
36
37 /* revision registers */
38 #define UNIPHIER_SSCID 0x0 /* ID Register */
39
40 /* operation registers */
41 #define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */
42 #define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */
43 #define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */
44 #define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */
45 #define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */
46 #define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */
47 #define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */
48 #define UNIPHIER_SSCOQM_S_MASK (0x3 << 17)
49 #define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17)
50 #define UNIPHIER_SSCOQM_S_ALL (0x1 << 17)
51 #define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */
52 #define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */
53 #define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */
54 #define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */
55 #define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */
56 #define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */
57 #define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/
58 #define UNIPHIER_SSCOPPQSEF_FE BIT(1)
59 #define UNIPHIER_SSCOPPQSEF_OE BIT(0)
60 #define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */
61 #define UNIPHIER_SSCOLPQS_EF BIT(2)
62 #define UNIPHIER_SSCOLPQS_EST BIT(1)
63 #define UNIPHIER_SSCOLPQS_QST BIT(0)
64
65 /* Is the operation region specified by address range? */
66 #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
67 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
68
69 /**
70 * uniphier_cache_data - UniPhier outer cache specific data
71 *
72 * @ctrl_base: virtual base address of control registers
73 * @rev_base: virtual base address of revision registers
74 * @op_base: virtual base address of operation registers
75 * @way_mask: each bit specifies if the way is present
76 * @nsets: number of associativity sets
77 * @line_size: line size in bytes
78 * @range_op_max_size: max size that can be handled by a single range operation
79 * @list: list node to include this level in the whole cache hierarchy
80 */
81 struct uniphier_cache_data {
82 void __iomem *ctrl_base;
83 void __iomem *rev_base;
84 void __iomem *op_base;
85 void __iomem *way_ctrl_base;
86 u32 way_mask;
87 u32 nsets;
88 u32 line_size;
89 u32 range_op_max_size;
90 struct list_head list;
91 };
92
93 /*
94 * List of the whole outer cache hierarchy. This list is only modified during
95 * the early boot stage, so no mutex is taken for the access to the list.
96 */
97 static LIST_HEAD(uniphier_cache_list);
98
99 /**
100 * __uniphier_cache_sync - perform a sync point for a particular cache level
101 *
102 * @data: cache controller specific data
103 */
__uniphier_cache_sync(struct uniphier_cache_data * data)104 static void __uniphier_cache_sync(struct uniphier_cache_data *data)
105 {
106 /* This sequence need not be atomic. Do not disable IRQ. */
107 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
108 data->op_base + UNIPHIER_SSCOPE);
109 /* need a read back to confirm */
110 readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
111 }
112
113 /**
114 * __uniphier_cache_maint_common - run a queue operation for a particular level
115 *
116 * @data: cache controller specific data
117 * @start: start address of range operation (don't care for "all" operation)
118 * @size: data size of range operation (don't care for "all" operation)
119 * @operation: flags to specify the desired cache operation
120 */
__uniphier_cache_maint_common(struct uniphier_cache_data * data,unsigned long start,unsigned long size,u32 operation)121 static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
122 unsigned long start,
123 unsigned long size,
124 u32 operation)
125 {
126 unsigned long flags;
127
128 /*
129 * No spin lock is necessary here because:
130 *
131 * [1] This outer cache controller is able to accept maintenance
132 * operations from multiple CPUs at a time in an SMP system; if a
133 * maintenance operation is under way and another operation is issued,
134 * the new one is stored in the queue. The controller performs one
135 * operation after another. If the queue is full, the status register,
136 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
137 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
138 * different instances for each CPU, i.e. each CPU can track the status
139 * of the maintenance operations triggered by itself.
140 *
141 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
142 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
143 * guarantees the registration sequence is atomic; the write access to
144 * them are arbitrated by the hardware. The first accessor to the
145 * register, UNIPHIER_SSCOQM, holds the access right and it is released
146 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU
147 * is holding the access right, other CPUs fail to register operations.
148 * One CPU should not hold the access right for a long time, so local
149 * IRQs should be disabled while the following sequence.
150 */
151 local_irq_save(flags);
152
153 /* clear the complete notification flag */
154 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
155
156 do {
157 /* set cache operation */
158 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
159 data->op_base + UNIPHIER_SSCOQM);
160
161 /* set address range if needed */
162 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
163 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
164 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
165 }
166 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
167 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
168
169 /* wait until the operation is completed */
170 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
171 UNIPHIER_SSCOLPQS_EF))
172 cpu_relax();
173
174 local_irq_restore(flags);
175 }
176
__uniphier_cache_maint_all(struct uniphier_cache_data * data,u32 operation)177 static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
178 u32 operation)
179 {
180 __uniphier_cache_maint_common(data, 0, 0,
181 UNIPHIER_SSCOQM_S_ALL | operation);
182
183 __uniphier_cache_sync(data);
184 }
185
__uniphier_cache_maint_range(struct uniphier_cache_data * data,unsigned long start,unsigned long end,u32 operation)186 static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
187 unsigned long start, unsigned long end,
188 u32 operation)
189 {
190 unsigned long size;
191
192 /*
193 * If the start address is not aligned,
194 * perform a cache operation for the first cache-line
195 */
196 start = start & ~(data->line_size - 1);
197
198 size = end - start;
199
200 if (unlikely(size >= (unsigned long)(-data->line_size))) {
201 /* this means cache operation for all range */
202 __uniphier_cache_maint_all(data, operation);
203 return;
204 }
205
206 /*
207 * If the end address is not aligned,
208 * perform a cache operation for the last cache-line
209 */
210 size = ALIGN(size, data->line_size);
211
212 while (size) {
213 unsigned long chunk_size = min_t(unsigned long, size,
214 data->range_op_max_size);
215
216 __uniphier_cache_maint_common(data, start, chunk_size,
217 UNIPHIER_SSCOQM_S_RANGE | operation);
218
219 start += chunk_size;
220 size -= chunk_size;
221 }
222
223 __uniphier_cache_sync(data);
224 }
225
__uniphier_cache_enable(struct uniphier_cache_data * data,bool on)226 static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
227 {
228 u32 val = 0;
229
230 if (on)
231 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
232
233 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
234 }
235
__uniphier_cache_set_active_ways(struct uniphier_cache_data * data)236 static void __init __uniphier_cache_set_active_ways(
237 struct uniphier_cache_data *data)
238 {
239 unsigned int cpu;
240
241 for_each_possible_cpu(cpu)
242 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
243 }
244
uniphier_cache_maint_range(unsigned long start,unsigned long end,u32 operation)245 static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
246 u32 operation)
247 {
248 struct uniphier_cache_data *data;
249
250 list_for_each_entry(data, &uniphier_cache_list, list)
251 __uniphier_cache_maint_range(data, start, end, operation);
252 }
253
uniphier_cache_maint_all(u32 operation)254 static void uniphier_cache_maint_all(u32 operation)
255 {
256 struct uniphier_cache_data *data;
257
258 list_for_each_entry(data, &uniphier_cache_list, list)
259 __uniphier_cache_maint_all(data, operation);
260 }
261
uniphier_cache_inv_range(unsigned long start,unsigned long end)262 static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
263 {
264 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
265 }
266
uniphier_cache_clean_range(unsigned long start,unsigned long end)267 static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
268 {
269 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
270 }
271
uniphier_cache_flush_range(unsigned long start,unsigned long end)272 static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
273 {
274 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
275 }
276
uniphier_cache_inv_all(void)277 static void __init uniphier_cache_inv_all(void)
278 {
279 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
280 }
281
uniphier_cache_flush_all(void)282 static void uniphier_cache_flush_all(void)
283 {
284 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
285 }
286
uniphier_cache_disable(void)287 static void uniphier_cache_disable(void)
288 {
289 struct uniphier_cache_data *data;
290
291 list_for_each_entry_reverse(data, &uniphier_cache_list, list)
292 __uniphier_cache_enable(data, false);
293
294 uniphier_cache_flush_all();
295 }
296
uniphier_cache_enable(void)297 static void __init uniphier_cache_enable(void)
298 {
299 struct uniphier_cache_data *data;
300
301 uniphier_cache_inv_all();
302
303 list_for_each_entry(data, &uniphier_cache_list, list) {
304 __uniphier_cache_enable(data, true);
305 __uniphier_cache_set_active_ways(data);
306 }
307 }
308
uniphier_cache_sync(void)309 static void uniphier_cache_sync(void)
310 {
311 struct uniphier_cache_data *data;
312
313 list_for_each_entry(data, &uniphier_cache_list, list)
314 __uniphier_cache_sync(data);
315 }
316
317 static const struct of_device_id uniphier_cache_match[] __initconst = {
318 { .compatible = "socionext,uniphier-system-cache" },
319 { /* sentinel */ }
320 };
321
__uniphier_cache_init(struct device_node * np,unsigned int * cache_level)322 static int __init __uniphier_cache_init(struct device_node *np,
323 unsigned int *cache_level)
324 {
325 struct uniphier_cache_data *data;
326 u32 level, cache_size;
327 struct device_node *next_np;
328 int ret = 0;
329
330 if (!of_match_node(uniphier_cache_match, np)) {
331 pr_err("L%d: not compatible with uniphier cache\n",
332 *cache_level);
333 return -EINVAL;
334 }
335
336 if (of_property_read_u32(np, "cache-level", &level)) {
337 pr_err("L%d: cache-level is not specified\n", *cache_level);
338 return -EINVAL;
339 }
340
341 if (level != *cache_level) {
342 pr_err("L%d: cache-level is unexpected value %d\n",
343 *cache_level, level);
344 return -EINVAL;
345 }
346
347 if (!of_property_read_bool(np, "cache-unified")) {
348 pr_err("L%d: cache-unified is not specified\n", *cache_level);
349 return -EINVAL;
350 }
351
352 data = kzalloc(sizeof(*data), GFP_KERNEL);
353 if (!data)
354 return -ENOMEM;
355
356 if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
357 !is_power_of_2(data->line_size)) {
358 pr_err("L%d: cache-line-size is unspecified or invalid\n",
359 *cache_level);
360 ret = -EINVAL;
361 goto err;
362 }
363
364 if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
365 !is_power_of_2(data->nsets)) {
366 pr_err("L%d: cache-sets is unspecified or invalid\n",
367 *cache_level);
368 ret = -EINVAL;
369 goto err;
370 }
371
372 if (of_property_read_u32(np, "cache-size", &cache_size) ||
373 cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
374 pr_err("L%d: cache-size is unspecified or invalid\n",
375 *cache_level);
376 ret = -EINVAL;
377 goto err;
378 }
379
380 data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
381 0);
382
383 data->ctrl_base = of_iomap(np, 0);
384 if (!data->ctrl_base) {
385 pr_err("L%d: failed to map control register\n", *cache_level);
386 ret = -ENOMEM;
387 goto err;
388 }
389
390 data->rev_base = of_iomap(np, 1);
391 if (!data->rev_base) {
392 pr_err("L%d: failed to map revision register\n", *cache_level);
393 ret = -ENOMEM;
394 goto err;
395 }
396
397 data->op_base = of_iomap(np, 2);
398 if (!data->op_base) {
399 pr_err("L%d: failed to map operation register\n", *cache_level);
400 ret = -ENOMEM;
401 goto err;
402 }
403
404 data->way_ctrl_base = data->ctrl_base + 0xc00;
405
406 if (*cache_level == 2) {
407 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
408 /*
409 * The size of range operation is limited to (1 << 22) or less
410 * for PH-sLD8 or older SoCs.
411 */
412 if (revision <= 0x16)
413 data->range_op_max_size = (u32)1 << 22;
414
415 /*
416 * Unfortunatly, the offset address of active way control base
417 * varies from SoC to SoC.
418 */
419 switch (revision) {
420 case 0x11: /* sLD3 */
421 data->way_ctrl_base = data->ctrl_base + 0x870;
422 break;
423 case 0x12: /* LD4 */
424 case 0x16: /* sld8 */
425 data->way_ctrl_base = data->ctrl_base + 0x840;
426 break;
427 default:
428 break;
429 }
430 }
431
432 data->range_op_max_size -= data->line_size;
433
434 INIT_LIST_HEAD(&data->list);
435 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
436
437 /*
438 * OK, this level has been successfully initialized. Look for the next
439 * level cache. Do not roll back even if the initialization of the
440 * next level cache fails because we want to continue with available
441 * cache levels.
442 */
443 next_np = of_find_next_cache_node(np);
444 if (next_np) {
445 (*cache_level)++;
446 ret = __uniphier_cache_init(next_np, cache_level);
447 }
448 of_node_put(next_np);
449
450 return ret;
451 err:
452 iounmap(data->op_base);
453 iounmap(data->rev_base);
454 iounmap(data->ctrl_base);
455 kfree(data);
456
457 return ret;
458 }
459
uniphier_cache_init(void)460 int __init uniphier_cache_init(void)
461 {
462 struct device_node *np = NULL;
463 unsigned int cache_level;
464 int ret = 0;
465
466 /* look for level 2 cache */
467 while ((np = of_find_matching_node(np, uniphier_cache_match)))
468 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
469 cache_level == 2)
470 break;
471
472 if (!np)
473 return -ENODEV;
474
475 ret = __uniphier_cache_init(np, &cache_level);
476 of_node_put(np);
477
478 if (ret) {
479 /*
480 * Error out iif L2 initialization fails. Continue with any
481 * error on L3 or outer because they are optional.
482 */
483 if (cache_level == 2) {
484 pr_err("failed to initialize L2 cache\n");
485 return ret;
486 }
487
488 cache_level--;
489 ret = 0;
490 }
491
492 outer_cache.inv_range = uniphier_cache_inv_range;
493 outer_cache.clean_range = uniphier_cache_clean_range;
494 outer_cache.flush_range = uniphier_cache_flush_range;
495 outer_cache.flush_all = uniphier_cache_flush_all;
496 outer_cache.disable = uniphier_cache_disable;
497 outer_cache.sync = uniphier_cache_sync;
498
499 uniphier_cache_enable();
500
501 pr_info("enabled outer cache (cache level: %d)\n", cache_level);
502
503 return ret;
504 }
505
