1 /*
2 * Copyright (c) 2024 Andes Technology Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/init.h>
8 #include <zephyr/kernel.h>
9 #include <zephyr/arch/riscv/csr.h>
10 #include <zephyr/drivers/cache.h>
11 #include <zephyr/logging/log.h>
12 #include <andes_csr.h>
13
14 LOG_MODULE_REGISTER(cache_andes, CONFIG_CACHE_LOG_LEVEL);
15
16 /* L1 CCTL Command */
17 #define CCTL_L1D_VA_INVAL 0
18 #define CCTL_L1D_VA_WB 1
19 #define CCTL_L1D_VA_WBINVAL 2
20 #define CCTL_L1D_WBINVAL_ALL 6
21 #define CCTL_L1D_WB_ALL 7
22 #define CCTL_L1I_VA_INVAL 8
23 #define CCTL_L1D_INVAL_ALL 23
24 #define CCTL_L1I_IX_INVAL 24
25
26 /* mcache_ctl bitfield */
27 #define MCACHE_CTL_IC_EN BIT(0)
28 #define MCACHE_CTL_DC_EN BIT(1)
29 #define MCACHE_CTL_CCTL_SUEN BIT(8)
30 #define MCACHE_CTL_DC_COHEN BIT(19)
31 #define MCACHE_CTL_DC_COHSTA BIT(20)
32
33 /* micm_cfg bitfield */
34 #define MICM_CFG_ISET BIT_MASK(3)
35 #define MICM_CFG_IWAY_SHIFT 3
36 #define MICM_CFG_ISZ_SHIFT 6
37
38 /* mdcm_cfg bitfield */
39 #define MDCM_CFG_DSZ_SHIFT 6
40
41 /* mmsc_cfg bitfield */
42 #define MMSC_CFG_CCTLCSR BIT(16)
43 #define MMSC_CFG_VCCTL_2 BIT(19)
44 #define MMSC_CFG_MSC_EXT BIT(31)
45 #define MMSC_CFG_RVARCH BIT64(52)
46
47 /* mmsc_cfg2 bitfield */
48 #define MMSC_CFG2_RVARCH BIT(20)
49
50 /* mrvarch_cfg bitfield */
51 #define MRVARCH_CFG_SMEPMP BIT(4)
52
53 #define K_CACHE_WB BIT(0)
54 #define K_CACHE_INVD BIT(1)
55 #define K_CACHE_WB_INVD (K_CACHE_WB | K_CACHE_INVD)
56
57 struct cache_config {
58 uint32_t instr_line_size;
59 uint32_t data_line_size;
60 uint32_t l2_cache_size;
61 uint32_t l2_cache_inclusive;
62 bool is_cctl_supported;
63 };
64
65 static struct cache_config cache_cfg;
66 static struct k_spinlock lock;
67
68 #if DT_NODE_HAS_COMPAT_STATUS(DT_INST(0, andestech_l2c), andestech_l2c, okay)
69 #include "cache_andes_l2.h"
70 #else
nds_l2_cache_enable(void)71 static ALWAYS_INLINE void nds_l2_cache_enable(void) { }
nds_l2_cache_disable(void)72 static ALWAYS_INLINE void nds_l2_cache_disable(void) { }
nds_l2_cache_range(void * addr,size_t size,int op)73 static ALWAYS_INLINE int nds_l2_cache_range(void *addr, size_t size, int op) { return 0; }
nds_l2_cache_all(int op)74 static ALWAYS_INLINE int nds_l2_cache_all(int op) { return 0; }
nds_l2_cache_is_inclusive(void)75 static ALWAYS_INLINE int nds_l2_cache_is_inclusive(void) { return 0; }
nds_l2_cache_init(uint8_t line_size)76 static ALWAYS_INLINE int nds_l2_cache_init(uint8_t line_size) { return 0; }
77 #endif /* DT_NODE_HAS_COMPAT_STATUS(DT_INST(0, andestech_l2c), andestech_l2c, okay) */
78
nds_cctl_range_operations(void * addr,size_t size,int line_size,int cmd)79 static ALWAYS_INLINE int nds_cctl_range_operations(void *addr, size_t size, int line_size, int cmd)
80 {
81 unsigned long last_byte, align_addr;
82 unsigned long status = csr_read(mstatus);
83
84 last_byte = (unsigned long)addr + size - 1;
85 align_addr = ROUND_DOWN(addr, line_size);
86
87 /*
88 * In memory access privilige U mode, applications should use ucctl CSRs
89 * for VA type commands.
90 */
91 if ((status & MSTATUS_MPRV) && !(status & MSTATUS_MPP)) {
92 while (align_addr <= last_byte) {
93 csr_write(NDS_UCCTLBEGINADDR, align_addr);
94 csr_write(NDS_UCCTLCOMMAND, cmd);
95 align_addr += line_size;
96 }
97 } else {
98 while (align_addr <= last_byte) {
99 csr_write(NDS_MCCTLBEGINADDR, align_addr);
100 csr_write(NDS_MCCTLCOMMAND, cmd);
101 align_addr += line_size;
102 }
103 }
104
105 return 0;
106 }
107
nds_l1i_cache_all(int op)108 static ALWAYS_INLINE int nds_l1i_cache_all(int op)
109 {
110 unsigned long sets, ways, end;
111 unsigned long status = csr_read(mstatus);
112
113 if (csr_read(NDS_MMSC_CFG) & MMSC_CFG_VCCTL_2) {
114 /*
115 * In memory access privilige U mode, applications can only use
116 * VA type commands for specific range.
117 */
118 if ((status & MSTATUS_MPRV) && !(status & MSTATUS_MPP)) {
119 return -ENOTSUP;
120 }
121 }
122
123 if (op == K_CACHE_INVD) {
124 sets = 0x40 << (csr_read(NDS_MICM_CFG) & MICM_CFG_ISET);
125 ways = ((csr_read(NDS_MICM_CFG) >> MICM_CFG_IWAY_SHIFT) & BIT_MASK(3)) + 1;
126 end = ways * sets * cache_cfg.instr_line_size;
127
128 for (int i = 0; i < end; i += cache_cfg.instr_line_size) {
129 csr_write(NDS_MCCTLBEGINADDR, i);
130 csr_write(NDS_MCCTLCOMMAND, CCTL_L1I_IX_INVAL);
131 }
132 }
133
134 return 0;
135 }
136
nds_l1d_cache_all(int op)137 static ALWAYS_INLINE int nds_l1d_cache_all(int op)
138 {
139 unsigned long status = csr_read(mstatus);
140
141 if (csr_read(NDS_MMSC_CFG) & MMSC_CFG_VCCTL_2) {
142 /*
143 * In memory access privilige U mode, applications can only use
144 * VA type commands for specific range.
145 */
146 if ((status & MSTATUS_MPRV) && !(status & MSTATUS_MPP)) {
147 return -ENOTSUP;
148 }
149 }
150
151 switch (op) {
152 case K_CACHE_WB:
153 csr_write(NDS_MCCTLCOMMAND, CCTL_L1D_WB_ALL);
154 break;
155 case K_CACHE_INVD:
156 csr_write(NDS_MCCTLCOMMAND, CCTL_L1D_INVAL_ALL);
157 break;
158 case K_CACHE_WB_INVD:
159 csr_write(NDS_MCCTLCOMMAND, CCTL_L1D_WBINVAL_ALL);
160 break;
161 default:
162 return -ENOTSUP;
163 }
164
165 return 0;
166 }
167
nds_l1i_cache_range(void * addr,size_t size,int op)168 static ALWAYS_INLINE int nds_l1i_cache_range(void *addr, size_t size, int op)
169 {
170 unsigned long cmd;
171
172 if (op == K_CACHE_INVD) {
173 cmd = CCTL_L1I_VA_INVAL;
174 nds_cctl_range_operations(addr, size, cache_cfg.instr_line_size, cmd);
175 }
176
177 return 0;
178 }
179
nds_l1d_cache_range(void * addr,size_t size,int op)180 static ALWAYS_INLINE int nds_l1d_cache_range(void *addr, size_t size, int op)
181 {
182 unsigned long cmd;
183
184 switch (op) {
185 case K_CACHE_WB:
186 cmd = CCTL_L1D_VA_WB;
187 break;
188 case K_CACHE_INVD:
189 cmd = CCTL_L1D_VA_INVAL;
190 break;
191 case K_CACHE_WB_INVD:
192 cmd = CCTL_L1D_VA_WBINVAL;
193 break;
194 default:
195 return -ENOTSUP;
196 }
197
198 nds_cctl_range_operations(addr, size, cache_cfg.data_line_size, cmd);
199
200 return 0;
201 }
202
cache_data_enable(void)203 void cache_data_enable(void)
204 {
205 if (IS_ENABLED(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1)) {
206 return;
207 }
208
209 K_SPINLOCK(&lock) {
210 nds_l2_cache_enable();
211
212 /* Enable D-cache coherence management */
213 csr_set(NDS_MCACHE_CTL, MCACHE_CTL_DC_COHEN);
214
215 /* Check if CPU support CM or not. */
216 if (csr_read(NDS_MCACHE_CTL) & MCACHE_CTL_DC_COHEN) {
217 /* Wait for cache coherence enabling completed */
218 while (!(csr_read(NDS_MCACHE_CTL) & MCACHE_CTL_DC_COHSTA)) {
219 ;
220 }
221 }
222
223 /* Enable D-cache */
224 csr_set(NDS_MCACHE_CTL, MCACHE_CTL_DC_EN);
225 }
226 }
227
cache_data_disable(void)228 void cache_data_disable(void)
229 {
230 unsigned long status = csr_read(mstatus);
231
232 if (IS_ENABLED(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1)) {
233 return;
234 }
235
236 if (csr_read(NDS_MMSC_CFG) & MMSC_CFG_VCCTL_2) {
237 if ((status & MSTATUS_MPRV) && !(status & MSTATUS_MPP)) {
238 if (!cache_cfg.l2_cache_inclusive) {
239 return;
240 }
241 }
242 }
243
244 K_SPINLOCK(&lock) {
245 if (cache_cfg.l2_cache_inclusive) {
246 nds_l2_cache_all(K_CACHE_WB_INVD);
247 } else {
248 nds_l1d_cache_all(K_CACHE_WB_INVD);
249 nds_l2_cache_all(K_CACHE_WB_INVD);
250 }
251
252 csr_clear(NDS_MCACHE_CTL, MCACHE_CTL_DC_EN);
253
254 /* Check if CPU support CM or not. */
255 if (csr_read(NDS_MCACHE_CTL) & MCACHE_CTL_DC_COHSTA) {
256 csr_clear(NDS_MCACHE_CTL, MCACHE_CTL_DC_COHEN);
257 /* Wait for cache coherence disabling completed */
258 while (csr_read(NDS_MCACHE_CTL) & MCACHE_CTL_DC_COHSTA) {
259 ;
260 }
261 }
262 nds_l2_cache_disable();
263 }
264 }
265
cache_instr_enable(void)266 void cache_instr_enable(void)
267 {
268 if (IS_ENABLED(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1)) {
269 return;
270 }
271
272 csr_set(NDS_MCACHE_CTL, MCACHE_CTL_IC_EN);
273 }
274
cache_instr_disable(void)275 void cache_instr_disable(void)
276 {
277 if (IS_ENABLED(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1)) {
278 return;
279 }
280
281 csr_clear(NDS_MCACHE_CTL, MCACHE_CTL_IC_EN);
282 }
283
cache_data_invd_all(void)284 int cache_data_invd_all(void)
285 {
286 unsigned long ret = 0;
287
288 if (!cache_cfg.is_cctl_supported) {
289 return -ENOTSUP;
290 }
291
292 K_SPINLOCK(&lock) {
293 if (cache_cfg.l2_cache_inclusive) {
294 ret |= nds_l2_cache_all(K_CACHE_WB);
295 ret |= nds_l2_cache_all(K_CACHE_INVD);
296 } else {
297 ret |= nds_l1d_cache_all(K_CACHE_WB);
298 ret |= nds_l2_cache_all(K_CACHE_WB);
299 ret |= nds_l2_cache_all(K_CACHE_INVD);
300 ret |= nds_l1d_cache_all(K_CACHE_INVD);
301 }
302 }
303
304 return ret;
305 }
306
cache_data_invd_range(void * addr,size_t size)307 int cache_data_invd_range(void *addr, size_t size)
308 {
309 unsigned long ret = 0;
310
311 if (!cache_cfg.is_cctl_supported) {
312 return -ENOTSUP;
313 }
314
315 K_SPINLOCK(&lock) {
316 if (cache_cfg.l2_cache_inclusive) {
317 ret |= nds_l2_cache_range(addr, size, K_CACHE_INVD);
318 } else {
319 ret |= nds_l2_cache_range(addr, size, K_CACHE_INVD);
320 ret |= nds_l1d_cache_range(addr, size, K_CACHE_INVD);
321 }
322 }
323
324 return ret;
325 }
326
cache_instr_invd_all(void)327 int cache_instr_invd_all(void)
328 {
329 unsigned long ret = 0;
330
331 if (!cache_cfg.is_cctl_supported) {
332 return -ENOTSUP;
333 }
334
335 if (IS_ENABLED(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1)) {
336 return -ENOTSUP;
337 }
338
339 if (IS_ENABLED(CONFIG_RISCV_PMP)) {
340 /* CCTL IX type command is not to RISC-V Smepmp */
341 if (IS_ENABLED(CONFIG_64BIT)) {
342 if (csr_read(NDS_MMSC_CFG) & MMSC_CFG_RVARCH) {
343 if (csr_read(NDS_MRVARCH_CFG) & MRVARCH_CFG_SMEPMP) {
344 return -ENOTSUP;
345 }
346 }
347 } else {
348 if ((csr_read(NDS_MMSC_CFG) & MMSC_CFG_MSC_EXT) &&
349 (csr_read(NDS_MMSC_CFG2) & MMSC_CFG2_RVARCH)) {
350 if (csr_read(NDS_MRVARCH_CFG) & MRVARCH_CFG_SMEPMP) {
351 return -ENOTSUP;
352 }
353 }
354 }
355 }
356
357 K_SPINLOCK(&lock) {
358 ret |= nds_l1i_cache_all(K_CACHE_INVD);
359 }
360
361 return ret;
362 }
363
cache_instr_invd_range(void * addr,size_t size)364 int cache_instr_invd_range(void *addr, size_t size)
365 {
366 unsigned long ret = 0;
367
368 if (!cache_cfg.is_cctl_supported) {
369 return -ENOTSUP;
370 }
371
372 if (IS_ENABLED(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1)) {
373 ARG_UNUSED(addr);
374 ARG_UNUSED(size);
375
376 return -ENOTSUP;
377 }
378
379 K_SPINLOCK(&lock) {
380 ret |= nds_l1i_cache_range(addr, size, K_CACHE_INVD);
381 }
382
383 return ret;
384 }
385
cache_data_flush_all(void)386 int cache_data_flush_all(void)
387 {
388 unsigned long ret = 0;
389
390 if (!cache_cfg.is_cctl_supported) {
391 return -ENOTSUP;
392 }
393
394 K_SPINLOCK(&lock) {
395 if (cache_cfg.l2_cache_inclusive) {
396 ret |= nds_l2_cache_all(K_CACHE_WB);
397 } else {
398 ret |= nds_l1d_cache_all(K_CACHE_WB);
399 ret |= nds_l2_cache_all(K_CACHE_WB);
400 }
401 }
402
403 return ret;
404 }
405
cache_data_flush_range(void * addr,size_t size)406 int cache_data_flush_range(void *addr, size_t size)
407 {
408 unsigned long ret = 0;
409
410 if (!cache_cfg.is_cctl_supported) {
411 return -ENOTSUP;
412 }
413
414 K_SPINLOCK(&lock) {
415 if (cache_cfg.l2_cache_inclusive) {
416 ret |= nds_l2_cache_range(addr, size, K_CACHE_WB);
417 } else {
418 ret |= nds_l1d_cache_range(addr, size, K_CACHE_WB);
419 ret |= nds_l2_cache_range(addr, size, K_CACHE_WB);
420 }
421 }
422
423 return ret;
424 }
425
cache_data_flush_and_invd_all(void)426 int cache_data_flush_and_invd_all(void)
427 {
428 unsigned long ret = 0;
429
430 if (!cache_cfg.is_cctl_supported) {
431 return -ENOTSUP;
432 }
433
434 K_SPINLOCK(&lock) {
435 if (cache_cfg.l2_cache_size) {
436 if (cache_cfg.l2_cache_inclusive) {
437 ret |= nds_l2_cache_all(K_CACHE_WB_INVD);
438 } else {
439 ret |= nds_l1d_cache_all(K_CACHE_WB);
440 ret |= nds_l2_cache_all(K_CACHE_WB_INVD);
441 ret |= nds_l1d_cache_all(K_CACHE_INVD);
442 }
443 } else {
444 ret |= nds_l1d_cache_all(K_CACHE_WB_INVD);
445 }
446 }
447
448 return ret;
449 }
450
cache_data_flush_and_invd_range(void * addr,size_t size)451 int cache_data_flush_and_invd_range(void *addr, size_t size)
452 {
453 unsigned long ret = 0;
454
455 if (!cache_cfg.is_cctl_supported) {
456 return -ENOTSUP;
457 }
458
459 K_SPINLOCK(&lock) {
460 if (cache_cfg.l2_cache_size) {
461 if (cache_cfg.l2_cache_inclusive) {
462 ret |= nds_l2_cache_range(addr, size, K_CACHE_WB_INVD);
463 } else {
464 ret |= nds_l1d_cache_range(addr, size, K_CACHE_WB);
465 ret |= nds_l2_cache_range(addr, size, K_CACHE_WB_INVD);
466 ret |= nds_l1d_cache_range(addr, size, K_CACHE_INVD);
467 }
468 } else {
469 ret |= nds_l1d_cache_range(addr, size, K_CACHE_WB_INVD);
470 }
471 }
472
473 return ret;
474 }
475
cache_instr_flush_all(void)476 int cache_instr_flush_all(void)
477 {
478 return -ENOTSUP;
479 }
480
cache_instr_flush_and_invd_all(void)481 int cache_instr_flush_and_invd_all(void)
482 {
483 return -ENOTSUP;
484 }
485
cache_instr_flush_range(void * addr,size_t size)486 int cache_instr_flush_range(void *addr, size_t size)
487 {
488 ARG_UNUSED(addr);
489 ARG_UNUSED(size);
490
491 return -ENOTSUP;
492 }
493
cache_instr_flush_and_invd_range(void * addr,size_t size)494 int cache_instr_flush_and_invd_range(void *addr, size_t size)
495 {
496 ARG_UNUSED(addr);
497 ARG_UNUSED(size);
498
499 return -ENOTSUP;
500 }
501
502 #if defined(CONFIG_DCACHE_LINE_SIZE_DETECT)
cache_data_line_size_get(void)503 size_t cache_data_line_size_get(void)
504 {
505 return cache_cfg.data_line_size;
506 }
507 #endif /* defined(CONFIG_DCACHE_LINE_SIZE_DETECT) */
508
509 #if defined(CONFIG_ICACHE_LINE_SIZE_DETECT)
cache_instr_line_size_get(void)510 size_t cache_instr_line_size_get(void)
511 {
512 return cache_cfg.instr_line_size;
513 }
514 #endif /* defined(CONFIG_ICACHE_LINE_SIZE_DETECT) */
515
andes_cache_init(void)516 static int andes_cache_init(void)
517 {
518 unsigned long line_size;
519
520 if (IS_ENABLED(CONFIG_ICACHE)) {
521 line_size = (csr_read(NDS_MICM_CFG) >> MICM_CFG_ISZ_SHIFT) & BIT_MASK(3);
522
523 if (line_size == 0) {
524 LOG_ERR("Platform doesn't support I-cache, "
525 "please disable CONFIG_ICACHE");
526 }
527 #if defined(CONFIG_ICACHE_LINE_SIZE_DETECT)
528 /* Icache line size */
529 if (line_size <= 5) {
530 cache_cfg.instr_line_size = 1 << (line_size + 2);
531 } else {
532 LOG_ERR("Unknown line size of I-cache");
533 }
534 #elif (CONFIG_ICACHE_LINE_SIZE != 0)
535 cache_cfg.instr_line_size = CONFIG_ICACHE_LINE_SIZE;
536 #else
537 LOG_ERR("Please specific the i-cache-line-size "
538 "CPU0 property of the DT");
539 #endif /* defined(CONFIG_ICACHE_LINE_SIZE_DETECT) */
540 }
541
542 if (IS_ENABLED(CONFIG_DCACHE)) {
543 line_size = (csr_read(NDS_MDCM_CFG) >> MDCM_CFG_DSZ_SHIFT) & BIT_MASK(3);
544 if (line_size == 0) {
545 LOG_ERR("Platform doesn't support D-cache, "
546 "please disable CONFIG_DCACHE");
547 }
548 #if defined(CONFIG_DCACHE_LINE_SIZE_DETECT)
549 /* Dcache line size */
550 if (line_size <= 5) {
551 cache_cfg.data_line_size = 1 << (line_size + 2);
552 } else {
553 LOG_ERR("Unknown line size of D-cache");
554 }
555 #elif (CONFIG_DCACHE_LINE_SIZE != 0)
556 cache_cfg.data_line_size = CONFIG_DCACHE_LINE_SIZE;
557 #else
558 LOG_ERR("Please specific the d-cache-line-size "
559 "CPU0 property of the DT");
560 #endif /* defined(CONFIG_DCACHE_LINE_SIZE_DETECT) */
561 }
562
563 if (csr_read(NDS_MMSC_CFG) & MMSC_CFG_CCTLCSR) {
564 cache_cfg.is_cctl_supported = true;
565 }
566
567 if (csr_read(NDS_MMSC_CFG) & MMSC_CFG_VCCTL_2) {
568 if (IS_ENABLED(CONFIG_PMP_STACK_GUARD)) {
569 csr_set(NDS_MCACHE_CTL, MCACHE_CTL_CCTL_SUEN);
570 }
571 }
572
573 cache_cfg.l2_cache_size = nds_l2_cache_init(cache_cfg.data_line_size);
574 cache_cfg.l2_cache_inclusive = nds_l2_cache_is_inclusive();
575
576 return 0;
577 }
578
579 SYS_INIT(andes_cache_init, PRE_KERNEL_1, CONFIG_CACHE_ANDES_INIT_PRIORITY);
580
