1 /*
2 * Copyright (c) 2019 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/device.h>
8 #include <xtensa/xtruntime.h>
9 #include <zephyr/irq_nextlevel.h>
10 #include <xtensa/hal.h>
11 #include <zephyr/init.h>
12 #include <zephyr/kernel.h>
13 #include <zephyr/pm/pm.h>
14 #include <zephyr/device.h>
15 #include <zephyr/cache.h>
16 #include <cpu_init.h>
17
18 #include <adsp_memory.h>
19 #include <adsp_shim.h>
20 #include <adsp_clk.h>
21 #include <adsp_imr_layout.h>
22 #include <cavs-idc.h>
23
24 #ifdef CONFIG_DYNAMIC_INTERRUPTS
25 #include <zephyr/sw_isr_table.h>
26 #endif
27
28 #define LOG_LEVEL CONFIG_SOC_LOG_LEVEL
29 #include <zephyr/logging/log.h>
30 LOG_MODULE_REGISTER(soc);
31
32 # define SHIM_GPDMA_BASE_OFFSET 0x6500
33 # define SHIM_GPDMA_BASE(x) (SHIM_GPDMA_BASE_OFFSET + (x) * 0x100)
34 # define SHIM_GPDMA_CLKCTL(x) (SHIM_GPDMA_BASE(x) + 0x4)
35 # define SHIM_CLKCTL_LPGPDMAFDCGB BIT(0)
36
37 #ifdef CONFIG_PM
38 #define SRAM_ALIAS_BASE 0x9E000000
39 #define SRAM_ALIAS_MASK 0xFF000000
40 #define SRAM_ALIAS_OFFSET 0x20000000
41
42 #define L2_INTERRUPT_NUMBER 4
43 #define L2_INTERRUPT_MASK (1<<L2_INTERRUPT_NUMBER)
44
45 #define L3_INTERRUPT_NUMBER 6
46 #define L3_INTERRUPT_MASK (1<<L3_INTERRUPT_NUMBER)
47
48 #define ALL_USED_INT_LEVELS_MASK (L2_INTERRUPT_MASK | L3_INTERRUPT_MASK)
49
50 /*
51 * @biref FW entry point called by ROM during normal boot flow
52 */
53 extern void rom_entry(void);
54 void mp_resume_entry(void);
55
56 struct core_state {
57 uint32_t a0;
58 uint32_t a1;
59 uint32_t excsave2;
60 uint32_t intenable;
61 uint32_t ps;
62 };
63
64 static struct core_state core_desc[CONFIG_MP_MAX_NUM_CPUS] = {{0}};
65
66 /**
67 * @brief Power down procedure.
68 *
69 * Locks its code in L1 cache and shuts down memories.
70 * NOTE: there's no return from this function.
71 *
72 * @param disable_lpsram flag if LPSRAM is to be disabled (whole)
73 * @param hpsram_pg_mask pointer to memory segments power gating mask
74 * (each bit corresponds to one ebb)
75 */
76 extern void power_down_cavs(bool disable_lpsram, uint32_t __sparse_cache * hpsram_pg_mask);
77
uncache_to_cache(void * address)78 static inline void __sparse_cache *uncache_to_cache(void *address)
79 {
80 return (void __sparse_cache *)((uintptr_t)(address) | SRAM_ALIAS_OFFSET);
81 }
82
_save_core_context(void)83 static ALWAYS_INLINE void _save_core_context(void)
84 {
85 uint32_t core_id = arch_proc_id();
86
87 core_desc[core_id].ps = XTENSA_RSR("PS");
88 core_desc[core_id].excsave2 = XTENSA_RSR(ZSR_CPU_STR);
89 __asm__ volatile("mov %0, a0" : "=r"(core_desc[core_id].a0));
90 __asm__ volatile("mov %0, a1" : "=r"(core_desc[core_id].a1));
91 sys_cache_data_flush_range(&core_desc[core_id], sizeof(struct core_state));
92 }
93
_restore_core_context(void)94 static ALWAYS_INLINE void _restore_core_context(void)
95 {
96 uint32_t core_id = arch_proc_id();
97
98 XTENSA_WSR("PS", core_desc[core_id].ps);
99 XTENSA_WSR(ZSR_CPU_STR, core_desc[core_id].excsave2);
100 __asm__ volatile("mov a0, %0" :: "r"(core_desc[core_id].a0));
101 __asm__ volatile("mov a1, %0" :: "r"(core_desc[core_id].a1));
102 __asm__ volatile("rsync");
103 }
104
power_gate_exit(void)105 static void __used power_gate_exit(void)
106 {
107 cpu_early_init();
108 sys_cache_data_flush_and_invd_all();
109 _restore_core_context();
110
111 /* Secondary core is resumed by set_dx */
112 if (arch_proc_id()) {
113 mp_resume_entry();
114 }
115 }
116
117 __asm__(".align 4\n\t"
118 ".global dsp_restore_vector\n\t"
119 "dsp_restore_vector:\n\t"
120 " movi a0, 0\n\t"
121 " movi a1, 1\n\t"
122 " movi a2, 0x40020\n\t"/* PS_UM|PS_WOE */
123 " wsr a2, PS\n\t"
124 " wsr a1, WINDOWSTART\n\t"
125 " wsr a0, WINDOWBASE\n\t"
126 " rsync\n\t"
127 " movi a1, z_interrupt_stacks\n\t"
128 " rsr a2, PRID\n\t"
129 " movi a3, " STRINGIFY(CONFIG_ISR_STACK_SIZE) "\n\t"
130 " mull a2, a2, a3\n\t"
131 " add a2, a2, a3\n\t"
132 " add a1, a1, a2\n\t"
133 " call0 power_gate_exit\n\t");
134
pm_state_set(enum pm_state state,uint8_t substate_id)135 void pm_state_set(enum pm_state state, uint8_t substate_id)
136 {
137 ARG_UNUSED(substate_id);
138 uint32_t cpu = arch_proc_id();
139
140 if (state == PM_STATE_SOFT_OFF) {
141 core_desc[cpu].intenable = XTENSA_RSR("INTENABLE");
142 z_xt_ints_off(0xffffffff);
143 xthal_window_spill();
144 _save_core_context();
145 soc_cpus_active[cpu] = false;
146 sys_cache_data_flush_and_invd_all();
147 if (cpu == 0) {
148 uint32_t hpsram_mask[HPSRAM_SEGMENTS] = {0};
149
150 struct imr_header hdr = {
151 .adsp_imr_magic = ADSP_IMR_MAGIC_VALUE,
152 .imr_restore_vector = rom_entry,
153 };
154 struct imr_layout *imr_layout =
155 sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)
156 L3_MEM_BASE_ADDR);
157
158 imr_layout->imr_state.header = hdr;
159
160 #ifdef CONFIG_ADSP_POWER_DOWN_HPSRAM
161 /* turn off all HPSRAM banks - get a full bitmap */
162 for (int i = 0; i < HPSRAM_SEGMENTS; i++) {
163 hpsram_mask[i] = HPSRAM_MEMMASK(i);
164 }
165 #endif /* CONFIG_ADSP_POWER_DOWN_HPSRAM */
166 /* do power down - this function won't return */
167 power_down_cavs(true, uncache_to_cache(&hpsram_mask[0]));
168 } else {
169 k_cpu_atomic_idle(arch_irq_lock());
170 }
171 } else {
172 __ASSERT(false, "invalid argument - unsupported power state");
173 }
174 }
175
176 /* Handle SOC specific activity after Low Power Mode Exit */
pm_state_exit_post_ops(enum pm_state state,uint8_t substate_id)177 void pm_state_exit_post_ops(enum pm_state state, uint8_t substate_id)
178 {
179 ARG_UNUSED(substate_id);
180 uint32_t cpu = arch_proc_id();
181
182 if (state == PM_STATE_SOFT_OFF) {
183 soc_cpus_active[cpu] = true;
184 sys_cache_data_flush_and_invd_all();
185 z_xt_ints_on(core_desc[cpu].intenable);
186 } else {
187 __ASSERT(false, "invalid argument - unsupported power state");
188 }
189
190 /**
191 * We don't have the key used to lock interruptions here.
192 * Just set PS.INTLEVEL to 0.
193 */
194 __asm__ volatile ("rsil a2, 0");
195 }
196 #endif /* CONFIG_PM */
197
198 #ifdef CONFIG_ARCH_HAS_CUSTOM_CPU_IDLE
199 /* xt-clang removes any NOPs more than 8. So we need to set
200 * no optimization to avoid those NOPs from being removed.
201 *
202 * This function is simply enough and full of hand written
203 * assembly that optimization is not really meaningful
204 * anyway. So we can skip optimization unconditionally.
205 * Re-evalulate its use and add #ifdef if this assumption
206 * is no longer valid.
207 */
208 __no_optimization
arch_cpu_idle(void)209 void arch_cpu_idle(void)
210 {
211 sys_trace_idle();
212
213 /* Just spin forever with interrupts unmasked, for platforms
214 * where WAITI can't be used or where its behavior is
215 * complicated (Intel DSPs will power gate on idle entry under
216 * some circumstances)
217 */
218 if (IS_ENABLED(CONFIG_XTENSA_CPU_IDLE_SPIN)) {
219 __asm__ volatile("rsil a0, 0");
220 __asm__ volatile("loop_forever: j loop_forever");
221 return;
222 }
223
224 /* Cribbed from SOF: workaround for a bug in some versions of
225 * the LX6 IP. Preprocessor ugliness avoids the need to
226 * figure out how to get the compiler to unroll a loop.
227 */
228 if (IS_ENABLED(CONFIG_XTENSA_WAITI_BUG)) {
229 #define NOP4 __asm__ volatile("nop; nop; nop; nop");
230 #define NOP32 NOP4 NOP4 NOP4 NOP4 NOP4 NOP4 NOP4 NOP4
231 #define NOP128() NOP32 NOP32 NOP32 NOP32
232 NOP128();
233 #undef NOP128
234 #undef NOP32
235 #undef NOP4
236 __asm__ volatile("isync; extw");
237 }
238
239 __asm__ volatile ("waiti 0");
240 }
241 #endif
242
power_init(void)243 __imr void power_init(void)
244 {
245 /* Request HP ring oscillator and
246 * wait for status to indicate it's ready.
247 */
248 CAVS_SHIM.clkctl |= CAVS_CLKCTL_RHROSCC;
249 while ((CAVS_SHIM.clkctl & CAVS_CLKCTL_RHROSCC) != CAVS_CLKCTL_RHROSCC) {
250 k_busy_wait(10);
251 }
252
253 /* Request HP Ring Oscillator
254 * Select HP Ring Oscillator
255 * High Power Domain PLL Clock Select device by 2
256 * Low Power Domain PLL Clock Select device by 4
257 * Disable Tensilica Core(s) Prevent Local Clock Gating
258 * - Disabling "prevent clock gating" means allowing clock gating
259 */
260 CAVS_SHIM.clkctl = (CAVS_CLKCTL_RHROSCC |
261 CAVS_CLKCTL_OCS |
262 CAVS_CLKCTL_LMCS);
263
264 /* Prevent LP GPDMA 0 & 1 clock gating */
265 sys_write32(SHIM_CLKCTL_LPGPDMAFDCGB, SHIM_GPDMA_CLKCTL(0));
266 sys_write32(SHIM_CLKCTL_LPGPDMAFDCGB, SHIM_GPDMA_CLKCTL(1));
267
268 /* Disable power gating for first cores */
269 CAVS_SHIM.pwrctl |= CAVS_PWRCTL_TCPDSPPG(0);
270
271 /* On cAVS 1.8+, we must demand ownership of the timestamping
272 * and clock generator registers. Lacking the former will
273 * prevent wall clock timer interrupts from arriving, even
274 * though the device itself is operational.
275 */
276 sys_write32(GENO_MDIVOSEL | GENO_DIOPTOSEL, DSP_INIT_GENO);
277 sys_write32(IOPO_DMIC_FLAG | IOPO_I2SSEL_MASK, DSP_INIT_IOPO);
278 }
279
