1 /*
2 * SPDX-FileCopyrightText: 2016-2021 Espressif Systems (Shanghai) CO LTD
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <stdlib.h>
8 #include <stdbool.h>
9 #include <string.h>
10 #include <sys/param.h>
11
12 #include "esp_attr.h"
13 #include "esp_err.h"
14 #include "esp_pm.h"
15 #include "esp_log.h"
16
17 #include "esp_private/crosscore_int.h"
18
19 #include "soc/rtc.h"
20 #include "hal/cpu_hal.h"
21 #include "hal/uart_ll.h"
22 #include "hal/uart_types.h"
23
24 #include "freertos/FreeRTOS.h"
25 #include "freertos/task.h"
26 #if CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
27 #include "freertos/xtensa_timer.h"
28 #include "xtensa/core-macros.h"
29 #endif
30
31 #include "esp_private/pm_impl.h"
32 #include "esp_private/pm_trace.h"
33 #include "esp_private/esp_timer_private.h"
34
35 #include "esp_sleep.h"
36
37 #include "sdkconfig.h"
38
39 // [refactor-todo] opportunity for further refactor
40 #if CONFIG_IDF_TARGET_ESP32
41 #include "esp32/clk.h"
42 #include "esp32/pm.h"
43 #include "driver/gpio.h"
44 #elif CONFIG_IDF_TARGET_ESP32S2
45 #include "esp32s2/clk.h"
46 #include "esp32s2/pm.h"
47 #include "driver/gpio.h"
48 #elif CONFIG_IDF_TARGET_ESP32S3
49 #include "esp32s3/clk.h"
50 #include "esp32s3/pm.h"
51 #elif CONFIG_IDF_TARGET_ESP32C3
52 #include "esp32c3/clk.h"
53 #include "esp32c3/pm.h"
54 #include "driver/gpio.h"
55 #elif CONFIG_IDF_TARGET_ESP32H2
56 #include "esp32h2/clk.h"
57 #include "esp32h2/pm.h"
58 #include "driver/gpio.h"
59 #endif
60
61 #define MHZ (1000000)
62
63 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
64 /* CCOMPARE update timeout, in CPU cycles. Any value above ~600 cycles will work
65 * for the purpose of detecting a deadlock.
66 */
67 #define CCOMPARE_UPDATE_TIMEOUT 1000000
68
69 /* When changing CCOMPARE, don't allow changes if the difference is less
70 * than this. This is to prevent setting CCOMPARE below CCOUNT.
71 */
72 #define CCOMPARE_MIN_CYCLES_IN_FUTURE 1000
73 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
74
75 /* When light sleep is used, wake this number of microseconds earlier than
76 * the next tick.
77 */
78 #define LIGHT_SLEEP_EARLY_WAKEUP_US 100
79
80 #if CONFIG_IDF_TARGET_ESP32
81 /* Minimal divider at which REF_CLK_FREQ can be obtained */
82 #define REF_CLK_DIV_MIN 10
83 #define DEFAULT_CPU_FREQ CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ
84 #elif CONFIG_IDF_TARGET_ESP32S2
85 /* Minimal divider at which REF_CLK_FREQ can be obtained */
86 #define REF_CLK_DIV_MIN 2
87 #define DEFAULT_CPU_FREQ CONFIG_ESP32S2_DEFAULT_CPU_FREQ_MHZ
88 #elif CONFIG_IDF_TARGET_ESP32S3
89 /* Minimal divider at which REF_CLK_FREQ can be obtained */
90 #define REF_CLK_DIV_MIN 2
91 #define DEFAULT_CPU_FREQ CONFIG_ESP32S3_DEFAULT_CPU_FREQ_MHZ
92 #elif CONFIG_IDF_TARGET_ESP32C3
93 #define REF_CLK_DIV_MIN 2
94 #define DEFAULT_CPU_FREQ CONFIG_ESP32C3_DEFAULT_CPU_FREQ_MHZ
95 #elif CONFIG_IDF_TARGET_ESP32H2
96 #define REF_CLK_DIV_MIN 2
97 #define DEFAULT_CPU_FREQ CONFIG_ESP32H2_DEFAULT_CPU_FREQ_MHZ
98 #endif
99
100 #ifdef CONFIG_PM_PROFILING
101 #define WITH_PROFILING
102 #endif
103
104 static portMUX_TYPE s_switch_lock = portMUX_INITIALIZER_UNLOCKED;
105 /* The following state variables are protected using s_switch_lock: */
106 /* Current sleep mode; When switching, contains old mode until switch is complete */
107 static pm_mode_t s_mode = PM_MODE_CPU_MAX;
108 /* True when switch is in progress */
109 static volatile bool s_is_switching;
110 /* Number of times each mode was locked */
111 static size_t s_mode_lock_counts[PM_MODE_COUNT];
112 /* Bit mask of locked modes. BIT(i) is set iff s_mode_lock_counts[i] > 0. */
113 static uint32_t s_mode_mask;
114
115 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
116
117 #define PERIPH_SKIP_LIGHT_SLEEP_NO 1
118
119 /* Indicates if light sleep shoule be skipped by peripherals. */
120 static skip_light_sleep_cb_t s_periph_skip_light_sleep_cb[PERIPH_SKIP_LIGHT_SLEEP_NO];
121
122 /* Indicates if light sleep entry was skipped in vApplicationSleep for given CPU.
123 * This in turn gets used in IDLE hook to decide if `waiti` needs
124 * to be invoked or not.
125 */
126 static bool s_skipped_light_sleep[portNUM_PROCESSORS];
127
128 #if portNUM_PROCESSORS == 2
129 /* When light sleep is finished on one CPU, it is possible that the other CPU
130 * will enter light sleep again very soon, before interrupts on the first CPU
131 * get a chance to run. To avoid such situation, set a flag for the other CPU to
132 * skip light sleep attempt.
133 */
134 static bool s_skip_light_sleep[portNUM_PROCESSORS];
135 #endif // portNUM_PROCESSORS == 2
136 #endif // CONFIG_FREERTOS_USE_TICKLESS_IDLE
137
138 /* A flag indicating that Idle hook has run on a given CPU;
139 * Next interrupt on the same CPU will take s_rtos_lock_handle.
140 */
141 static bool s_core_idle[portNUM_PROCESSORS];
142
143 /* When no RTOS tasks are active, these locks are released to allow going into
144 * a lower power mode. Used by ISR hook and idle hook.
145 */
146 static esp_pm_lock_handle_t s_rtos_lock_handle[portNUM_PROCESSORS];
147
148 /* Lookup table of CPU frequency configs to be used in each mode.
149 * Initialized by esp_pm_impl_init and modified by esp_pm_configure.
150 */
151 static rtc_cpu_freq_config_t s_cpu_freq_by_mode[PM_MODE_COUNT];
152
153 /* Whether automatic light sleep is enabled */
154 static bool s_light_sleep_en = false;
155
156 /* When configuration is changed, current frequency may not match the
157 * newly configured frequency for the current mode. This is an indicator
158 * to the mode switch code to get the actual current frequency instead of
159 * relying on the current mode.
160 */
161 static bool s_config_changed = false;
162
163 #ifdef WITH_PROFILING
164 /* Time, in microseconds, spent so far in each mode */
165 static pm_time_t s_time_in_mode[PM_MODE_COUNT];
166 /* Timestamp, in microseconds, when the mode switch last happened */
167 static pm_time_t s_last_mode_change_time;
168 /* User-readable mode names, used by esp_pm_impl_dump_stats */
169 static const char* s_mode_names[] = {
170 "SLEEP",
171 "APB_MIN",
172 "APB_MAX",
173 "CPU_MAX"
174 };
175 #endif // WITH_PROFILING
176
177 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
178 /* Indicates to the ISR hook that CCOMPARE needs to be updated on the given CPU.
179 * Used in conjunction with cross-core interrupt to update CCOMPARE on the other CPU.
180 */
181 static volatile bool s_need_update_ccompare[portNUM_PROCESSORS];
182
183 /* Divider and multiplier used to adjust (ccompare - ccount) duration.
184 * Only set to non-zero values when switch is in progress.
185 */
186 static uint32_t s_ccount_div;
187 static uint32_t s_ccount_mul;
188
189 static void update_ccompare(void);
190 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
191
192 static const char* TAG = "pm";
193
194 static void do_switch(pm_mode_t new_mode);
195 static void leave_idle(void);
196 static void on_freq_update(uint32_t old_ticks_per_us, uint32_t ticks_per_us);
197 #if CONFIG_PM_SLP_DEFAULT_PARAMS_OPT
198 static void esp_pm_light_sleep_default_params_config(int min_freq_mhz, int max_freq_mhz);
199 #endif
200
esp_pm_impl_get_mode(esp_pm_lock_type_t type,int arg)201 pm_mode_t esp_pm_impl_get_mode(esp_pm_lock_type_t type, int arg)
202 {
203 (void) arg;
204 if (type == ESP_PM_CPU_FREQ_MAX) {
205 return PM_MODE_CPU_MAX;
206 } else if (type == ESP_PM_APB_FREQ_MAX) {
207 return PM_MODE_APB_MAX;
208 } else if (type == ESP_PM_NO_LIGHT_SLEEP) {
209 return PM_MODE_APB_MIN;
210 } else {
211 // unsupported mode
212 abort();
213 }
214 }
215
esp_pm_configure(const void * vconfig)216 esp_err_t esp_pm_configure(const void* vconfig)
217 {
218 #ifndef CONFIG_PM_ENABLE
219 return ESP_ERR_NOT_SUPPORTED;
220 #endif
221
222 #if CONFIG_IDF_TARGET_ESP32
223 const esp_pm_config_esp32_t* config = (const esp_pm_config_esp32_t*) vconfig;
224 #elif CONFIG_IDF_TARGET_ESP32S2
225 const esp_pm_config_esp32s2_t* config = (const esp_pm_config_esp32s2_t*) vconfig;
226 #elif CONFIG_IDF_TARGET_ESP32S3
227 const esp_pm_config_esp32s3_t* config = (const esp_pm_config_esp32s3_t*) vconfig;
228 #elif CONFIG_IDF_TARGET_ESP32C3
229 const esp_pm_config_esp32c3_t* config = (const esp_pm_config_esp32c3_t*) vconfig;
230 #elif CONFIG_IDF_TARGET_ESP32H2
231 const esp_pm_config_esp32h2_t* config = (const esp_pm_config_esp32h2_t*) vconfig;
232 #endif
233
234 #ifndef CONFIG_FREERTOS_USE_TICKLESS_IDLE
235 if (config->light_sleep_enable) {
236 return ESP_ERR_NOT_SUPPORTED;
237 }
238 #endif
239
240 int min_freq_mhz = config->min_freq_mhz;
241 int max_freq_mhz = config->max_freq_mhz;
242
243 if (min_freq_mhz > max_freq_mhz) {
244 return ESP_ERR_INVALID_ARG;
245 }
246
247 rtc_cpu_freq_config_t freq_config;
248 if (!rtc_clk_cpu_freq_mhz_to_config(min_freq_mhz, &freq_config)) {
249 ESP_LOGW(TAG, "invalid min_freq_mhz value (%d)", min_freq_mhz);
250 return ESP_ERR_INVALID_ARG;
251 }
252
253 int xtal_freq_mhz = (int) rtc_clk_xtal_freq_get();
254 if (min_freq_mhz < xtal_freq_mhz && min_freq_mhz * MHZ / REF_CLK_FREQ < REF_CLK_DIV_MIN) {
255 ESP_LOGW(TAG, "min_freq_mhz should be >= %d", REF_CLK_FREQ * REF_CLK_DIV_MIN / MHZ);
256 return ESP_ERR_INVALID_ARG;
257 }
258
259 if (!rtc_clk_cpu_freq_mhz_to_config(max_freq_mhz, &freq_config)) {
260 ESP_LOGW(TAG, "invalid max_freq_mhz value (%d)", max_freq_mhz);
261 return ESP_ERR_INVALID_ARG;
262 }
263
264 #if CONFIG_IDF_TARGET_ESP32
265 int apb_max_freq = max_freq_mhz; /* CPU frequency in APB_MAX mode */
266 if (max_freq_mhz == 240) {
267 /* We can't switch between 240 and 80/160 without disabling PLL,
268 * so use 240MHz CPU frequency when 80MHz APB frequency is requested.
269 */
270 apb_max_freq = 240;
271 } else if (max_freq_mhz == 160 || max_freq_mhz == 80) {
272 /* Otherwise, can use 80MHz
273 * CPU frequency when 80MHz APB frequency is requested.
274 */
275 apb_max_freq = 80;
276 }
277 #else
278 int apb_max_freq = MIN(max_freq_mhz, 80); /* CPU frequency in APB_MAX mode */
279 #endif
280
281 apb_max_freq = MAX(apb_max_freq, min_freq_mhz);
282
283 ESP_LOGI(TAG, "Frequency switching config: "
284 "CPU_MAX: %d, APB_MAX: %d, APB_MIN: %d, Light sleep: %s",
285 max_freq_mhz,
286 apb_max_freq,
287 min_freq_mhz,
288 config->light_sleep_enable ? "ENABLED" : "DISABLED");
289
290 portENTER_CRITICAL(&s_switch_lock);
291
292 bool res __attribute__((unused));
293 res = rtc_clk_cpu_freq_mhz_to_config(max_freq_mhz, &s_cpu_freq_by_mode[PM_MODE_CPU_MAX]);
294 assert(res);
295 res = rtc_clk_cpu_freq_mhz_to_config(apb_max_freq, &s_cpu_freq_by_mode[PM_MODE_APB_MAX]);
296 assert(res);
297 res = rtc_clk_cpu_freq_mhz_to_config(min_freq_mhz, &s_cpu_freq_by_mode[PM_MODE_APB_MIN]);
298 assert(res);
299 s_cpu_freq_by_mode[PM_MODE_LIGHT_SLEEP] = s_cpu_freq_by_mode[PM_MODE_APB_MIN];
300 s_light_sleep_en = config->light_sleep_enable;
301 s_config_changed = true;
302 portEXIT_CRITICAL(&s_switch_lock);
303
304 #if CONFIG_PM_SLP_DISABLE_GPIO && SOC_GPIO_SUPPORT_SLP_SWITCH
305 esp_sleep_enable_gpio_switch(config->light_sleep_enable);
306 #endif
307
308 #if CONFIG_PM_POWER_DOWN_CPU_IN_LIGHT_SLEEP && SOC_PM_SUPPORT_CPU_PD
309 esp_err_t ret = esp_sleep_cpu_pd_low_init(config->light_sleep_enable);
310 if (config->light_sleep_enable && ret != ESP_OK) {
311 ESP_LOGW(TAG, "Failed to enable CPU power down during light sleep.");
312 }
313 #endif
314
315 #if CONFIG_PM_SLP_DEFAULT_PARAMS_OPT
316 if (config->light_sleep_enable) {
317 esp_pm_light_sleep_default_params_config(min_freq_mhz, max_freq_mhz);
318 }
319 #endif
320
321 return ESP_OK;
322 }
323
esp_pm_get_configuration(void * vconfig)324 esp_err_t esp_pm_get_configuration(void* vconfig)
325 {
326 if (vconfig == NULL) {
327 return ESP_ERR_INVALID_ARG;
328 }
329
330 #if CONFIG_IDF_TARGET_ESP32
331 esp_pm_config_esp32_t* config = (esp_pm_config_esp32_t*) vconfig;
332 #elif CONFIG_IDF_TARGET_ESP32S2
333 esp_pm_config_esp32s2_t* config = (esp_pm_config_esp32s2_t*) vconfig;
334 #elif CONFIG_IDF_TARGET_ESP32S3
335 esp_pm_config_esp32s3_t* config = (esp_pm_config_esp32s3_t*) vconfig;
336 #elif CONFIG_IDF_TARGET_ESP32C3
337 esp_pm_config_esp32c3_t* config = (esp_pm_config_esp32c3_t*) vconfig;
338 #elif CONFIG_IDF_TARGET_ESP32H2
339 esp_pm_config_esp32h2_t* config = (esp_pm_config_esp32h2_t*) vconfig;
340 #endif
341
342 portENTER_CRITICAL(&s_switch_lock);
343 config->light_sleep_enable = s_light_sleep_en;
344 config->max_freq_mhz = s_cpu_freq_by_mode[PM_MODE_CPU_MAX].freq_mhz;
345 config->min_freq_mhz = s_cpu_freq_by_mode[PM_MODE_APB_MIN].freq_mhz;
346 portEXIT_CRITICAL(&s_switch_lock);
347
348 return ESP_OK;
349 }
350
get_lowest_allowed_mode(void)351 static pm_mode_t IRAM_ATTR get_lowest_allowed_mode(void)
352 {
353 /* TODO: optimize using ffs/clz */
354 if (s_mode_mask >= BIT(PM_MODE_CPU_MAX)) {
355 return PM_MODE_CPU_MAX;
356 } else if (s_mode_mask >= BIT(PM_MODE_APB_MAX)) {
357 return PM_MODE_APB_MAX;
358 } else if (s_mode_mask >= BIT(PM_MODE_APB_MIN) || !s_light_sleep_en) {
359 return PM_MODE_APB_MIN;
360 } else {
361 return PM_MODE_LIGHT_SLEEP;
362 }
363 }
364
esp_pm_impl_switch_mode(pm_mode_t mode,pm_mode_switch_t lock_or_unlock,pm_time_t now)365 void IRAM_ATTR esp_pm_impl_switch_mode(pm_mode_t mode,
366 pm_mode_switch_t lock_or_unlock, pm_time_t now)
367 {
368 bool need_switch = false;
369 uint32_t mode_mask = BIT(mode);
370 portENTER_CRITICAL_SAFE(&s_switch_lock);
371 uint32_t count;
372 if (lock_or_unlock == MODE_LOCK) {
373 count = ++s_mode_lock_counts[mode];
374 } else {
375 count = s_mode_lock_counts[mode]--;
376 }
377 if (count == 1) {
378 if (lock_or_unlock == MODE_LOCK) {
379 s_mode_mask |= mode_mask;
380 } else {
381 s_mode_mask &= ~mode_mask;
382 }
383 need_switch = true;
384 }
385
386 pm_mode_t new_mode = s_mode;
387 if (need_switch) {
388 new_mode = get_lowest_allowed_mode();
389 #ifdef WITH_PROFILING
390 if (s_last_mode_change_time != 0) {
391 pm_time_t diff = now - s_last_mode_change_time;
392 s_time_in_mode[s_mode] += diff;
393 }
394 s_last_mode_change_time = now;
395 #endif // WITH_PROFILING
396 }
397 portEXIT_CRITICAL_SAFE(&s_switch_lock);
398 if (need_switch) {
399 do_switch(new_mode);
400 }
401 }
402
403 /**
404 * @brief Update clock dividers in esp_timer and FreeRTOS, and adjust CCOMPARE
405 * values on both CPUs.
406 * @param old_ticks_per_us old CPU frequency
407 * @param ticks_per_us new CPU frequency
408 */
on_freq_update(uint32_t old_ticks_per_us,uint32_t ticks_per_us)409 static void IRAM_ATTR on_freq_update(uint32_t old_ticks_per_us, uint32_t ticks_per_us)
410 {
411 uint32_t old_apb_ticks_per_us = MIN(old_ticks_per_us, 80);
412 uint32_t apb_ticks_per_us = MIN(ticks_per_us, 80);
413 /* Update APB frequency value used by the timer */
414 if (old_apb_ticks_per_us != apb_ticks_per_us) {
415 esp_timer_private_update_apb_freq(apb_ticks_per_us);
416 }
417
418 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
419 #ifdef XT_RTOS_TIMER_INT
420 /* Calculate new tick divisor */
421 _xt_tick_divisor = ticks_per_us * MHZ / XT_TICK_PER_SEC;
422 #endif
423
424 int core_id = xPortGetCoreID();
425 if (s_rtos_lock_handle[core_id] != NULL) {
426 ESP_PM_TRACE_ENTER(CCOMPARE_UPDATE, core_id);
427 /* ccount_div and ccount_mul are used in esp_pm_impl_update_ccompare
428 * to calculate new CCOMPARE value.
429 */
430 s_ccount_div = old_ticks_per_us;
431 s_ccount_mul = ticks_per_us;
432
433 /* Update CCOMPARE value on this CPU */
434 update_ccompare();
435
436 #if portNUM_PROCESSORS == 2
437 /* Send interrupt to the other CPU to update CCOMPARE value */
438 int other_core_id = (core_id == 0) ? 1 : 0;
439
440 s_need_update_ccompare[other_core_id] = true;
441 esp_crosscore_int_send_freq_switch(other_core_id);
442
443 int timeout = 0;
444 while (s_need_update_ccompare[other_core_id]) {
445 if (++timeout == CCOMPARE_UPDATE_TIMEOUT) {
446 assert(false && "failed to update CCOMPARE, possible deadlock");
447 }
448 }
449 #endif // portNUM_PROCESSORS == 2
450
451 s_ccount_mul = 0;
452 s_ccount_div = 0;
453 ESP_PM_TRACE_EXIT(CCOMPARE_UPDATE, core_id);
454 }
455 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
456 }
457
458 /**
459 * Perform the switch to new power mode.
460 * Currently only changes the CPU frequency and adjusts clock dividers.
461 * No light sleep yet.
462 * @param new_mode mode to switch to
463 */
do_switch(pm_mode_t new_mode)464 static void IRAM_ATTR do_switch(pm_mode_t new_mode)
465 {
466 const int core_id = xPortGetCoreID();
467
468 do {
469 portENTER_CRITICAL_ISR(&s_switch_lock);
470 if (!s_is_switching) {
471 break;
472 }
473 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
474 if (s_need_update_ccompare[core_id]) {
475 s_need_update_ccompare[core_id] = false;
476 }
477 #endif
478 portEXIT_CRITICAL_ISR(&s_switch_lock);
479 } while (true);
480 if (new_mode == s_mode) {
481 portEXIT_CRITICAL_ISR(&s_switch_lock);
482 return;
483 }
484 s_is_switching = true;
485 bool config_changed = s_config_changed;
486 s_config_changed = false;
487 portEXIT_CRITICAL_ISR(&s_switch_lock);
488
489 rtc_cpu_freq_config_t new_config = s_cpu_freq_by_mode[new_mode];
490 rtc_cpu_freq_config_t old_config;
491
492 if (!config_changed) {
493 old_config = s_cpu_freq_by_mode[s_mode];
494 } else {
495 rtc_clk_cpu_freq_get_config(&old_config);
496 }
497
498 if (new_config.freq_mhz != old_config.freq_mhz) {
499 uint32_t old_ticks_per_us = old_config.freq_mhz;
500 uint32_t new_ticks_per_us = new_config.freq_mhz;
501
502 bool switch_down = new_ticks_per_us < old_ticks_per_us;
503
504 ESP_PM_TRACE_ENTER(FREQ_SWITCH, core_id);
505 if (switch_down) {
506 on_freq_update(old_ticks_per_us, new_ticks_per_us);
507 }
508 rtc_clk_cpu_freq_set_config_fast(&new_config);
509 if (!switch_down) {
510 on_freq_update(old_ticks_per_us, new_ticks_per_us);
511 }
512 ESP_PM_TRACE_EXIT(FREQ_SWITCH, core_id);
513 }
514
515 portENTER_CRITICAL_ISR(&s_switch_lock);
516 s_mode = new_mode;
517 s_is_switching = false;
518 portEXIT_CRITICAL_ISR(&s_switch_lock);
519 }
520
521 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
522 /**
523 * @brief Calculate new CCOMPARE value based on s_ccount_{mul,div}
524 *
525 * Adjusts CCOMPARE value so that the interrupt happens at the same time as it
526 * would happen without the frequency change.
527 * Assumes that the new_frequency = old_frequency * s_ccount_mul / s_ccount_div.
528 */
update_ccompare(void)529 static void IRAM_ATTR update_ccompare(void)
530 {
531 #if CONFIG_PM_UPDATE_CCOMPARE_HLI_WORKAROUND
532 /* disable level 4 and below */
533 uint32_t irq_status = XTOS_SET_INTLEVEL(XCHAL_DEBUGLEVEL - 2);
534 #endif
535 uint32_t ccount = cpu_hal_get_cycle_count();
536 uint32_t ccompare = XTHAL_GET_CCOMPARE(XT_TIMER_INDEX);
537 if ((ccompare - CCOMPARE_MIN_CYCLES_IN_FUTURE) - ccount < UINT32_MAX / 2) {
538 uint32_t diff = ccompare - ccount;
539 uint32_t diff_scaled = (diff * s_ccount_mul + s_ccount_div - 1) / s_ccount_div;
540 if (diff_scaled < _xt_tick_divisor) {
541 uint32_t new_ccompare = ccount + diff_scaled;
542 XTHAL_SET_CCOMPARE(XT_TIMER_INDEX, new_ccompare);
543 }
544 }
545 #if CONFIG_PM_UPDATE_CCOMPARE_HLI_WORKAROUND
546 XTOS_RESTORE_INTLEVEL(irq_status);
547 #endif
548 }
549 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
550
leave_idle(void)551 static void IRAM_ATTR leave_idle(void)
552 {
553 int core_id = xPortGetCoreID();
554 if (s_core_idle[core_id]) {
555 // TODO: possible optimization: raise frequency here first
556 esp_pm_lock_acquire(s_rtos_lock_handle[core_id]);
557 s_core_idle[core_id] = false;
558 }
559 }
560
561 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
562
esp_pm_register_skip_light_sleep_callback(skip_light_sleep_cb_t cb)563 esp_err_t esp_pm_register_skip_light_sleep_callback(skip_light_sleep_cb_t cb)
564 {
565 for (int i = 0; i < PERIPH_SKIP_LIGHT_SLEEP_NO; i++) {
566 if (s_periph_skip_light_sleep_cb[i] == cb) {
567 return ESP_OK;
568 } else if (s_periph_skip_light_sleep_cb[i] == NULL) {
569 s_periph_skip_light_sleep_cb[i] = cb;
570 return ESP_OK;
571 }
572 }
573 return ESP_ERR_NO_MEM;
574 }
575
esp_pm_unregister_skip_light_sleep_callback(skip_light_sleep_cb_t cb)576 esp_err_t esp_pm_unregister_skip_light_sleep_callback(skip_light_sleep_cb_t cb)
577 {
578 for (int i = 0; i < PERIPH_SKIP_LIGHT_SLEEP_NO; i++) {
579 if (s_periph_skip_light_sleep_cb[i] == cb) {
580 s_periph_skip_light_sleep_cb[i] = NULL;
581 return ESP_OK;
582 }
583 }
584 return ESP_ERR_INVALID_STATE;
585 }
586
periph_should_skip_light_sleep(void)587 static inline bool IRAM_ATTR periph_should_skip_light_sleep(void)
588 {
589 if (s_light_sleep_en) {
590 for (int i = 0; i < PERIPH_SKIP_LIGHT_SLEEP_NO; i++) {
591 if (s_periph_skip_light_sleep_cb[i]) {
592 if (s_periph_skip_light_sleep_cb[i]() == true) {
593 return true;
594 }
595 }
596 }
597 }
598 return false;
599 }
600
should_skip_light_sleep(int core_id)601 static inline bool IRAM_ATTR should_skip_light_sleep(int core_id)
602 {
603 #if portNUM_PROCESSORS == 2
604 if (s_skip_light_sleep[core_id]) {
605 s_skip_light_sleep[core_id] = false;
606 s_skipped_light_sleep[core_id] = true;
607 return true;
608 }
609 #endif // portNUM_PROCESSORS == 2
610
611 if (s_mode != PM_MODE_LIGHT_SLEEP || s_is_switching || periph_should_skip_light_sleep()) {
612 s_skipped_light_sleep[core_id] = true;
613 } else {
614 s_skipped_light_sleep[core_id] = false;
615 }
616 return s_skipped_light_sleep[core_id];
617 }
618
other_core_should_skip_light_sleep(int core_id)619 static inline void IRAM_ATTR other_core_should_skip_light_sleep(int core_id)
620 {
621 #if portNUM_PROCESSORS == 2
622 s_skip_light_sleep[!core_id] = true;
623 #endif
624 }
625
vApplicationSleep(TickType_t xExpectedIdleTime)626 void IRAM_ATTR vApplicationSleep( TickType_t xExpectedIdleTime )
627 {
628 portENTER_CRITICAL(&s_switch_lock);
629 int core_id = xPortGetCoreID();
630 if (!should_skip_light_sleep(core_id)) {
631 /* Calculate how much we can sleep */
632 int64_t next_esp_timer_alarm = esp_timer_get_next_alarm_for_wake_up();
633 int64_t now = esp_timer_get_time();
634 int64_t time_until_next_alarm = next_esp_timer_alarm - now;
635 int64_t wakeup_delay_us = portTICK_PERIOD_MS * 1000LL * xExpectedIdleTime;
636 int64_t sleep_time_us = MIN(wakeup_delay_us, time_until_next_alarm);
637 if (sleep_time_us >= configEXPECTED_IDLE_TIME_BEFORE_SLEEP * portTICK_PERIOD_MS * 1000LL) {
638 esp_sleep_enable_timer_wakeup(sleep_time_us - LIGHT_SLEEP_EARLY_WAKEUP_US);
639 #ifdef CONFIG_PM_TRACE
640 /* to force tracing GPIOs to keep state */
641 esp_sleep_pd_config(ESP_PD_DOMAIN_RTC_PERIPH, ESP_PD_OPTION_ON);
642 #endif
643 /* Enter sleep */
644 ESP_PM_TRACE_ENTER(SLEEP, core_id);
645 int64_t sleep_start = esp_timer_get_time();
646 esp_light_sleep_start();
647 int64_t slept_us = esp_timer_get_time() - sleep_start;
648 ESP_PM_TRACE_EXIT(SLEEP, core_id);
649
650 uint32_t slept_ticks = slept_us / (portTICK_PERIOD_MS * 1000LL);
651 if (slept_ticks > 0) {
652 /* Adjust RTOS tick count based on the amount of time spent in sleep */
653 vTaskStepTick(slept_ticks);
654
655 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
656 /* Trigger tick interrupt, since sleep time was longer
657 * than portTICK_PERIOD_MS. Note that setting INTSET does not
658 * work for timer interrupt, and changing CCOMPARE would clear
659 * the interrupt flag.
660 */
661 cpu_hal_set_cycle_count(XTHAL_GET_CCOMPARE(XT_TIMER_INDEX) - 16);
662 while (!(XTHAL_GET_INTERRUPT() & BIT(XT_TIMER_INTNUM))) {
663 ;
664 }
665 #else
666 portYIELD_WITHIN_API();
667 #endif
668 }
669 other_core_should_skip_light_sleep(core_id);
670 }
671 }
672 portEXIT_CRITICAL(&s_switch_lock);
673 }
674 #endif //CONFIG_FREERTOS_USE_TICKLESS_IDLE
675
676 #ifdef WITH_PROFILING
esp_pm_impl_dump_stats(FILE * out)677 void esp_pm_impl_dump_stats(FILE* out)
678 {
679 pm_time_t time_in_mode[PM_MODE_COUNT];
680
681 portENTER_CRITICAL_ISR(&s_switch_lock);
682 memcpy(time_in_mode, s_time_in_mode, sizeof(time_in_mode));
683 pm_time_t last_mode_change_time = s_last_mode_change_time;
684 pm_mode_t cur_mode = s_mode;
685 pm_time_t now = pm_get_time();
686 portEXIT_CRITICAL_ISR(&s_switch_lock);
687
688 time_in_mode[cur_mode] += now - last_mode_change_time;
689
690 fprintf(out, "\nMode stats:\n");
691 fprintf(out, "%-8s %-10s %-10s %-10s\n", "Mode", "CPU_freq", "Time(us)", "Time(%)");
692 for (int i = 0; i < PM_MODE_COUNT; ++i) {
693 if (i == PM_MODE_LIGHT_SLEEP && !s_light_sleep_en) {
694 /* don't display light sleep mode if it's not enabled */
695 continue;
696 }
697 fprintf(out, "%-8s %-3dM%-7s %-10lld %-2d%%\n",
698 s_mode_names[i],
699 s_cpu_freq_by_mode[i].freq_mhz,
700 "", //Empty space to align columns
701 time_in_mode[i],
702 (int) (time_in_mode[i] * 100 / now));
703 }
704 }
705 #endif // WITH_PROFILING
706
esp_pm_impl_get_cpu_freq(pm_mode_t mode)707 int esp_pm_impl_get_cpu_freq(pm_mode_t mode)
708 {
709 int freq_mhz;
710 if (mode >= PM_MODE_LIGHT_SLEEP && mode < PM_MODE_COUNT) {
711 portENTER_CRITICAL(&s_switch_lock);
712 freq_mhz = s_cpu_freq_by_mode[mode].freq_mhz;
713 portEXIT_CRITICAL(&s_switch_lock);
714 } else {
715 abort();
716 }
717 return freq_mhz;
718 }
719
esp_pm_impl_init(void)720 void esp_pm_impl_init(void)
721 {
722 #if defined(CONFIG_ESP_CONSOLE_UART)
723 //This clock source should be a source which won't be affected by DFS
724 uint32_t clk_source;
725 #if CONFIG_IDF_TARGET_ESP32 || CONFIG_IDF_TARGET_ESP32S2
726 clk_source = UART_SCLK_REF_TICK;
727 #else
728 clk_source = UART_SCLK_XTAL;
729 #endif
730 while(!uart_ll_is_tx_idle(UART_LL_GET_HW(CONFIG_ESP_CONSOLE_UART_NUM)));
731 /* When DFS is enabled, override system setting and use REFTICK as UART clock source */
732 uart_ll_set_sclk(UART_LL_GET_HW(CONFIG_ESP_CONSOLE_UART_NUM), clk_source);
733 uart_ll_set_baudrate(UART_LL_GET_HW(CONFIG_ESP_CONSOLE_UART_NUM), CONFIG_ESP_CONSOLE_UART_BAUDRATE);
734 #endif // CONFIG_ESP_CONSOLE_UART
735
736 #ifdef CONFIG_PM_TRACE
737 esp_pm_trace_init();
738 #endif
739
740 #if CONFIG_PM_SLP_DISABLE_GPIO && SOC_GPIO_SUPPORT_SLP_SWITCH
741 esp_sleep_config_gpio_isolate();
742 #endif
743 ESP_ERROR_CHECK(esp_pm_lock_create(ESP_PM_CPU_FREQ_MAX, 0, "rtos0",
744 &s_rtos_lock_handle[0]));
745 ESP_ERROR_CHECK(esp_pm_lock_acquire(s_rtos_lock_handle[0]));
746
747 #if portNUM_PROCESSORS == 2
748 ESP_ERROR_CHECK(esp_pm_lock_create(ESP_PM_CPU_FREQ_MAX, 0, "rtos1",
749 &s_rtos_lock_handle[1]));
750 ESP_ERROR_CHECK(esp_pm_lock_acquire(s_rtos_lock_handle[1]));
751 #endif // portNUM_PROCESSORS == 2
752
753 /* Configure all modes to use the default CPU frequency.
754 * This will be modified later by a call to esp_pm_configure.
755 */
756 rtc_cpu_freq_config_t default_config;
757 if (!rtc_clk_cpu_freq_mhz_to_config(DEFAULT_CPU_FREQ, &default_config)) {
758 assert(false && "unsupported frequency");
759 }
760 for (size_t i = 0; i < PM_MODE_COUNT; ++i) {
761 s_cpu_freq_by_mode[i] = default_config;
762 }
763
764 #ifdef CONFIG_PM_DFS_INIT_AUTO
765 int xtal_freq = (int) rtc_clk_xtal_freq_get();
766 #if CONFIG_IDF_TARGET_ESP32
767 esp_pm_config_esp32_t cfg = {
768 #elif CONFIG_IDF_TARGET_ESP32S2
769 esp_pm_config_esp32s2_t cfg = {
770 #elif CONFIG_IDF_TARGET_ESP32S3
771 esp_pm_config_esp32s3_t cfg = {
772 #elif CONFIG_IDF_TARGET_ESP32C3
773 esp_pm_config_esp32c3_t cfg = {
774 #elif CONFIG_IDF_TARGET_ESP32H2
775 esp_pm_config_esp32h2_t cfg = {
776 #endif
777 .max_freq_mhz = DEFAULT_CPU_FREQ,
778 .min_freq_mhz = xtal_freq,
779 };
780
781 esp_pm_configure(&cfg);
782 #endif //CONFIG_PM_DFS_INIT_AUTO
783 }
784
785 void esp_pm_impl_idle_hook(void)
786 {
787 int core_id = xPortGetCoreID();
788 uint32_t state = portSET_INTERRUPT_MASK_FROM_ISR();
789 if (!s_core_idle[core_id]
790 #ifdef CONFIG_FREERTOS_USE_TICKLESS_IDLE
791 && !periph_should_skip_light_sleep()
792 #endif
793 ) {
794 esp_pm_lock_release(s_rtos_lock_handle[core_id]);
795 s_core_idle[core_id] = true;
796 }
797 portCLEAR_INTERRUPT_MASK_FROM_ISR(state);
798 ESP_PM_TRACE_ENTER(IDLE, core_id);
799 }
800
801 void IRAM_ATTR esp_pm_impl_isr_hook(void)
802 {
803 int core_id = xPortGetCoreID();
804 ESP_PM_TRACE_ENTER(ISR_HOOK, core_id);
805 /* Prevent higher level interrupts (than the one this function was called from)
806 * from happening in this section, since they will also call into esp_pm_impl_isr_hook.
807 */
808 uint32_t state = portSET_INTERRUPT_MASK_FROM_ISR();
809 #if defined(CONFIG_FREERTOS_SYSTICK_USES_CCOUNT) && (portNUM_PROCESSORS == 2)
810 if (s_need_update_ccompare[core_id]) {
811 update_ccompare();
812 s_need_update_ccompare[core_id] = false;
813 } else {
814 leave_idle();
815 }
816 #else
817 leave_idle();
818 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT && portNUM_PROCESSORS == 2
819 portCLEAR_INTERRUPT_MASK_FROM_ISR(state);
820 ESP_PM_TRACE_EXIT(ISR_HOOK, core_id);
821 }
822
823 void esp_pm_impl_waiti(void)
824 {
825 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
826 int core_id = xPortGetCoreID();
827 if (s_skipped_light_sleep[core_id]) {
828 cpu_hal_waiti();
829 /* Interrupt took the CPU out of waiti and s_rtos_lock_handle[core_id]
830 * is now taken. However since we are back to idle task, we can release
831 * the lock so that vApplicationSleep can attempt to enter light sleep.
832 */
833 esp_pm_impl_idle_hook();
834 s_skipped_light_sleep[core_id] = false;
835 }
836 #else
837 cpu_hal_waiti();
838 #endif // CONFIG_FREERTOS_USE_TICKLESS_IDLE
839 }
840
841 #define PERIPH_INFORM_OUT_LIGHT_SLEEP_OVERHEAD_NO 1
842
843 /* Inform peripherals of light sleep wakeup overhead time */
844 static inform_out_light_sleep_overhead_cb_t s_periph_inform_out_light_sleep_overhead_cb[PERIPH_INFORM_OUT_LIGHT_SLEEP_OVERHEAD_NO];
845
846 esp_err_t esp_pm_register_inform_out_light_sleep_overhead_callback(inform_out_light_sleep_overhead_cb_t cb)
847 {
848 for (int i = 0; i < PERIPH_INFORM_OUT_LIGHT_SLEEP_OVERHEAD_NO; i++) {
849 if (s_periph_inform_out_light_sleep_overhead_cb[i] == cb) {
850 return ESP_OK;
851 } else if (s_periph_inform_out_light_sleep_overhead_cb[i] == NULL) {
852 s_periph_inform_out_light_sleep_overhead_cb[i] = cb;
853 return ESP_OK;
854 }
855 }
856 return ESP_ERR_NO_MEM;
857 }
858
859 esp_err_t esp_pm_unregister_inform_out_light_sleep_overhead_callback(inform_out_light_sleep_overhead_cb_t cb)
860 {
861 for (int i = 0; i < PERIPH_INFORM_OUT_LIGHT_SLEEP_OVERHEAD_NO; i++) {
862 if (s_periph_inform_out_light_sleep_overhead_cb[i] == cb) {
863 s_periph_inform_out_light_sleep_overhead_cb[i] = NULL;
864 return ESP_OK;
865 }
866 }
867 return ESP_ERR_INVALID_STATE;
868 }
869
870 void periph_inform_out_light_sleep_overhead(uint32_t out_light_sleep_time)
871 {
872 for (int i = 0; i < PERIPH_INFORM_OUT_LIGHT_SLEEP_OVERHEAD_NO; i++) {
873 if (s_periph_inform_out_light_sleep_overhead_cb[i]) {
874 s_periph_inform_out_light_sleep_overhead_cb[i](out_light_sleep_time);
875 }
876 }
877 }
878
879 static update_light_sleep_default_params_config_cb_t s_light_sleep_default_params_config_cb = NULL;
880
881 void esp_pm_register_light_sleep_default_params_config_callback(update_light_sleep_default_params_config_cb_t cb)
882 {
883 if (s_light_sleep_default_params_config_cb == NULL) {
884 s_light_sleep_default_params_config_cb = cb;
885 }
886 }
887
888 void esp_pm_unregister_light_sleep_default_params_config_callback(void)
889 {
890 if (s_light_sleep_default_params_config_cb) {
891 s_light_sleep_default_params_config_cb = NULL;
892 }
893 }
894
895 #if CONFIG_PM_SLP_DEFAULT_PARAMS_OPT
896 static void esp_pm_light_sleep_default_params_config(int min_freq_mhz, int max_freq_mhz)
897 {
898 if (s_light_sleep_default_params_config_cb) {
899 (*s_light_sleep_default_params_config_cb)(min_freq_mhz, max_freq_mhz);
900 }
901 }
902 #endif
903
