1 /*
2 * SPDX-FileCopyrightText: 2016-2023 Espressif Systems (Shanghai) CO LTD
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <stdlib.h>
8 #include <stdbool.h>
9 #include <string.h>
10 #include <stdint.h>
11 #include <sys/param.h>
12
13 #include "esp_attr.h"
14 #include "esp_err.h"
15 #include "esp_pm.h"
16 #include "esp_log.h"
17 #include "esp_cpu.h"
18
19 #include "esp_private/crosscore_int.h"
20
21 #include "soc/rtc.h"
22 #include "hal/uart_ll.h"
23 #include "hal/uart_types.h"
24 #include "driver/uart.h"
25 #include "driver/gpio.h"
26
27 #include "freertos/FreeRTOS.h"
28 #include "freertos/task.h"
29 #if CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
30 #include "freertos/xtensa_timer.h"
31 #include "xtensa/core-macros.h"
32 #endif
33
34 #if SOC_SPI_MEM_SUPPORT_TIME_TUNING
35 #include "esp_private/mspi_timing_tuning.h"
36 #endif
37
38 #include "esp_private/pm_impl.h"
39 #include "esp_private/pm_trace.h"
40 #include "esp_private/esp_timer_private.h"
41 #include "esp_private/esp_clk.h"
42 #include "esp_private/sleep_cpu.h"
43 #include "esp_private/sleep_gpio.h"
44 #include "esp_private/sleep_modem.h"
45 #include "esp_sleep.h"
46
47 #include "sdkconfig.h"
48
49
50 #define MHZ (1000000)
51
52 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
53 /* CCOMPARE update timeout, in CPU cycles. Any value above ~600 cycles will work
54 * for the purpose of detecting a deadlock.
55 */
56 #define CCOMPARE_UPDATE_TIMEOUT 1000000
57
58 /* When changing CCOMPARE, don't allow changes if the difference is less
59 * than this. This is to prevent setting CCOMPARE below CCOUNT.
60 */
61 #define CCOMPARE_MIN_CYCLES_IN_FUTURE 1000
62 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
63
64 /* When light sleep is used, wake this number of microseconds earlier than
65 * the next tick.
66 */
67 #define LIGHT_SLEEP_EARLY_WAKEUP_US 100
68
69 #if CONFIG_IDF_TARGET_ESP32
70 /* Minimal divider at which REF_CLK_FREQ can be obtained */
71 #define REF_CLK_DIV_MIN 10
72 #elif CONFIG_IDF_TARGET_ESP32S2
73 /* Minimal divider at which REF_CLK_FREQ can be obtained */
74 #define REF_CLK_DIV_MIN 2
75 #elif CONFIG_IDF_TARGET_ESP32S3
76 /* Minimal divider at which REF_CLK_FREQ can be obtained */
77 #define REF_CLK_DIV_MIN 2 // TODO: IDF-5660
78 #elif CONFIG_IDF_TARGET_ESP32C3
79 #define REF_CLK_DIV_MIN 2
80 #elif CONFIG_IDF_TARGET_ESP32C2
81 #define REF_CLK_DIV_MIN 2
82 #elif CONFIG_IDF_TARGET_ESP32C6
83 #define REF_CLK_DIV_MIN 2
84 #elif CONFIG_IDF_TARGET_ESP32H2
85 #define REF_CLK_DIV_MIN 2
86 #endif
87
88 #ifdef CONFIG_PM_PROFILING
89 #define WITH_PROFILING
90 #endif
91
92 static portMUX_TYPE s_switch_lock = portMUX_INITIALIZER_UNLOCKED;
93 /* The following state variables are protected using s_switch_lock: */
94 /* Current sleep mode; When switching, contains old mode until switch is complete */
95 static pm_mode_t s_mode = PM_MODE_CPU_MAX;
96 /* True when switch is in progress */
97 static volatile bool s_is_switching;
98 /* Number of times each mode was locked */
99 static size_t s_mode_lock_counts[PM_MODE_COUNT];
100 /* Bit mask of locked modes. BIT(i) is set iff s_mode_lock_counts[i] > 0. */
101 static uint32_t s_mode_mask;
102
103 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
104
105 #define PERIPH_SKIP_LIGHT_SLEEP_NO 2
106
107 /* Indicates if light sleep shoule be skipped by peripherals. */
108 static skip_light_sleep_cb_t s_periph_skip_light_sleep_cb[PERIPH_SKIP_LIGHT_SLEEP_NO];
109
110 /* Indicates if light sleep entry was skipped in vApplicationSleep for given CPU.
111 * This in turn gets used in IDLE hook to decide if `waiti` needs
112 * to be invoked or not.
113 */
114 static bool s_skipped_light_sleep[portNUM_PROCESSORS];
115
116 #if portNUM_PROCESSORS == 2
117 /* When light sleep is finished on one CPU, it is possible that the other CPU
118 * will enter light sleep again very soon, before interrupts on the first CPU
119 * get a chance to run. To avoid such situation, set a flag for the other CPU to
120 * skip light sleep attempt.
121 */
122 static bool s_skip_light_sleep[portNUM_PROCESSORS];
123 #endif // portNUM_PROCESSORS == 2
124 #endif // CONFIG_FREERTOS_USE_TICKLESS_IDLE
125
126 /* A flag indicating that Idle hook has run on a given CPU;
127 * Next interrupt on the same CPU will take s_rtos_lock_handle.
128 */
129 static bool s_core_idle[portNUM_PROCESSORS];
130
131 /* When no RTOS tasks are active, these locks are released to allow going into
132 * a lower power mode. Used by ISR hook and idle hook.
133 */
134 static esp_pm_lock_handle_t s_rtos_lock_handle[portNUM_PROCESSORS];
135
136 /* Lookup table of CPU frequency configs to be used in each mode.
137 * Initialized by esp_pm_impl_init and modified by esp_pm_configure.
138 */
139 static rtc_cpu_freq_config_t s_cpu_freq_by_mode[PM_MODE_COUNT];
140
141 /* Whether automatic light sleep is enabled */
142 static bool s_light_sleep_en = false;
143
144 /* When configuration is changed, current frequency may not match the
145 * newly configured frequency for the current mode. This is an indicator
146 * to the mode switch code to get the actual current frequency instead of
147 * relying on the current mode.
148 */
149 static bool s_config_changed = false;
150
151 #ifdef WITH_PROFILING
152 /* Time, in microseconds, spent so far in each mode */
153 static pm_time_t s_time_in_mode[PM_MODE_COUNT];
154 /* Timestamp, in microseconds, when the mode switch last happened */
155 static pm_time_t s_last_mode_change_time;
156 /* User-readable mode names, used by esp_pm_impl_dump_stats */
157 static const char* s_mode_names[] = {
158 "SLEEP",
159 "APB_MIN",
160 "APB_MAX",
161 "CPU_MAX"
162 };
163 static uint32_t s_light_sleep_counts, s_light_sleep_reject_counts;
164 #endif // WITH_PROFILING
165
166 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
167 /* Indicates to the ISR hook that CCOMPARE needs to be updated on the given CPU.
168 * Used in conjunction with cross-core interrupt to update CCOMPARE on the other CPU.
169 */
170 static volatile bool s_need_update_ccompare[portNUM_PROCESSORS];
171
172 /* Divider and multiplier used to adjust (ccompare - ccount) duration.
173 * Only set to non-zero values when switch is in progress.
174 */
175 static uint32_t s_ccount_div;
176 static uint32_t s_ccount_mul;
177
178 static void update_ccompare(void);
179 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
180
181 static const char* TAG = "pm";
182
183 static void do_switch(pm_mode_t new_mode);
184 static void leave_idle(void);
185 static void on_freq_update(uint32_t old_ticks_per_us, uint32_t ticks_per_us);
186
esp_pm_impl_get_mode(esp_pm_lock_type_t type,int arg)187 pm_mode_t esp_pm_impl_get_mode(esp_pm_lock_type_t type, int arg)
188 {
189 (void) arg;
190 if (type == ESP_PM_CPU_FREQ_MAX) {
191 return PM_MODE_CPU_MAX;
192 } else if (type == ESP_PM_APB_FREQ_MAX) {
193 return PM_MODE_APB_MAX;
194 } else if (type == ESP_PM_NO_LIGHT_SLEEP) {
195 return PM_MODE_APB_MIN;
196 } else {
197 // unsupported mode
198 abort();
199 }
200 }
201
esp_pm_sleep_configure(const void * vconfig)202 static esp_err_t esp_pm_sleep_configure(const void *vconfig)
203 {
204 esp_err_t err = ESP_OK;
205 const esp_pm_config_t* config = (const esp_pm_config_t*) vconfig;
206
207 #if SOC_PM_SUPPORT_CPU_PD
208 err = sleep_cpu_configure(config->light_sleep_enable);
209 if (err != ESP_OK) {
210 return err;
211 }
212 #endif
213
214 err = sleep_modem_configure(config->max_freq_mhz, config->min_freq_mhz, config->light_sleep_enable);
215 return err;
216 }
217
esp_pm_configure(const void * vconfig)218 esp_err_t esp_pm_configure(const void* vconfig)
219 {
220 #ifndef CONFIG_PM_ENABLE
221 return ESP_ERR_NOT_SUPPORTED;
222 #endif
223
224 const esp_pm_config_t* config = (const esp_pm_config_t*) vconfig;
225
226 #ifndef CONFIG_FREERTOS_USE_TICKLESS_IDLE
227 if (config->light_sleep_enable) {
228 return ESP_ERR_NOT_SUPPORTED;
229 }
230 #endif
231
232 int min_freq_mhz = config->min_freq_mhz;
233 int max_freq_mhz = config->max_freq_mhz;
234
235 if (min_freq_mhz > max_freq_mhz) {
236 return ESP_ERR_INVALID_ARG;
237 }
238
239 rtc_cpu_freq_config_t freq_config;
240 if (!rtc_clk_cpu_freq_mhz_to_config(min_freq_mhz, &freq_config)) {
241 ESP_LOGW(TAG, "invalid min_freq_mhz value (%d)", min_freq_mhz);
242 return ESP_ERR_INVALID_ARG;
243 }
244
245 int xtal_freq_mhz = esp_clk_xtal_freq() / MHZ;
246 if (min_freq_mhz < xtal_freq_mhz && min_freq_mhz * MHZ / REF_CLK_FREQ < REF_CLK_DIV_MIN) {
247 ESP_LOGW(TAG, "min_freq_mhz should be >= %d", REF_CLK_FREQ * REF_CLK_DIV_MIN / MHZ);
248 return ESP_ERR_INVALID_ARG;
249 }
250
251 if (!rtc_clk_cpu_freq_mhz_to_config(max_freq_mhz, &freq_config)) {
252 ESP_LOGW(TAG, "invalid max_freq_mhz value (%d)", max_freq_mhz);
253 return ESP_ERR_INVALID_ARG;
254 }
255
256 #if CONFIG_IDF_TARGET_ESP32
257 int apb_max_freq = max_freq_mhz; /* CPU frequency in APB_MAX mode */
258 if (max_freq_mhz == 240) {
259 /* We can't switch between 240 and 80/160 without disabling PLL,
260 * so use 240MHz CPU frequency when 80MHz APB frequency is requested.
261 */
262 apb_max_freq = 240;
263 } else if (max_freq_mhz == 160 || max_freq_mhz == 80) {
264 /* Otherwise, can use 80MHz
265 * CPU frequency when 80MHz APB frequency is requested.
266 */
267 apb_max_freq = 80;
268 }
269 #else
270 /* Maximum SOC APB clock frequency is 40 MHz, maximum Modem (WiFi,
271 * Bluetooth, etc..) APB clock frequency is 80 MHz */
272 int apb_clk_freq = esp_clk_apb_freq() / MHZ;
273 #if CONFIG_ESP_WIFI_ENABLED || CONFIG_BT_ENABLED || CONFIG_IEEE802154_ENABLED
274 apb_clk_freq = MAX(apb_clk_freq, MODEM_REQUIRED_MIN_APB_CLK_FREQ / MHZ);
275 #endif
276 int apb_max_freq = MIN(max_freq_mhz, apb_clk_freq); /* CPU frequency in APB_MAX mode */
277 #endif
278
279 apb_max_freq = MAX(apb_max_freq, min_freq_mhz);
280
281 ESP_LOGI(TAG, "Frequency switching config: "
282 "CPU_MAX: %d, APB_MAX: %d, APB_MIN: %d, Light sleep: %s",
283 max_freq_mhz,
284 apb_max_freq,
285 min_freq_mhz,
286 config->light_sleep_enable ? "ENABLED" : "DISABLED");
287
288 portENTER_CRITICAL(&s_switch_lock);
289
290 bool res __attribute__((unused));
291 res = rtc_clk_cpu_freq_mhz_to_config(max_freq_mhz, &s_cpu_freq_by_mode[PM_MODE_CPU_MAX]);
292 assert(res);
293 res = rtc_clk_cpu_freq_mhz_to_config(apb_max_freq, &s_cpu_freq_by_mode[PM_MODE_APB_MAX]);
294 assert(res);
295 res = rtc_clk_cpu_freq_mhz_to_config(min_freq_mhz, &s_cpu_freq_by_mode[PM_MODE_APB_MIN]);
296 assert(res);
297 s_cpu_freq_by_mode[PM_MODE_LIGHT_SLEEP] = s_cpu_freq_by_mode[PM_MODE_APB_MIN];
298 s_light_sleep_en = config->light_sleep_enable;
299 s_config_changed = true;
300 portEXIT_CRITICAL(&s_switch_lock);
301
302 esp_pm_sleep_configure(config);
303
304 return ESP_OK;
305 }
306
esp_pm_get_configuration(void * vconfig)307 esp_err_t esp_pm_get_configuration(void* vconfig)
308 {
309 if (vconfig == NULL) {
310 return ESP_ERR_INVALID_ARG;
311 }
312
313 esp_pm_config_t* config = (esp_pm_config_t*) vconfig;
314
315 portENTER_CRITICAL(&s_switch_lock);
316 config->light_sleep_enable = s_light_sleep_en;
317 config->max_freq_mhz = s_cpu_freq_by_mode[PM_MODE_CPU_MAX].freq_mhz;
318 config->min_freq_mhz = s_cpu_freq_by_mode[PM_MODE_APB_MIN].freq_mhz;
319 portEXIT_CRITICAL(&s_switch_lock);
320
321 return ESP_OK;
322 }
323
get_lowest_allowed_mode(void)324 static pm_mode_t IRAM_ATTR get_lowest_allowed_mode(void)
325 {
326 /* TODO: optimize using ffs/clz */
327 if (s_mode_mask >= BIT(PM_MODE_CPU_MAX)) {
328 return PM_MODE_CPU_MAX;
329 } else if (s_mode_mask >= BIT(PM_MODE_APB_MAX)) {
330 return PM_MODE_APB_MAX;
331 } else if (s_mode_mask >= BIT(PM_MODE_APB_MIN) || !s_light_sleep_en) {
332 return PM_MODE_APB_MIN;
333 } else {
334 return PM_MODE_LIGHT_SLEEP;
335 }
336 }
337
esp_pm_impl_switch_mode(pm_mode_t mode,pm_mode_switch_t lock_or_unlock,pm_time_t now)338 void IRAM_ATTR esp_pm_impl_switch_mode(pm_mode_t mode,
339 pm_mode_switch_t lock_or_unlock, pm_time_t now)
340 {
341 bool need_switch = false;
342 uint32_t mode_mask = BIT(mode);
343 portENTER_CRITICAL_SAFE(&s_switch_lock);
344 uint32_t count;
345 if (lock_or_unlock == MODE_LOCK) {
346 count = ++s_mode_lock_counts[mode];
347 } else {
348 count = s_mode_lock_counts[mode]--;
349 }
350 if (count == 1) {
351 if (lock_or_unlock == MODE_LOCK) {
352 s_mode_mask |= mode_mask;
353 } else {
354 s_mode_mask &= ~mode_mask;
355 }
356 need_switch = true;
357 }
358
359 pm_mode_t new_mode = s_mode;
360 if (need_switch) {
361 new_mode = get_lowest_allowed_mode();
362 #ifdef WITH_PROFILING
363 if (s_last_mode_change_time != 0) {
364 pm_time_t diff = now - s_last_mode_change_time;
365 s_time_in_mode[s_mode] += diff;
366 }
367 s_last_mode_change_time = now;
368 #endif // WITH_PROFILING
369 }
370 portEXIT_CRITICAL_SAFE(&s_switch_lock);
371 if (need_switch) {
372 do_switch(new_mode);
373 }
374 }
375
376 /**
377 * @brief Update clock dividers in esp_timer and FreeRTOS, and adjust CCOMPARE
378 * values on both CPUs.
379 * @param old_ticks_per_us old CPU frequency
380 * @param ticks_per_us new CPU frequency
381 */
on_freq_update(uint32_t old_ticks_per_us,uint32_t ticks_per_us)382 static void IRAM_ATTR on_freq_update(uint32_t old_ticks_per_us, uint32_t ticks_per_us)
383 {
384 uint32_t old_apb_ticks_per_us = MIN(old_ticks_per_us, 80);
385 uint32_t apb_ticks_per_us = MIN(ticks_per_us, 80);
386 /* Update APB frequency value used by the timer */
387 if (old_apb_ticks_per_us != apb_ticks_per_us) {
388 esp_timer_private_update_apb_freq(apb_ticks_per_us);
389 }
390
391 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
392 #ifdef XT_RTOS_TIMER_INT
393 /* Calculate new tick divisor */
394 _xt_tick_divisor = ticks_per_us * MHZ / XT_TICK_PER_SEC;
395 #endif
396
397 int core_id = xPortGetCoreID();
398 if (s_rtos_lock_handle[core_id] != NULL) {
399 ESP_PM_TRACE_ENTER(CCOMPARE_UPDATE, core_id);
400 /* ccount_div and ccount_mul are used in esp_pm_impl_update_ccompare
401 * to calculate new CCOMPARE value.
402 */
403 s_ccount_div = old_ticks_per_us;
404 s_ccount_mul = ticks_per_us;
405
406 /* Update CCOMPARE value on this CPU */
407 update_ccompare();
408
409 #if portNUM_PROCESSORS == 2
410 /* Send interrupt to the other CPU to update CCOMPARE value */
411 int other_core_id = (core_id == 0) ? 1 : 0;
412
413 s_need_update_ccompare[other_core_id] = true;
414 esp_crosscore_int_send_freq_switch(other_core_id);
415
416 int timeout = 0;
417 while (s_need_update_ccompare[other_core_id]) {
418 if (++timeout == CCOMPARE_UPDATE_TIMEOUT) {
419 assert(false && "failed to update CCOMPARE, possible deadlock");
420 }
421 }
422 #endif // portNUM_PROCESSORS == 2
423
424 s_ccount_mul = 0;
425 s_ccount_div = 0;
426 ESP_PM_TRACE_EXIT(CCOMPARE_UPDATE, core_id);
427 }
428 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
429 }
430
431 /**
432 * Perform the switch to new power mode.
433 * Currently only changes the CPU frequency and adjusts clock dividers.
434 * No light sleep yet.
435 * @param new_mode mode to switch to
436 */
do_switch(pm_mode_t new_mode)437 static void IRAM_ATTR do_switch(pm_mode_t new_mode)
438 {
439 const int core_id = xPortGetCoreID();
440
441 do {
442 portENTER_CRITICAL_ISR(&s_switch_lock);
443 if (!s_is_switching) {
444 break;
445 }
446 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
447 if (s_need_update_ccompare[core_id]) {
448 s_need_update_ccompare[core_id] = false;
449 }
450 #endif
451 portEXIT_CRITICAL_ISR(&s_switch_lock);
452 } while (true);
453 if (new_mode == s_mode) {
454 portEXIT_CRITICAL_ISR(&s_switch_lock);
455 return;
456 }
457 s_is_switching = true;
458 bool config_changed = s_config_changed;
459 s_config_changed = false;
460 portEXIT_CRITICAL_ISR(&s_switch_lock);
461
462 rtc_cpu_freq_config_t new_config = s_cpu_freq_by_mode[new_mode];
463 rtc_cpu_freq_config_t old_config;
464
465 if (!config_changed) {
466 old_config = s_cpu_freq_by_mode[s_mode];
467 } else {
468 rtc_clk_cpu_freq_get_config(&old_config);
469 }
470
471 if (new_config.freq_mhz != old_config.freq_mhz) {
472 uint32_t old_ticks_per_us = old_config.freq_mhz;
473 uint32_t new_ticks_per_us = new_config.freq_mhz;
474
475 bool switch_down = new_ticks_per_us < old_ticks_per_us;
476
477 ESP_PM_TRACE_ENTER(FREQ_SWITCH, core_id);
478 if (switch_down) {
479 on_freq_update(old_ticks_per_us, new_ticks_per_us);
480 }
481 if (new_config.source == SOC_CPU_CLK_SRC_PLL) {
482 rtc_clk_cpu_freq_set_config_fast(&new_config);
483 #if SOC_SPI_MEM_SUPPORT_TIME_TUNING
484 mspi_timing_change_speed_mode_cache_safe(false);
485 #endif
486 } else {
487 #if SOC_SPI_MEM_SUPPORT_TIME_TUNING
488 mspi_timing_change_speed_mode_cache_safe(true);
489 #endif
490 rtc_clk_cpu_freq_set_config_fast(&new_config);
491 }
492 if (!switch_down) {
493 on_freq_update(old_ticks_per_us, new_ticks_per_us);
494 }
495 ESP_PM_TRACE_EXIT(FREQ_SWITCH, core_id);
496 }
497
498 portENTER_CRITICAL_ISR(&s_switch_lock);
499 s_mode = new_mode;
500 s_is_switching = false;
501 portEXIT_CRITICAL_ISR(&s_switch_lock);
502 }
503
504 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
505 /**
506 * @brief Calculate new CCOMPARE value based on s_ccount_{mul,div}
507 *
508 * Adjusts CCOMPARE value so that the interrupt happens at the same time as it
509 * would happen without the frequency change.
510 * Assumes that the new_frequency = old_frequency * s_ccount_mul / s_ccount_div.
511 */
update_ccompare(void)512 static void IRAM_ATTR update_ccompare(void)
513 {
514 #if CONFIG_PM_UPDATE_CCOMPARE_HLI_WORKAROUND
515 /* disable level 4 and below */
516 uint32_t irq_status = XTOS_SET_INTLEVEL(XCHAL_DEBUGLEVEL - 2);
517 #endif
518 uint32_t ccount = esp_cpu_get_cycle_count();
519 uint32_t ccompare = XTHAL_GET_CCOMPARE(XT_TIMER_INDEX);
520 if ((ccompare - CCOMPARE_MIN_CYCLES_IN_FUTURE) - ccount < UINT32_MAX / 2) {
521 uint32_t diff = ccompare - ccount;
522 uint32_t diff_scaled = (diff * s_ccount_mul + s_ccount_div - 1) / s_ccount_div;
523 if (diff_scaled < _xt_tick_divisor) {
524 uint32_t new_ccompare = ccount + diff_scaled;
525 XTHAL_SET_CCOMPARE(XT_TIMER_INDEX, new_ccompare);
526 }
527 }
528 #if CONFIG_PM_UPDATE_CCOMPARE_HLI_WORKAROUND
529 XTOS_RESTORE_INTLEVEL(irq_status);
530 #endif
531 }
532 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
533
leave_idle(void)534 static void IRAM_ATTR leave_idle(void)
535 {
536 int core_id = xPortGetCoreID();
537 if (s_core_idle[core_id]) {
538 // TODO: possible optimization: raise frequency here first
539 esp_pm_lock_acquire(s_rtos_lock_handle[core_id]);
540 s_core_idle[core_id] = false;
541 }
542 }
543
544 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
545
esp_pm_register_skip_light_sleep_callback(skip_light_sleep_cb_t cb)546 esp_err_t esp_pm_register_skip_light_sleep_callback(skip_light_sleep_cb_t cb)
547 {
548 for (int i = 0; i < PERIPH_SKIP_LIGHT_SLEEP_NO; i++) {
549 if (s_periph_skip_light_sleep_cb[i] == cb) {
550 return ESP_OK;
551 } else if (s_periph_skip_light_sleep_cb[i] == NULL) {
552 s_periph_skip_light_sleep_cb[i] = cb;
553 return ESP_OK;
554 }
555 }
556 return ESP_ERR_NO_MEM;
557 }
558
esp_pm_unregister_skip_light_sleep_callback(skip_light_sleep_cb_t cb)559 esp_err_t esp_pm_unregister_skip_light_sleep_callback(skip_light_sleep_cb_t cb)
560 {
561 for (int i = 0; i < PERIPH_SKIP_LIGHT_SLEEP_NO; i++) {
562 if (s_periph_skip_light_sleep_cb[i] == cb) {
563 s_periph_skip_light_sleep_cb[i] = NULL;
564 return ESP_OK;
565 }
566 }
567 return ESP_ERR_INVALID_STATE;
568 }
569
periph_should_skip_light_sleep(void)570 static inline bool IRAM_ATTR periph_should_skip_light_sleep(void)
571 {
572 if (s_light_sleep_en) {
573 for (int i = 0; i < PERIPH_SKIP_LIGHT_SLEEP_NO; i++) {
574 if (s_periph_skip_light_sleep_cb[i]) {
575 if (s_periph_skip_light_sleep_cb[i]() == true) {
576 return true;
577 }
578 }
579 }
580 }
581 return false;
582 }
583
should_skip_light_sleep(int core_id)584 static inline bool IRAM_ATTR should_skip_light_sleep(int core_id)
585 {
586 #if portNUM_PROCESSORS == 2
587 if (s_skip_light_sleep[core_id]) {
588 s_skip_light_sleep[core_id] = false;
589 s_skipped_light_sleep[core_id] = true;
590 return true;
591 }
592 #endif // portNUM_PROCESSORS == 2
593
594 if (s_mode != PM_MODE_LIGHT_SLEEP || s_is_switching || periph_should_skip_light_sleep()) {
595 s_skipped_light_sleep[core_id] = true;
596 } else {
597 s_skipped_light_sleep[core_id] = false;
598 }
599 return s_skipped_light_sleep[core_id];
600 }
601
other_core_should_skip_light_sleep(int core_id)602 static inline void IRAM_ATTR other_core_should_skip_light_sleep(int core_id)
603 {
604 #if portNUM_PROCESSORS == 2
605 s_skip_light_sleep[!core_id] = true;
606 #endif
607 }
608
vApplicationSleep(TickType_t xExpectedIdleTime)609 void IRAM_ATTR vApplicationSleep( TickType_t xExpectedIdleTime )
610 {
611 portENTER_CRITICAL(&s_switch_lock);
612 int core_id = xPortGetCoreID();
613 if (!should_skip_light_sleep(core_id)) {
614 /* Calculate how much we can sleep */
615 int64_t next_esp_timer_alarm = esp_timer_get_next_alarm_for_wake_up();
616 int64_t now = esp_timer_get_time();
617 int64_t time_until_next_alarm = next_esp_timer_alarm - now;
618 int64_t wakeup_delay_us = portTICK_PERIOD_MS * 1000LL * xExpectedIdleTime;
619 int64_t sleep_time_us = MIN(wakeup_delay_us, time_until_next_alarm);
620 if (sleep_time_us >= configEXPECTED_IDLE_TIME_BEFORE_SLEEP * portTICK_PERIOD_MS * 1000LL) {
621 esp_sleep_enable_timer_wakeup(sleep_time_us - LIGHT_SLEEP_EARLY_WAKEUP_US);
622 #if CONFIG_PM_TRACE && SOC_PM_SUPPORT_RTC_PERIPH_PD
623 /* to force tracing GPIOs to keep state */
624 esp_sleep_pd_config(ESP_PD_DOMAIN_RTC_PERIPH, ESP_PD_OPTION_ON);
625 #endif
626 /* Enter sleep */
627 ESP_PM_TRACE_ENTER(SLEEP, core_id);
628 int64_t sleep_start = esp_timer_get_time();
629 if (esp_light_sleep_start() != ESP_OK){
630 #ifdef WITH_PROFILING
631 s_light_sleep_reject_counts++;
632 } else {
633 s_light_sleep_counts++;
634 #endif
635 }
636 int64_t slept_us = esp_timer_get_time() - sleep_start;
637 ESP_PM_TRACE_EXIT(SLEEP, core_id);
638
639 uint32_t slept_ticks = slept_us / (portTICK_PERIOD_MS * 1000LL);
640 if (slept_ticks > 0) {
641 /* Adjust RTOS tick count based on the amount of time spent in sleep */
642 vTaskStepTick(slept_ticks);
643
644 #ifdef CONFIG_FREERTOS_SYSTICK_USES_CCOUNT
645 /* Trigger tick interrupt, since sleep time was longer
646 * than portTICK_PERIOD_MS. Note that setting INTSET does not
647 * work for timer interrupt, and changing CCOMPARE would clear
648 * the interrupt flag.
649 */
650 esp_cpu_set_cycle_count(XTHAL_GET_CCOMPARE(XT_TIMER_INDEX) - 16);
651 while (!(XTHAL_GET_INTERRUPT() & BIT(XT_TIMER_INTNUM))) {
652 ;
653 }
654 #else
655 portYIELD_WITHIN_API();
656 #endif
657 }
658 other_core_should_skip_light_sleep(core_id);
659 }
660 }
661 portEXIT_CRITICAL(&s_switch_lock);
662 }
663 #endif //CONFIG_FREERTOS_USE_TICKLESS_IDLE
664
665 #ifdef WITH_PROFILING
esp_pm_impl_dump_stats(FILE * out)666 void esp_pm_impl_dump_stats(FILE* out)
667 {
668 pm_time_t time_in_mode[PM_MODE_COUNT];
669
670 portENTER_CRITICAL_ISR(&s_switch_lock);
671 memcpy(time_in_mode, s_time_in_mode, sizeof(time_in_mode));
672 pm_time_t last_mode_change_time = s_last_mode_change_time;
673 pm_mode_t cur_mode = s_mode;
674 pm_time_t now = pm_get_time();
675 bool light_sleep_en = s_light_sleep_en;
676 uint32_t light_sleep_counts = s_light_sleep_counts;
677 uint32_t light_sleep_reject_counts = s_light_sleep_reject_counts;
678 portEXIT_CRITICAL_ISR(&s_switch_lock);
679
680 time_in_mode[cur_mode] += now - last_mode_change_time;
681
682 fprintf(out, "\nMode stats:\n");
683 fprintf(out, "%-8s %-10s %-10s %-10s\n", "Mode", "CPU_freq", "Time(us)", "Time(%)");
684 for (int i = 0; i < PM_MODE_COUNT; ++i) {
685 if (i == PM_MODE_LIGHT_SLEEP && !light_sleep_en) {
686 /* don't display light sleep mode if it's not enabled */
687 continue;
688 }
689 fprintf(out, "%-8s %-3"PRIu32"M%-7s %-10lld %-2d%%\n",
690 s_mode_names[i],
691 s_cpu_freq_by_mode[i].freq_mhz,
692 "", //Empty space to align columns
693 time_in_mode[i],
694 (int) (time_in_mode[i] * 100 / now));
695 }
696 if (light_sleep_en){
697 fprintf(out, "\nSleep stats:\n");
698 fprintf(out, "light_sleep_counts:%ld light_sleep_reject_counts:%ld\n", light_sleep_counts, light_sleep_reject_counts);
699 }
700 }
701 #endif // WITH_PROFILING
702
esp_pm_impl_get_cpu_freq(pm_mode_t mode)703 int esp_pm_impl_get_cpu_freq(pm_mode_t mode)
704 {
705 int freq_mhz;
706 if (mode >= PM_MODE_LIGHT_SLEEP && mode < PM_MODE_COUNT) {
707 portENTER_CRITICAL(&s_switch_lock);
708 freq_mhz = s_cpu_freq_by_mode[mode].freq_mhz;
709 portEXIT_CRITICAL(&s_switch_lock);
710 } else {
711 abort();
712 }
713 return freq_mhz;
714 }
715
esp_pm_impl_init(void)716 void esp_pm_impl_init(void)
717 {
718 #if defined(CONFIG_ESP_CONSOLE_UART)
719 //This clock source should be a source which won't be affected by DFS
720 uart_sclk_t clk_source = UART_SCLK_DEFAULT;
721 #if SOC_UART_SUPPORT_REF_TICK
722 clk_source = UART_SCLK_REF_TICK;
723 #elif SOC_UART_SUPPORT_XTAL_CLK
724 clk_source = UART_SCLK_XTAL;
725 #else
726 #error "No UART clock source is aware of DFS"
727 #endif // SOC_UART_SUPPORT_xxx
728 while(!uart_ll_is_tx_idle(UART_LL_GET_HW(CONFIG_ESP_CONSOLE_UART_NUM)));
729 /* When DFS is enabled, override system setting and use REFTICK as UART clock source */
730 uart_ll_set_sclk(UART_LL_GET_HW(CONFIG_ESP_CONSOLE_UART_NUM), clk_source);
731
732 uint32_t sclk_freq;
733 esp_err_t err = uart_get_sclk_freq(clk_source, &sclk_freq);
734 assert(err == ESP_OK);
735 uart_ll_set_baudrate(UART_LL_GET_HW(CONFIG_ESP_CONSOLE_UART_NUM), CONFIG_ESP_CONSOLE_UART_BAUDRATE, sclk_freq);
736 #endif // CONFIG_ESP_CONSOLE_UART
737
738 #ifdef CONFIG_PM_TRACE
739 esp_pm_trace_init();
740 #endif
741
742 ESP_ERROR_CHECK(esp_pm_lock_create(ESP_PM_CPU_FREQ_MAX, 0, "rtos0",
743 &s_rtos_lock_handle[0]));
744 ESP_ERROR_CHECK(esp_pm_lock_acquire(s_rtos_lock_handle[0]));
745
746 #if portNUM_PROCESSORS == 2
747 ESP_ERROR_CHECK(esp_pm_lock_create(ESP_PM_CPU_FREQ_MAX, 0, "rtos1",
748 &s_rtos_lock_handle[1]));
749 ESP_ERROR_CHECK(esp_pm_lock_acquire(s_rtos_lock_handle[1]));
750 #endif // portNUM_PROCESSORS == 2
751
752 /* Configure all modes to use the default CPU frequency.
753 * This will be modified later by a call to esp_pm_configure.
754 */
755 rtc_cpu_freq_config_t default_config;
756 if (!rtc_clk_cpu_freq_mhz_to_config(CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ, &default_config)) {
757 assert(false && "unsupported frequency");
758 }
759 for (size_t i = 0; i < PM_MODE_COUNT; ++i) {
760 s_cpu_freq_by_mode[i] = default_config;
761 }
762
763 #ifdef CONFIG_PM_DFS_INIT_AUTO
764 int xtal_freq_mhz = esp_clk_xtal_freq() / MHZ;
765 esp_pm_config_t cfg = {
766 .max_freq_mhz = CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ,
767 .min_freq_mhz = xtal_freq_mhz,
768 };
769
770 esp_pm_configure(&cfg);
771 #endif //CONFIG_PM_DFS_INIT_AUTO
772 }
773
esp_pm_impl_idle_hook(void)774 void esp_pm_impl_idle_hook(void)
775 {
776 int core_id = xPortGetCoreID();
777 #if CONFIG_FREERTOS_SMP
778 uint32_t state = portDISABLE_INTERRUPTS();
779 #else
780 uint32_t state = portSET_INTERRUPT_MASK_FROM_ISR();
781 #endif
782 if (!s_core_idle[core_id]
783 #ifdef CONFIG_FREERTOS_USE_TICKLESS_IDLE
784 && !periph_should_skip_light_sleep()
785 #endif
786 ) {
787 esp_pm_lock_release(s_rtos_lock_handle[core_id]);
788 s_core_idle[core_id] = true;
789 }
790 #if CONFIG_FREERTOS_SMP
791 portRESTORE_INTERRUPTS(state);
792 #else
793 portCLEAR_INTERRUPT_MASK_FROM_ISR(state);
794 #endif
795 ESP_PM_TRACE_ENTER(IDLE, core_id);
796 }
797
esp_pm_impl_isr_hook(void)798 void IRAM_ATTR esp_pm_impl_isr_hook(void)
799 {
800 int core_id = xPortGetCoreID();
801 ESP_PM_TRACE_ENTER(ISR_HOOK, core_id);
802 /* Prevent higher level interrupts (than the one this function was called from)
803 * from happening in this section, since they will also call into esp_pm_impl_isr_hook.
804 */
805 #if CONFIG_FREERTOS_SMP
806 uint32_t state = portDISABLE_INTERRUPTS();
807 #else
808 uint32_t state = portSET_INTERRUPT_MASK_FROM_ISR();
809 #endif
810 #if defined(CONFIG_FREERTOS_SYSTICK_USES_CCOUNT) && (portNUM_PROCESSORS == 2)
811 if (s_need_update_ccompare[core_id]) {
812 update_ccompare();
813 s_need_update_ccompare[core_id] = false;
814 } else {
815 leave_idle();
816 }
817 #else
818 leave_idle();
819 #endif // CONFIG_FREERTOS_SYSTICK_USES_CCOUNT && portNUM_PROCESSORS == 2
820 #if CONFIG_FREERTOS_SMP
821 portRESTORE_INTERRUPTS(state);
822 #else
823 portCLEAR_INTERRUPT_MASK_FROM_ISR(state);
824 #endif
825 ESP_PM_TRACE_EXIT(ISR_HOOK, core_id);
826 }
827
esp_pm_impl_waiti(void)828 void esp_pm_impl_waiti(void)
829 {
830 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
831 int core_id = xPortGetCoreID();
832 if (s_skipped_light_sleep[core_id]) {
833 esp_cpu_wait_for_intr();
834 /* Interrupt took the CPU out of waiti and s_rtos_lock_handle[core_id]
835 * is now taken. However since we are back to idle task, we can release
836 * the lock so that vApplicationSleep can attempt to enter light sleep.
837 */
838 esp_pm_impl_idle_hook();
839 }
840 s_skipped_light_sleep[core_id] = true;
841 #else
842 esp_cpu_wait_for_intr();
843 #endif // CONFIG_FREERTOS_USE_TICKLESS_IDLE
844 }
845
