1 /*
2  * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
3  *
4  * SPDX-License-Identifier: Apache-2.0
5  */
6 
7 #include <stddef.h>
8 #include <string.h>
9 #include <zephyr/kernel.h>
10 #include <sys/param.h>
11 
12 #include "esp_err.h"
13 #include "esp_attr.h"
14 #include "esp_log.h"
15 #include "esp_heap_caps.h"
16 #include "esp_sleep.h"
17 #include "soc/soc_caps.h"
18 #include "esp_private/esp_regdma.h"
19 #include "esp_private/esp_pau.h"
20 #include "esp_private/sleep_retention.h"
21 #include "sdkconfig.h"
22 #include "esp_pmu.h"
23 
24 #if SOC_PM_PAU_REGDMA_UPDATE_CACHE_BEFORE_WAIT_COMPARE
25 #include "soc/pmu_reg.h" // for PMU_DATE_REG, it can provide full 32 bit read and write access
26 #endif
27 
28 static __attribute__((unused)) const char *TAG = "sleep";
29 
30 struct sleep_retention_module_object {
31     sleep_retention_module_callbacks_t cbs;         /* A callback list that can extend more sleep retention event callbacks */
32     sleep_retention_module_bitmap_t    dependents;  /* A bitmap identifying all modules that the current module depends on */
33     sleep_retention_module_bitmap_t    references;  /* A bitmap indicating all other modules that depend on (or reference) the current module,
34                                                      * It will update at runtime based on whether the module is referenced by other modules */
35     sleep_retention_module_attribute_t attributes;  /* A bitmap indicating attribute of the current module */
36 };
37 
sleep_retention_module_object_ctor(struct sleep_retention_module_object * const self,sleep_retention_module_callbacks_t * cbs)38 static inline void sleep_retention_module_object_ctor(struct sleep_retention_module_object * const self, sleep_retention_module_callbacks_t *cbs)
39 {
40     self->cbs = *cbs;
41     self->dependents = 0;
42     self->references = 0;
43     self->attributes = 0;
44 }
45 
sleep_retention_module_object_dtor(struct sleep_retention_module_object * const self)46 static inline void sleep_retention_module_object_dtor(struct sleep_retention_module_object * const self)
47 {
48     self->cbs = (sleep_retention_module_callbacks_t) { .create = { .handle = NULL, .arg = NULL } };
49 }
50 
set_dependencies(struct sleep_retention_module_object * const self,sleep_retention_module_bitmap_t depends)51 static inline void set_dependencies(struct sleep_retention_module_object * const self, sleep_retention_module_bitmap_t depends)
52 {
53     self->dependents = depends;
54 }
55 
clr_dependencies(struct sleep_retention_module_object * const self)56 static inline void clr_dependencies(struct sleep_retention_module_object * const self)
57 {
58     self->dependents = 0;
59 }
60 
get_dependencies(struct sleep_retention_module_object * const self)61 static inline sleep_retention_module_bitmap_t get_dependencies(struct sleep_retention_module_object * const self)
62 {
63     return self->dependents;
64 }
65 
set_reference(struct sleep_retention_module_object * const self,sleep_retention_module_t module)66 static inline void set_reference(struct sleep_retention_module_object * const self, sleep_retention_module_t module)
67 {
68     self->references |= BIT(module);
69 }
70 
clr_reference(struct sleep_retention_module_object * const self,sleep_retention_module_t module)71 static inline void clr_reference(struct sleep_retention_module_object * const self, sleep_retention_module_t module)
72 {
73     self->references &= ~BIT(module);
74 }
75 
get_references(struct sleep_retention_module_object * const self)76 static inline sleep_retention_module_bitmap_t get_references(struct sleep_retention_module_object * const self)
77 {
78     return self->references;
79 }
80 
references_exist(struct sleep_retention_module_object * const self)81 static inline bool references_exist(struct sleep_retention_module_object * const self)
82 {
83     return (get_references(self) != 0);
84 }
85 
set_attributes(struct sleep_retention_module_object * const self,sleep_retention_module_attribute_t attributes)86 static inline void set_attributes(struct sleep_retention_module_object * const self, sleep_retention_module_attribute_t attributes)
87 {
88     self->attributes = attributes;
89 }
90 
clr_attributes(struct sleep_retention_module_object * const self)91 static inline void clr_attributes(struct sleep_retention_module_object * const self)
92 {
93     self->attributes = 0;
94 }
95 
get_attributes(struct sleep_retention_module_object * const self)96 static inline sleep_retention_module_attribute_t get_attributes(struct sleep_retention_module_object * const self)
97 {
98     return self->attributes;
99 }
100 
module_is_passive(struct sleep_retention_module_object * const self)101 static inline bool module_is_passive(struct sleep_retention_module_object * const self)
102 {
103     return (get_attributes(self) & SLEEP_RETENTION_MODULE_ATTR_PASSIVE) ? true : false;
104 }
105 
module_is_inited(sleep_retention_module_t module)106 static inline bool module_is_inited(sleep_retention_module_t module)
107 {
108     return (sleep_retention_get_inited_modules() & BIT(module)) ? true : false;
109 }
110 
module_is_created(sleep_retention_module_t module)111 static inline bool module_is_created(sleep_retention_module_t module)
112 {
113     return (sleep_retention_get_created_modules() & BIT(module)) ? true : false;
114 }
115 
116 /**
117  * Internal structure which holds all requested sleep retention parameters
118  */
119 typedef struct {
120     /* The hardware retention module (REGDMA and PMU) uses 4 linked lists to
121      * record the hardware context information that needs to be backed up and
122      * restored when switching between different power states. The 4 linked
123      * lists are linked by 8 types of nodes. The 4 linked lists can reuse some
124      * nodes with each other, or separate their own unique nodes after branch
125      * type nodes.
126      * The REGDMA module iterates the entire linked list from the head of a
127      * linked list and backs up and restores the corresponding register context
128      * information according to the configuration information of the linked list
129      * nodes.
130      * The PMU module triggers REGDMA to use the corresponding linked list when
131      * switching between different power states. For example:
132      *
133      * +---------------+---------------+-------------------+-----------+
134      * |    Current    |   The next    | The entry will be | Retention |
135      * |   PMU state   |   PMU state   |  used by REGDMA   |   clock   |
136      * +---------------+---------------+-------------------+-----------+
137      * | PMU_HP_ACTIVE | PMU_HP_SLEEP  |     entry0        |    XTAL   |
138      * | PMU_HP_SLEEP  | PMU_HP_ACTIVE |     entry0        |    XTAL   |
139      * | PMU_HP_MODEM  | PMU_HP_SLEEP  |     ------        |    XTAL   |
140      * | PMU_HP_SLEEP  | PMU_HP_MODEM  |     entry1        |    XTAL   |
141      * | PMU_HP_MODEM  | PMU_HP_ACTIVE |     entry2        |    PLL    |
142      * |---------------------------------------------------------------|
143      * | PMU_HP_ACTIVE | PMU_HP_ACTIVE |     entry3        |    PLL    | (Clock BUG)
144      * +---------------+---------------+-------------------+-----------+
145      *
146      *           +--------+    +-------------------------+    +-------------+                            +-----------+    +--------+    +-----+
147      * entry2 -> |        | -> | WiFi MAC Minimum System | -> |             | -------------------------> | ######### | -> | ###### | -> | End |
148      *           |  SOC   |    +-------------------------+    |   Digital   |                            | Bluetooth |    | Zigbee |    +-----+
149      *           | System |             +--------+            | Peripherals |    +------+    +------+    |   / BLE   |    |        |    +-----+
150      * entry0 -> |        | ----------> |        | ---------> |             | -> |      | -> |      | -> |           | -> |        | -> | End |
151      *           +--------+             | Modem  |            +-------------+    | WiFi |    | WiFi |    +-----------+    +--------+    +-----+
152      *                                  | System |                               | MAC  |    |  BB  |    +-----+
153      * entry1 ------------------------> |        |-----------------------------> |      | -> |      | -> | End |
154      *                                  +--------+                               +------+    +------+    +-----+
155      *
156      * The entry3 (alias: extra linked list) is used for backup and restore of
157      * modules (such as BLE or 15.4 modules) with retention clock bugs.
158      *
159      *           +---------+    +----------+    +-------------+    +-----+
160      * entry3 -> | BLE MAC | -> | 15.4 MAC | -> | BLE/15.4 BB | -> | End |
161      *           +---------+    +----------+    +-------------+    +-----+
162      *
163      * Using it (extra linked list) for retention has the following constraints:
164      * 1. The PLL clock must be enabled (can be done with esp_pm_lock_acquire()
165      *    interface to acquire a pm lock of type ESP_PM_APB_FREQ_MAX.
166      * 2. When using the sleep_retention_entries_create() interface to create an
167      *    extra linked list, the node owner must be equal to BIT(3).
168      * 3. Use the sleep_retention_do_extra_retention() interface to backup or
169      *    restore the register context, which ensures only one backup or restore
170      *    when multiple modules (BLE and 15.4) exists.
171      */
172 #define SLEEP_RETENTION_REGDMA_LINK_NR_PRIORITIES       (8u)
173 #define SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY    (0)
174 #define SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY     (SLEEP_RETENTION_REGDMA_LINK_NR_PRIORITIES - 1)
175 #define SLEEP_RETENTION_MODULE_INVALID                  ((sleep_retention_module_t)(-1)) /* the final node does not belong to any module */
176     struct {
177         sleep_retention_entries_t entries;
178         uint32_t entries_bitmap: REGDMA_LINK_ENTRY_NUM,
179                  runtime_bitmap: REGDMA_LINK_ENTRY_NUM,
180                  reserved: 32-(2*REGDMA_LINK_ENTRY_NUM);
181         void *entries_tail;
182     } lists[SLEEP_RETENTION_REGDMA_LINK_NR_PRIORITIES];
183     regdma_link_priority_t highpri;
184     uint32_t inited_modules;
185     uint32_t created_modules;
186 
187     struct sleep_retention_module_object instance[32];
188 
189 #if SOC_PM_RETENTION_HAS_CLOCK_BUG
190 #define EXTRA_LINK_NUM  (REGDMA_LINK_ENTRY_NUM - 1)
191 #endif
192 } sleep_retention_t;
193 
194 static DRAM_ATTR __attribute__((unused)) sleep_retention_t s_retention = {
195     .highpri = (uint8_t)-1, .inited_modules = 0, .created_modules = 0
196 };
197 
198 K_SEM_DEFINE(s_retention_lock, 0, 5);
199 
200 #define SLEEP_RETENTION_ENTRY_BITMAP_MASK       (BIT(REGDMA_LINK_ENTRY_NUM) - 1)
201 #define SLEEP_RETENTION_ENTRY_BITMAP(bitmap)    ((bitmap) & SLEEP_RETENTION_ENTRY_BITMAP_MASK)
202 
203 static esp_err_t sleep_retention_entries_create_impl(const sleep_retention_entries_config_t retent[], int num, regdma_link_priority_t priority, sleep_retention_module_t module);
204 static void sleep_retention_entries_join(void);
205 
module_num2map(sleep_retention_module_t module)206 static inline sleep_retention_module_bitmap_t module_num2map(sleep_retention_module_t module)
207 {
208     return (module == SLEEP_RETENTION_MODULE_INVALID) ? 0 : BIT(module);
209 }
210 
sleep_retention_entries_require_branch(uint32_t owner,uint32_t runtime_bitmap)211 static inline bool sleep_retention_entries_require_branch(uint32_t owner, uint32_t runtime_bitmap)
212 {
213     bool use_new_entry      = SLEEP_RETENTION_ENTRY_BITMAP(owner & ~runtime_bitmap) ? true : false;
214     bool intersection_exist = SLEEP_RETENTION_ENTRY_BITMAP(owner &  runtime_bitmap) ? true : false;
215     return use_new_entry && intersection_exist;
216 }
217 
sleep_retention_entries_check_and_create_default(uint32_t owner,uint32_t runtime_bitmap,uint32_t entries_bitmap,regdma_link_priority_t priority,sleep_retention_module_t module)218 static esp_err_t sleep_retention_entries_check_and_create_default(uint32_t owner, uint32_t runtime_bitmap, uint32_t entries_bitmap, regdma_link_priority_t priority, sleep_retention_module_t module)
219 {
220     assert(sleep_retention_entries_require_branch(owner, runtime_bitmap));
221 
222     static sleep_retention_entries_config_t dummy = { REGDMA_LINK_WAIT_INIT(0xffff, 0, 0, 0, 1, 1), 0 };
223     dummy.owner = SLEEP_RETENTION_ENTRY_BITMAP(owner & ~entries_bitmap);
224     if (dummy.owner) {
225         return sleep_retention_entries_create_impl(&dummy, 1, priority, module);
226     }
227     return ESP_OK;
228 }
229 
sleep_retention_entries_check_and_create_final_default(void)230 static esp_err_t sleep_retention_entries_check_and_create_final_default(void)
231 {
232     static const sleep_retention_entries_config_t final_dummy = { REGDMA_LINK_WAIT_INIT(0xffff, 0, 0, 0, 1, 1), SLEEP_RETENTION_ENTRY_BITMAP_MASK };
233 
234     esp_err_t err = ESP_OK;
235     k_sem_take(&s_retention_lock, K_FOREVER);
236     if (s_retention.lists[SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY].entries_bitmap == 0) {
237         err = sleep_retention_entries_create_impl(&final_dummy, 1, SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY, SLEEP_RETENTION_MODULE_INVALID);
238     }
239     k_sem_give(&s_retention_lock);
240     return err;
241 }
242 
sleep_retention_entries_update(uint32_t owner,void * new_link,regdma_link_priority_t priority)243 static void sleep_retention_entries_update(uint32_t owner, void *new_link, regdma_link_priority_t priority)
244 {
245     k_sem_take(&s_retention_lock, K_FOREVER);
246     sleep_retention_entries_t retention_entries = {
247         (owner & BIT(0)) ? new_link : s_retention.lists[priority].entries[0],
248         (owner & BIT(1)) ? new_link : s_retention.lists[priority].entries[1],
249         (owner & BIT(2)) ? new_link : s_retention.lists[priority].entries[2],
250         (owner & BIT(3)) ? new_link : s_retention.lists[priority].entries[3]
251     };
252     if (s_retention.lists[priority].entries_bitmap == 0) {
253         s_retention.lists[priority].entries_tail = new_link;
254     }
255     memcpy(s_retention.lists[priority].entries, retention_entries, sizeof(sleep_retention_entries_t));
256     s_retention.lists[priority].runtime_bitmap  = owner;
257     s_retention.lists[priority].entries_bitmap |= owner;
258     k_sem_give(&s_retention_lock);
259 }
260 
sleep_retention_entries_try_create(const regdma_link_config_t * config,uint32_t owner,regdma_link_priority_t priority,sleep_retention_module_t module)261 static void * sleep_retention_entries_try_create(const regdma_link_config_t *config, uint32_t owner, regdma_link_priority_t priority, sleep_retention_module_t module)
262 {
263     void *link = NULL;
264     assert(owner > 0 && owner < BIT(REGDMA_LINK_ENTRY_NUM));
265 
266     k_sem_take(&s_retention_lock, K_FOREVER);
267     if (sleep_retention_entries_require_branch(owner, s_retention.lists[priority].runtime_bitmap)) {
268         if (sleep_retention_entries_check_and_create_default(owner, s_retention.lists[priority].runtime_bitmap,
269             s_retention.lists[priority].entries_bitmap, priority, module_num2map(module)) == ESP_OK) { /* branch node can't as tail node */
270             link = regdma_link_init_safe(
271                         config, true, module_num2map(module),
272                         (owner & BIT(0)) ? s_retention.lists[priority].entries[0] : NULL,
273                         (owner & BIT(1)) ? s_retention.lists[priority].entries[1] : NULL,
274                         (owner & BIT(2)) ? s_retention.lists[priority].entries[2] : NULL,
275                         (owner & BIT(3)) ? s_retention.lists[priority].entries[3] : NULL
276                     );
277         }
278     } else {
279         link = regdma_link_init_safe(config, false, module_num2map(module), s_retention.lists[priority].entries[__builtin_ffs(owner) - 1]);
280     }
281     k_sem_give(&s_retention_lock);
282     return link;
283 }
284 
sleep_retention_entries_try_create_bonding(const regdma_link_config_t * config,uint32_t owner,regdma_link_priority_t priority,sleep_retention_module_t module)285 static void * sleep_retention_entries_try_create_bonding(const regdma_link_config_t *config, uint32_t owner, regdma_link_priority_t priority, sleep_retention_module_t module)
286 {
287     assert(owner > 0 && owner < BIT(REGDMA_LINK_ENTRY_NUM));
288     k_sem_take(&s_retention_lock, K_FOREVER);
289     void *link = regdma_link_init_safe(
290                 config, true, module_num2map(module),
291                 (owner & BIT(0)) ? s_retention.lists[priority].entries[0] : NULL,
292                 (owner & BIT(1)) ? s_retention.lists[priority].entries[1] : NULL,
293                 (owner & BIT(2)) ? s_retention.lists[priority].entries[2] : NULL,
294                 (owner & BIT(3)) ? s_retention.lists[priority].entries[3] : NULL
295             );
296     k_sem_give(&s_retention_lock);
297     return link;
298 }
299 
sleep_retention_entries_stats(void)300 static void sleep_retention_entries_stats(void)
301 {
302     k_sem_take(&s_retention_lock, K_FOREVER);
303     if (s_retention.highpri >= SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY && s_retention.highpri <= SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY) {
304         for (int entry = 0; entry < ARRAY_SIZE(s_retention.lists[s_retention.highpri].entries); entry++) {
305             regdma_link_stats(s_retention.lists[s_retention.highpri].entries[entry], entry);
306         }
307     }
308     k_sem_give(&s_retention_lock);
309 }
310 
311 #if REGDMA_LINK_DBG
sleep_retention_dump_entries(FILE * out)312 void sleep_retention_dump_entries(FILE *out)
313 {
314     k_sem_take(&s_retention_lock, K_FOREVER);
315     if (s_retention.highpri >= SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY && s_retention.highpri <= SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY) {
316         for (int entry = 0; entry < ARRAY_SIZE(s_retention.lists[s_retention.highpri].entries); entry++) {
317             fprintf(out, "\nsleep retention entries[%d] context:\n", entry);
318             regdma_link_dump(out, s_retention.lists[s_retention.highpri].entries[entry], entry);
319         }
320     }
321     k_sem_give(&s_retention_lock);
322 }
323 #endif
324 
sleep_retention_find_link_by_id(int id)325 void * sleep_retention_find_link_by_id(int id)
326 {
327     void *link = NULL;
328     k_sem_take(&s_retention_lock, K_FOREVER);
329     if (s_retention.highpri >= SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY &&
330         s_retention.highpri <= SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY) {
331         for (int entry = 0; (link == NULL && entry < ARRAY_SIZE(s_retention.lists[s_retention.highpri].entries)); entry++) {
332             link = regdma_find_link_by_id(s_retention.lists[s_retention.highpri].entries[entry], entry, id);
333         }
334     }
335     k_sem_give(&s_retention_lock);
336     return link;
337 }
338 
sleep_retention_entries_owner_bitmap(sleep_retention_entries_t * entries,sleep_retention_entries_t * tails)339 static uint32_t sleep_retention_entries_owner_bitmap(sleep_retention_entries_t *entries, sleep_retention_entries_t *tails)
340 {
341     uint32_t owner = 0;
342     k_sem_take(&s_retention_lock, K_FOREVER);
343     for (int entry = 0; entry < ARRAY_SIZE(*entries); entry++) {
344         owner |= regdma_link_get_owner_bitmap((*entries)[entry], (*tails)[entry], entry);
345     }
346     k_sem_give(&s_retention_lock);
347     return owner;
348 }
349 
sleep_retention_entries_get_destroy_context(regdma_link_priority_t priority,sleep_retention_module_t module,sleep_retention_entries_t * destroy_entries,void ** destroy_tail,sleep_retention_entries_t * next_entries,void ** prev_tail)350 static bool sleep_retention_entries_get_destroy_context(regdma_link_priority_t priority, sleep_retention_module_t module, sleep_retention_entries_t *destroy_entries, void **destroy_tail, sleep_retention_entries_t *next_entries, void **prev_tail)
351 {
352     bool exist = false;
353     sleep_retention_entries_t destroy_tails, prev_tails;
354 
355     memset(&destroy_tails, 0, sizeof(sleep_retention_entries_t));
356     memset(&prev_tails, 0, sizeof(sleep_retention_entries_t));
357 
358     k_sem_take(&s_retention_lock, K_FOREVER);
359     for (int entry = 0; entry < ARRAY_SIZE(s_retention.lists[priority].entries); entry++) {
360         (*destroy_entries)[entry] = regdma_find_module_link_head(
361                 s_retention.lists[priority].entries[entry], s_retention.lists[priority].entries_tail, entry, module_num2map(module));
362         destroy_tails     [entry] = regdma_find_module_link_tail(
363                 s_retention.lists[priority].entries[entry], s_retention.lists[priority].entries_tail, entry, module_num2map(module));
364         (*next_entries)   [entry] = regdma_find_next_module_link_head(
365                 s_retention.lists[priority].entries[entry], s_retention.lists[priority].entries_tail, entry, module_num2map(module));
366         prev_tails        [entry] = regdma_find_prev_module_link_tail(
367                 s_retention.lists[priority].entries[entry], s_retention.lists[priority].entries_tail, entry, module_num2map(module));
368         if ((*destroy_entries)[entry] && destroy_tails[entry]) {
369             exist = true;
370         }
371         assert(destroy_tails[entry] == destroy_tails[0]);
372         assert(prev_tails[entry] == prev_tails[0]);
373     }
374     *destroy_tail = destroy_tails[0];
375     *prev_tail = prev_tails[0];
376     k_sem_give(&s_retention_lock);
377     return exist;
378 }
379 
sleep_retention_entries_context_update(regdma_link_priority_t priority)380 static void sleep_retention_entries_context_update(regdma_link_priority_t priority)
381 {
382     k_sem_take(&s_retention_lock, K_FOREVER);
383     sleep_retention_entries_t tails = {
384         s_retention.lists[priority].entries_tail, s_retention.lists[priority].entries_tail,
385         s_retention.lists[priority].entries_tail, s_retention.lists[priority].entries_tail
386     };
387     s_retention.lists[priority].entries_bitmap = sleep_retention_entries_owner_bitmap(&s_retention.lists[priority].entries, &tails);
388     s_retention.lists[priority].runtime_bitmap = sleep_retention_entries_owner_bitmap(&s_retention.lists[priority].entries, &s_retention.lists[priority].entries);
389     k_sem_give(&s_retention_lock);
390 }
391 
sleep_retention_entries_dettach(regdma_link_priority_t priority,sleep_retention_entries_t * destroy_entries,void * destroy_tail,sleep_retention_entries_t * next_entries,void * prev_tail)392 static bool sleep_retention_entries_dettach(regdma_link_priority_t priority, sleep_retention_entries_t *destroy_entries, void *destroy_tail, sleep_retention_entries_t *next_entries, void *prev_tail)
393 {
394     k_sem_take(&s_retention_lock, K_FOREVER);
395     bool is_head = (memcmp(destroy_entries, &s_retention.lists[priority].entries, sizeof(sleep_retention_entries_t)) == 0);
396     bool is_tail = (destroy_tail == s_retention.lists[priority].entries_tail);
397 
398     if (is_head && is_tail) {
399         memset(s_retention.lists[priority].entries, 0, sizeof(sleep_retention_entries_t));
400         s_retention.lists[priority].entries_tail = NULL;
401     } else if (is_head) {
402         memcpy(&s_retention.lists[priority].entries, next_entries, sizeof(sleep_retention_entries_t));
403     } else if (is_tail) {
404         s_retention.lists[priority].entries_tail = prev_tail;
405     } else {
406         regdma_link_update_next_safe(prev_tail, (*next_entries)[0], (*next_entries)[1], (*next_entries)[2], (*next_entries)[3]);
407     }
408     sleep_retention_entries_context_update(priority);
409 
410     regdma_link_update_next_safe(destroy_tail, NULL, NULL, NULL, NULL);
411     k_sem_give(&s_retention_lock);
412     return (is_head || is_tail);
413 }
414 
sleep_retention_entries_destroy_wrapper(sleep_retention_entries_t * destroy_entries)415 static void sleep_retention_entries_destroy_wrapper(sleep_retention_entries_t *destroy_entries)
416 {
417     for (int entry = 0; entry < ARRAY_SIZE(*destroy_entries); entry++) {
418         regdma_link_destroy((*destroy_entries)[entry], entry);
419     }
420 }
421 
sleep_retention_entries_check_and_distroy_final_default(void)422 static void sleep_retention_entries_check_and_distroy_final_default(void)
423 {
424     k_sem_take(&s_retention_lock, K_FOREVER);
425     assert(s_retention.highpri == SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY);
426     assert(s_retention.created_modules == 0);
427     sleep_retention_entries_destroy_wrapper(&s_retention.lists[SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY].entries);
428     k_sem_give(&s_retention_lock);
429 }
430 
sleep_retention_entries_all_destroy_wrapper(sleep_retention_module_t module)431 static void sleep_retention_entries_all_destroy_wrapper(sleep_retention_module_t module)
432 {
433     void *destroy_tail = NULL, *prev_tail = NULL;
434     sleep_retention_entries_t destroy_entries, next_entries;
435 
436     memset(&destroy_entries, 0, sizeof(sleep_retention_entries_t));
437     memset(&next_entries, 0, sizeof(sleep_retention_entries_t));
438 
439     k_sem_take(&s_retention_lock, K_FOREVER);
440     regdma_link_priority_t priority = 0;
441     do {
442         bool exist = sleep_retention_entries_get_destroy_context(priority, module, &destroy_entries, &destroy_tail, &next_entries, &prev_tail);
443         if (s_retention.lists[priority].entries_bitmap && exist) {
444             if (sleep_retention_entries_dettach(priority, &destroy_entries, destroy_tail, &next_entries, prev_tail)) {
445                 sleep_retention_entries_join();
446             }
447             sleep_retention_entries_destroy_wrapper(&destroy_entries);
448         } else {
449             priority++;
450         }
451     } while (priority < SLEEP_RETENTION_REGDMA_LINK_NR_PRIORITIES);
452     s_retention.created_modules &= ~module_num2map(module);
453     k_sem_give(&s_retention_lock);
454 }
455 
sleep_retention_entries_do_destroy(sleep_retention_module_t module)456 static void sleep_retention_entries_do_destroy(sleep_retention_module_t module)
457 {
458     assert(SLEEP_RETENTION_MODULE_MIN <= module && module <= SLEEP_RETENTION_MODULE_MAX);
459     k_sem_take(&s_retention_lock, K_FOREVER);
460     sleep_retention_entries_join();
461     sleep_retention_entries_stats();
462     sleep_retention_entries_all_destroy_wrapper(module);
463     k_sem_give(&s_retention_lock);
464 }
465 
sleep_retention_entries_destroy(sleep_retention_module_t module)466 static void sleep_retention_entries_destroy(sleep_retention_module_t module)
467 {
468     assert(SLEEP_RETENTION_MODULE_MIN <= module && module <= SLEEP_RETENTION_MODULE_MAX);
469     k_sem_take(&s_retention_lock, K_FOREVER);
470     sleep_retention_entries_do_destroy(module);
471     if (s_retention.created_modules == 0) {
472         sleep_retention_entries_check_and_distroy_final_default();
473         pmu_sleep_disable_regdma_backup();
474         memset((void *)s_retention.lists, 0, sizeof(s_retention.lists));
475         s_retention.highpri = (uint8_t)-1;
476     }
477     k_sem_give(&s_retention_lock);
478 }
479 
sleep_retention_entries_create_impl(const sleep_retention_entries_config_t retent[],int num,regdma_link_priority_t priority,sleep_retention_module_t module)480 static esp_err_t sleep_retention_entries_create_impl(const sleep_retention_entries_config_t retent[], int num, regdma_link_priority_t priority, sleep_retention_module_t module)
481 {
482     esp_err_t err = ESP_OK;
483     k_sem_take(&s_retention_lock, K_FOREVER);
484     for (int i = num - 1; (i >= 0) && (err == ESP_OK); i--) {
485 #if SOC_PM_RETENTION_HAS_CLOCK_BUG
486         if ((retent[i].owner > BIT(EXTRA_LINK_NUM)) && (retent[i].config.id != 0xffff)) {
487             k_sem_give(&s_retention_lock);
488             sleep_retention_entries_do_destroy(module);
489             return ESP_ERR_NOT_SUPPORTED;
490         }
491 #endif
492 #if SOC_PM_PAU_REGDMA_UPDATE_CACHE_BEFORE_WAIT_COMPARE
493         /* There is a bug in REGDMA wait mode, when two wait nodes need to wait for the
494          * same value (_val & _mask), the second wait node will immediately return to
495          * wait done, The reason is that the wait mode comparison output logic immediate
496          * compares the value of the previous wait register cached inside the
497          * digital logic before reading out he register contents specified by _backup.
498          */
499         #define config_is_wait_mode(_config)   (regdma_link_get_config_mode(_config) == REGDMA_LINK_MODE_WAIT)
500         if ((retent[i].config.id != 0xffff) && config_is_wait_mode(&(retent[i].config)) && (retent[i].config.id != 0xfffe)) {
501             uint32_t value = retent[i].config.write_wait.value;
502             uint32_t mask  = retent[i].config.write_wait.mask;
503             bool skip_b = retent[i].config.head.skip_b;
504             bool skip_r = retent[i].config.head.skip_r;
505             sleep_retention_entries_config_t wait_bug_workaround[] = {
506                 [0] = { .config = REGDMA_LINK_WRITE_INIT(0xfffe, PMU_DATE_REG, ~value, mask, skip_b, skip_r), .owner = retent[i].owner },
507                 [1] = { .config = REGDMA_LINK_WAIT_INIT (0xfffe, PMU_DATE_REG, ~value, mask, skip_b, skip_r), .owner = retent[i].owner }
508             };
509             err = sleep_retention_entries_create_impl(wait_bug_workaround, ARRAY_SIZE(wait_bug_workaround), priority, module);
510         }
511 #endif
512         if (err == ESP_OK) {
513             void *link = sleep_retention_entries_try_create(&retent[i].config, retent[i].owner, priority, module);
514             if (link == NULL) {
515                 k_sem_give(&s_retention_lock);
516                 sleep_retention_entries_do_destroy(module);
517                 return ESP_ERR_NO_MEM;
518             }
519             sleep_retention_entries_update(retent[i].owner, link, priority);
520         } else {
521             break;
522         }
523     }
524     k_sem_give(&s_retention_lock);
525     return err;
526 }
527 
sleep_retention_entries_create_bonding(regdma_link_priority_t priority,sleep_retention_module_t module)528 static esp_err_t sleep_retention_entries_create_bonding(regdma_link_priority_t priority, sleep_retention_module_t module)
529 {
530     static const sleep_retention_entries_config_t bonding_dummy = { REGDMA_LINK_WAIT_INIT(0xffff, 0, 0, 0, 1, 1), SLEEP_RETENTION_ENTRY_BITMAP_MASK };
531 
532     k_sem_take(&s_retention_lock, K_FOREVER);
533     void *link = sleep_retention_entries_try_create_bonding(&bonding_dummy.config, bonding_dummy.owner, priority, module);
534     if (link == NULL) {
535         k_sem_give(&s_retention_lock);
536         sleep_retention_entries_do_destroy(module);
537         return ESP_ERR_NO_MEM;
538     }
539     sleep_retention_entries_update(bonding_dummy.owner, link, priority);
540     k_sem_give(&s_retention_lock);
541     return ESP_OK;
542 }
543 
sleep_retention_entries_join(void)544 static void sleep_retention_entries_join(void)
545 {
546     void *entries_tail = NULL;
547     k_sem_take(&s_retention_lock, K_FOREVER);
548     s_retention.highpri = SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY;
549     for (regdma_link_priority_t priority = 0; priority < SLEEP_RETENTION_REGDMA_LINK_NR_PRIORITIES; priority++) {
550         if (s_retention.lists[priority].entries_bitmap == 0) continue;
551         if (priority < s_retention.highpri) { s_retention.highpri = priority; }
552         if (entries_tail) {
553             regdma_link_update_next_safe(
554                     entries_tail,
555                     s_retention.lists[priority].entries[0],
556                     s_retention.lists[priority].entries[1],
557                     s_retention.lists[priority].entries[2],
558                     s_retention.lists[priority].entries[3]
559                 );
560         }
561         entries_tail = s_retention.lists[priority].entries_tail;
562     }
563     pau_regdma_set_entry_link_addr(&(s_retention.lists[s_retention.highpri].entries));
564     k_sem_give(&s_retention_lock);
565 }
566 
sleep_retention_entries_create_wrapper(const sleep_retention_entries_config_t retent[],int num,regdma_link_priority_t priority,sleep_retention_module_t module)567 static esp_err_t sleep_retention_entries_create_wrapper(const sleep_retention_entries_config_t retent[], int num, regdma_link_priority_t priority, sleep_retention_module_t module)
568 {
569     k_sem_take(&s_retention_lock, K_FOREVER);
570     esp_err_t err = sleep_retention_entries_create_bonding(priority, module);
571     if(err) goto error;
572     err = sleep_retention_entries_create_impl(retent, num, priority, module);
573     if(err) goto error;
574     err = sleep_retention_entries_create_bonding(priority, module);
575     if(err) goto error;
576     s_retention.created_modules |= module_num2map(module);
577     sleep_retention_entries_join();
578 
579 error:
580     k_sem_give(&s_retention_lock);
581     return err;
582 }
583 
sleep_retention_entries_create(const sleep_retention_entries_config_t retent[],int num,regdma_link_priority_t priority,sleep_retention_module_t module)584 esp_err_t sleep_retention_entries_create(const sleep_retention_entries_config_t retent[], int num, regdma_link_priority_t priority, sleep_retention_module_t module)
585 {
586     if (retent == NULL || num <= 0) {
587         return ESP_ERR_INVALID_ARG;
588     }
589     if (priority >= SLEEP_RETENTION_REGDMA_LINK_NR_PRIORITIES) {
590         return ESP_ERR_INVALID_ARG;
591     }
592     if (module < SLEEP_RETENTION_MODULE_MIN || module > SLEEP_RETENTION_MODULE_MAX) {
593         return ESP_ERR_INVALID_ARG;
594     }
595     esp_err_t err = sleep_retention_entries_check_and_create_final_default();
596     if (err)  goto error;
597     err = sleep_retention_entries_create_wrapper(retent, num, priority, module);
598     if (err)  goto error;
599     pmu_sleep_enable_regdma_backup();
600     ESP_ERROR_CHECK(esp_deep_sleep_register_hook(&pmu_sleep_disable_regdma_backup));
601 
602 error:
603     return err;
604 }
605 
sleep_retention_entries_get(sleep_retention_entries_t * entries)606 void sleep_retention_entries_get(sleep_retention_entries_t *entries)
607 {
608     memset(entries, 0, sizeof(sleep_retention_entries_t));
609     k_sem_take(&s_retention_lock, K_FOREVER);
610     if (s_retention.highpri >= SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY &&
611         s_retention.highpri <= SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY) {
612         memcpy(entries, &s_retention.lists[s_retention.highpri].entries, sizeof(sleep_retention_entries_t));
613     }
614     k_sem_give(&s_retention_lock);
615 }
616 
sleep_retention_get_inited_modules(void)617 uint32_t IRAM_ATTR sleep_retention_get_inited_modules(void)
618 {
619     return s_retention.inited_modules;
620 }
621 
sleep_retention_get_created_modules(void)622 uint32_t IRAM_ATTR sleep_retention_get_created_modules(void)
623 {
624     return s_retention.created_modules;
625 }
626 
sleep_retention_module_init(sleep_retention_module_t module,sleep_retention_module_init_param_t * param)627 esp_err_t sleep_retention_module_init(sleep_retention_module_t module, sleep_retention_module_init_param_t *param)
628 {
629     if (module < SLEEP_RETENTION_MODULE_MIN || module > SLEEP_RETENTION_MODULE_MAX) {
630         return ESP_ERR_INVALID_ARG;
631     }
632     if (param == NULL || param->cbs.create.handle == NULL) {
633         return ESP_ERR_INVALID_ARG;
634     }
635 
636     esp_err_t err = ESP_OK;
637     k_sem_take(&s_retention_lock, K_FOREVER);
638     if (module_is_created(module) || module_is_inited(module)) {
639         err = ESP_ERR_INVALID_STATE;
640     } else {
641         sleep_retention_module_object_ctor(&s_retention.instance[module], &param->cbs);
642         set_dependencies(&s_retention.instance[module], param->depends);
643         set_attributes(&s_retention.instance[module], param->attribute);
644         s_retention.inited_modules |= module_num2map(module);
645     }
646     k_sem_give(&s_retention_lock);
647     return err;
648 }
649 
sleep_retention_module_deinit(sleep_retention_module_t module)650 esp_err_t sleep_retention_module_deinit(sleep_retention_module_t module)
651 {
652     if (module < SLEEP_RETENTION_MODULE_MIN || module > SLEEP_RETENTION_MODULE_MAX) {
653         return ESP_ERR_INVALID_ARG;
654     }
655 
656     esp_err_t err = ESP_OK;
657     bool do_lock_release = false;
658     k_sem_take(&s_retention_lock, K_FOREVER);
659     if (module_is_created(module) || !module_is_inited(module)) {
660         err = ESP_ERR_INVALID_STATE;
661     } else {
662         clr_attributes(&s_retention.instance[module]);
663         clr_dependencies(&s_retention.instance[module]);
664         sleep_retention_module_object_dtor(&s_retention.instance[module]);
665         s_retention.inited_modules &= ~module_num2map(module);
666         do_lock_release = (sleep_retention_get_inited_modules() == 0);
667     }
668     k_sem_give(&s_retention_lock);
669 
670     if (do_lock_release) {
671         k_sem_give(&s_retention_lock);
672     }
673     return err;
674 }
675 
sleep_retention_passive_module_allocate(sleep_retention_module_t module)676 static esp_err_t sleep_retention_passive_module_allocate(sleep_retention_module_t module)
677 {
678     assert(module >= SLEEP_RETENTION_MODULE_MIN && module <= SLEEP_RETENTION_MODULE_MAX);
679 
680     esp_err_t err = ESP_OK;
681     k_sem_take(&s_retention_lock, K_FOREVER);
682     assert(module_is_passive(&s_retention.instance[module]) && "Illegal dependency");
683     assert(module_is_inited(module) && "All passive module must be inited first!");
684     if (!module_is_created(module)) {
685         sleep_retention_module_bitmap_t depends = get_dependencies(&s_retention.instance[module]);
686         for (int i = 0; (err == ESP_OK) && depends; depends >>= 1, i++) {
687             if (depends & BIT(0)) {
688                 set_reference(&s_retention.instance[i], module);
689                 err = sleep_retention_passive_module_allocate(i);
690             }
691         }
692         if (err == ESP_OK) {
693             sleep_retention_callback_t fn = s_retention.instance[module].cbs.create.handle;
694             if (fn) {
695                 err = (*fn)(s_retention.instance[module].cbs.create.arg);
696             }
697         }
698     }
699     k_sem_give(&s_retention_lock);
700     return err;
701 }
702 
sleep_retention_module_allocate(sleep_retention_module_t module)703 esp_err_t sleep_retention_module_allocate(sleep_retention_module_t module)
704 {
705     if (module < SLEEP_RETENTION_MODULE_MIN || module > SLEEP_RETENTION_MODULE_MAX) {
706         return ESP_ERR_INVALID_ARG;
707     }
708 
709     esp_err_t err = ESP_OK;
710     k_sem_take(&s_retention_lock, K_FOREVER);
711     if (!module_is_passive(&s_retention.instance[module])) {
712         if (module_is_inited(module) && !module_is_created(module)) {
713             sleep_retention_module_bitmap_t depends = get_dependencies(&s_retention.instance[module]);
714             for (int i = 0; (err == ESP_OK) && depends; depends >>= 1, i++) {
715                 if (depends & BIT(0)) {
716                     set_reference(&s_retention.instance[i], module);
717                     if (module_is_passive(&s_retention.instance[i])) { /* the callee ensures this module is inited */
718                         err = sleep_retention_passive_module_allocate(i);
719                     }
720                 }
721             }
722             if (err == ESP_OK) {
723                 sleep_retention_callback_t fn = s_retention.instance[module].cbs.create.handle;
724                 if (fn) {
725                     err = (*fn)(s_retention.instance[module].cbs.create.arg);
726                 }
727             }
728         } else {
729             err = ESP_ERR_INVALID_STATE;
730         }
731     } else {
732         err = ESP_ERR_NOT_ALLOWED;
733     }
734     k_sem_give(&s_retention_lock);
735     return err;
736 }
737 
sleep_retention_passive_module_free(sleep_retention_module_t module)738 static esp_err_t sleep_retention_passive_module_free(sleep_retention_module_t module)
739 {
740     assert(module >= SLEEP_RETENTION_MODULE_MIN && module <= SLEEP_RETENTION_MODULE_MAX);
741 
742     esp_err_t err = ESP_OK;
743     k_sem_take(&s_retention_lock, K_FOREVER);
744     assert(module_is_passive(&s_retention.instance[module]) && "Illegal dependency");
745     assert(module_is_inited(module) && "All passive module must be inited first!");
746     if (module_is_created(module)) {
747         if (!references_exist(&s_retention.instance[module])) {
748             sleep_retention_entries_destroy(module);
749 
750             sleep_retention_module_bitmap_t depends = get_dependencies(&s_retention.instance[module]);
751             for (int i = 0; (err == ESP_OK) && depends; depends >>= 1, i++) {
752                 if (depends & BIT(0)) {
753                     clr_reference(&s_retention.instance[i], module);
754                     err = sleep_retention_passive_module_free(i);
755                 }
756             }
757         }
758     }
759     k_sem_give(&s_retention_lock);
760     return err;
761 }
762 
sleep_retention_module_free(sleep_retention_module_t module)763 esp_err_t sleep_retention_module_free(sleep_retention_module_t module)
764 {
765     if (module < SLEEP_RETENTION_MODULE_MIN || module > SLEEP_RETENTION_MODULE_MAX) {
766         return ESP_ERR_INVALID_ARG;
767     }
768 
769     esp_err_t err = ESP_OK;
770     k_sem_take(&s_retention_lock, K_FOREVER);
771     if (!module_is_passive(&s_retention.instance[module])) {
772         if (module_is_inited(module) && module_is_created(module)) {
773             sleep_retention_entries_destroy(module);
774 
775             sleep_retention_module_bitmap_t depends = get_dependencies(&s_retention.instance[module]);
776             for (int i = 0; (err == ESP_OK) && depends; depends >>= 1, i++) {
777                 if (depends & BIT(0)) {
778                     clr_reference(&s_retention.instance[i], module);
779                     if (module_is_passive(&s_retention.instance[i])) {
780                         err = sleep_retention_passive_module_free(i);
781                     }
782                 }
783             }
784         } else {
785             err = ESP_ERR_INVALID_STATE;
786         }
787     } else {
788         err = ESP_ERR_NOT_ALLOWED;
789     }
790     k_sem_give(&s_retention_lock);
791     return err;
792 }
793 
794 #if SOC_PM_RETENTION_HAS_CLOCK_BUG
sleep_retention_do_extra_retention(bool backup_or_restore)795 void IRAM_ATTR sleep_retention_do_extra_retention(bool backup_or_restore)
796 {
797     if (s_retention.highpri < SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY ||
798         s_retention.highpri > SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY) {
799         return;
800     }
801     // Set extra linked list head pointer to hardware
802     pau_regdma_set_extra_link_addr(s_retention.lists[s_retention.highpri].entries[EXTRA_LINK_NUM]);
803     if (backup_or_restore) {
804         pau_regdma_trigger_extra_link_backup();
805     } else {
806         pau_regdma_trigger_extra_link_restore();
807     }
808 }
809 #endif
810 
811 #if SOC_PM_RETENTION_SW_TRIGGER_REGDMA
sleep_retention_do_system_retention(bool backup_or_restore)812 void IRAM_ATTR sleep_retention_do_system_retention(bool backup_or_restore)
813 {
814     #define SYSTEM_LINK_NUM (0)
815     if (s_retention.highpri >= SLEEP_RETENTION_REGDMA_LINK_HIGHEST_PRIORITY &&
816         s_retention.highpri <= SLEEP_RETENTION_REGDMA_LINK_LOWEST_PRIORITY) {
817         // Set extra linked list head pointer to hardware
818         pau_regdma_set_system_link_addr(s_retention.lists[s_retention.highpri].entries[SYSTEM_LINK_NUM]);
819         // When PD TOP, we need to prevent the PMU from triggering the REGDMA backup, because REGDMA will power off
820         pmu_sleep_disable_regdma_backup();
821         if (backup_or_restore) {
822             pau_regdma_trigger_system_link_backup();
823         } else {
824             pau_regdma_trigger_system_link_restore();
825         }
826     }
827 }
828 #endif
829