1 /*
2 * Copyright (c) 2016 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 /**
8 * @file @brief mutex kernel services
9 *
10 * This module contains routines for handling mutex locking and unlocking.
11 *
12 * Mutexes implement a priority inheritance algorithm that boosts the priority
13 * level of the owning thread to match the priority level of the highest
14 * priority thread waiting on the mutex.
15 *
16 * Each mutex that contributes to priority inheritance must be released in the
17 * reverse order in which it was acquired. Furthermore each subsequent mutex
18 * that contributes to raising the owning thread's priority level must be
19 * acquired at a point after the most recent "bumping" of the priority level.
20 *
21 * For example, if thread A has two mutexes contributing to the raising of its
22 * priority level, the second mutex M2 must be acquired by thread A after
23 * thread A's priority level was bumped due to owning the first mutex M1.
24 * When releasing the mutex, thread A must release M2 before it releases M1.
25 * Failure to follow this nested model may result in threads running at
26 * unexpected priority levels (too high, or too low).
27 */
28
29 #include <zephyr/kernel.h>
30 #include <zephyr/kernel_structs.h>
31 #include <zephyr/toolchain.h>
32 #include <ksched.h>
33 #include <kthread.h>
34 #include <wait_q.h>
35 #include <errno.h>
36 #include <zephyr/init.h>
37 #include <zephyr/internal/syscall_handler.h>
38 #include <zephyr/tracing/tracing.h>
39 #include <zephyr/sys/check.h>
40 #include <zephyr/logging/log.h>
41 #include <zephyr/llext/symbol.h>
42 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
43
44 /* We use a global spinlock here because some of the synchronization
45 * is protecting things like owner thread priorities which aren't
46 * "part of" a single k_mutex. Should move those bits of the API
47 * under the scheduler lock so we can break this up.
48 */
49 static struct k_spinlock lock;
50
51 #ifdef CONFIG_OBJ_CORE_MUTEX
52 static struct k_obj_type obj_type_mutex;
53 #endif /* CONFIG_OBJ_CORE_MUTEX */
54
z_impl_k_mutex_init(struct k_mutex * mutex)55 int z_impl_k_mutex_init(struct k_mutex *mutex)
56 {
57 mutex->owner = NULL;
58 mutex->lock_count = 0U;
59
60 z_waitq_init(&mutex->wait_q);
61
62 k_object_init(mutex);
63
64 #ifdef CONFIG_OBJ_CORE_MUTEX
65 k_obj_core_init_and_link(K_OBJ_CORE(mutex), &obj_type_mutex);
66 #endif /* CONFIG_OBJ_CORE_MUTEX */
67
68 SYS_PORT_TRACING_OBJ_INIT(k_mutex, mutex, 0);
69
70 return 0;
71 }
72
73 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_init(struct k_mutex * mutex)74 static inline int z_vrfy_k_mutex_init(struct k_mutex *mutex)
75 {
76 K_OOPS(K_SYSCALL_OBJ_INIT(mutex, K_OBJ_MUTEX));
77 return z_impl_k_mutex_init(mutex);
78 }
79 #include <zephyr/syscalls/k_mutex_init_mrsh.c>
80 #endif /* CONFIG_USERSPACE */
81
new_prio_for_inheritance(int32_t target,int32_t limit)82 static int32_t new_prio_for_inheritance(int32_t target, int32_t limit)
83 {
84 int new_prio = z_is_prio_higher(target, limit) ? target : limit;
85
86 new_prio = z_get_new_prio_with_ceiling(new_prio);
87
88 return new_prio;
89 }
90
adjust_owner_prio(struct k_mutex * mutex,int32_t new_prio)91 static bool adjust_owner_prio(struct k_mutex *mutex, int32_t new_prio)
92 {
93 if (mutex->owner->base.prio != new_prio) {
94
95 LOG_DBG("%p (ready (y/n): %c) prio changed to %d (was %d)",
96 mutex->owner, z_is_thread_ready(mutex->owner) ?
97 'y' : 'n',
98 new_prio, mutex->owner->base.prio);
99
100 return z_thread_prio_set(mutex->owner, new_prio);
101 }
102 return false;
103 }
104
z_impl_k_mutex_lock(struct k_mutex * mutex,k_timeout_t timeout)105 int z_impl_k_mutex_lock(struct k_mutex *mutex, k_timeout_t timeout)
106 {
107 int new_prio;
108 k_spinlock_key_t key;
109 bool resched = false;
110
111 __ASSERT(!arch_is_in_isr(), "mutexes cannot be used inside ISRs");
112
113 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mutex, lock, mutex, timeout);
114
115 key = k_spin_lock(&lock);
116
117 if (likely((mutex->lock_count == 0U) || (mutex->owner == _current))) {
118
119 mutex->owner_orig_prio = (mutex->lock_count == 0U) ?
120 _current->base.prio :
121 mutex->owner_orig_prio;
122
123 mutex->lock_count++;
124 mutex->owner = _current;
125
126 LOG_DBG("%p took mutex %p, count: %d, orig prio: %d",
127 _current, mutex, mutex->lock_count,
128 mutex->owner_orig_prio);
129
130 k_spin_unlock(&lock, key);
131
132 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, 0);
133
134 return 0;
135 }
136
137 if (unlikely(K_TIMEOUT_EQ(timeout, K_NO_WAIT))) {
138 k_spin_unlock(&lock, key);
139
140 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, -EBUSY);
141
142 return -EBUSY;
143 }
144
145 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_mutex, lock, mutex, timeout);
146
147 new_prio = new_prio_for_inheritance(_current->base.prio,
148 mutex->owner->base.prio);
149
150 LOG_DBG("adjusting prio up on mutex %p", mutex);
151
152 if (z_is_prio_higher(new_prio, mutex->owner->base.prio)) {
153 resched = adjust_owner_prio(mutex, new_prio);
154 }
155
156 int got_mutex = z_pend_curr(&lock, key, &mutex->wait_q, timeout);
157
158 LOG_DBG("on mutex %p got_mutex value: %d", mutex, got_mutex);
159
160 LOG_DBG("%p got mutex %p (y/n): %c", _current, mutex,
161 got_mutex ? 'y' : 'n');
162
163 if (got_mutex == 0) {
164 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, 0);
165 return 0;
166 }
167
168 /* timed out */
169
170 LOG_DBG("%p timeout on mutex %p", _current, mutex);
171
172 key = k_spin_lock(&lock);
173
174 /*
175 * Check if mutex was unlocked after this thread was unpended.
176 * If so, skip adjusting owner's priority down.
177 */
178 if (likely(mutex->owner != NULL)) {
179 struct k_thread *waiter = z_waitq_head(&mutex->wait_q);
180
181 new_prio = (waiter != NULL) ?
182 new_prio_for_inheritance(waiter->base.prio, mutex->owner_orig_prio) :
183 mutex->owner_orig_prio;
184
185 LOG_DBG("adjusting prio down on mutex %p", mutex);
186
187 resched = adjust_owner_prio(mutex, new_prio) || resched;
188 }
189
190 if (resched) {
191 z_reschedule(&lock, key);
192 } else {
193 k_spin_unlock(&lock, key);
194 }
195
196 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, -EAGAIN);
197
198 return -EAGAIN;
199 }
200
201 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_lock(struct k_mutex * mutex,k_timeout_t timeout)202 static inline int z_vrfy_k_mutex_lock(struct k_mutex *mutex,
203 k_timeout_t timeout)
204 {
205 K_OOPS(K_SYSCALL_OBJ(mutex, K_OBJ_MUTEX));
206 return z_impl_k_mutex_lock(mutex, timeout);
207 }
208 #include <zephyr/syscalls/k_mutex_lock_mrsh.c>
209 #endif /* CONFIG_USERSPACE */
210
z_impl_k_mutex_unlock(struct k_mutex * mutex)211 int z_impl_k_mutex_unlock(struct k_mutex *mutex)
212 {
213 struct k_thread *new_owner;
214
215 __ASSERT(!arch_is_in_isr(), "mutexes cannot be used inside ISRs");
216
217 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mutex, unlock, mutex);
218
219 CHECKIF(mutex->owner == NULL) {
220 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, -EINVAL);
221
222 return -EINVAL;
223 }
224 /*
225 * The current thread does not own the mutex.
226 */
227 CHECKIF(mutex->owner != _current) {
228 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, -EPERM);
229
230 return -EPERM;
231 }
232
233 /*
234 * Attempt to unlock a mutex which is unlocked. mutex->lock_count
235 * cannot be zero if the current thread is equal to mutex->owner,
236 * therefore no underflow check is required. Use assert to catch
237 * undefined behavior.
238 */
239 __ASSERT_NO_MSG(mutex->lock_count > 0U);
240
241 LOG_DBG("mutex %p lock_count: %d", mutex, mutex->lock_count);
242
243 /*
244 * If we are the owner and count is greater than 1, then decrement
245 * the count and return and keep current thread as the owner.
246 */
247 if (mutex->lock_count > 1U) {
248 mutex->lock_count--;
249 goto k_mutex_unlock_return;
250 }
251
252 k_spinlock_key_t key = k_spin_lock(&lock);
253
254 adjust_owner_prio(mutex, mutex->owner_orig_prio);
255
256 /* Get the new owner, if any */
257 new_owner = z_unpend_first_thread(&mutex->wait_q);
258
259 mutex->owner = new_owner;
260
261 LOG_DBG("new owner of mutex %p: %p (prio: %d)",
262 mutex, new_owner, new_owner ? new_owner->base.prio : -1000);
263
264 if (new_owner != NULL) {
265 /*
266 * new owner is already of higher or equal prio than first
267 * waiter since the wait queue is priority-based: no need to
268 * adjust its priority
269 */
270 mutex->owner_orig_prio = new_owner->base.prio;
271 arch_thread_return_value_set(new_owner, 0);
272 z_ready_thread(new_owner);
273 z_reschedule(&lock, key);
274 } else {
275 mutex->lock_count = 0U;
276 k_spin_unlock(&lock, key);
277 }
278
279
280 k_mutex_unlock_return:
281 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, 0);
282
283 return 0;
284 }
285
286 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_unlock(struct k_mutex * mutex)287 static inline int z_vrfy_k_mutex_unlock(struct k_mutex *mutex)
288 {
289 K_OOPS(K_SYSCALL_OBJ(mutex, K_OBJ_MUTEX));
290 return z_impl_k_mutex_unlock(mutex);
291 }
292 #include <zephyr/syscalls/k_mutex_unlock_mrsh.c>
293 #endif /* CONFIG_USERSPACE */
294
295 #ifdef CONFIG_OBJ_CORE_MUTEX
init_mutex_obj_core_list(void)296 static int init_mutex_obj_core_list(void)
297 {
298 /* Initialize mutex object type */
299
300 z_obj_type_init(&obj_type_mutex, K_OBJ_TYPE_MUTEX_ID,
301 offsetof(struct k_mutex, obj_core));
302
303 /* Initialize and link statically defined mutexes */
304
305 STRUCT_SECTION_FOREACH(k_mutex, mutex) {
306 k_obj_core_init_and_link(K_OBJ_CORE(mutex), &obj_type_mutex);
307 }
308
309 return 0;
310 }
311
312 SYS_INIT(init_mutex_obj_core_list, PRE_KERNEL_1,
313 CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
314 #endif /* CONFIG_OBJ_CORE_MUTEX */
315
