1 /*
2 * Copyright (c) 2016 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 /**
8 * @file @brief mutex kernel services
9 *
10 * This module contains routines for handling mutex locking and unlocking.
11 *
12 * Mutexes implement a priority inheritance algorithm that boosts the priority
13 * level of the owning thread to match the priority level of the highest
14 * priority thread waiting on the mutex.
15 *
16 * Each mutex that contributes to priority inheritance must be released in the
17 * reverse order in which it was acquired. Furthermore each subsequent mutex
18 * that contributes to raising the owning thread's priority level must be
19 * acquired at a point after the most recent "bumping" of the priority level.
20 *
21 * For example, if thread A has two mutexes contributing to the raising of its
22 * priority level, the second mutex M2 must be acquired by thread A after
23 * thread A's priority level was bumped due to owning the first mutex M1.
24 * When releasing the mutex, thread A must release M2 before it releases M1.
25 * Failure to follow this nested model may result in threads running at
26 * unexpected priority levels (too high, or too low).
27 */
28
29 #include <zephyr/kernel.h>
30 #include <zephyr/kernel_structs.h>
31 #include <zephyr/toolchain.h>
32 #include <ksched.h>
33 #include <wait_q.h>
34 #include <errno.h>
35 #include <zephyr/init.h>
36 #include <zephyr/internal/syscall_handler.h>
37 #include <zephyr/tracing/tracing.h>
38 #include <zephyr/sys/check.h>
39 #include <zephyr/logging/log.h>
40 #include <zephyr/llext/symbol.h>
41 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
42
43 /* We use a global spinlock here because some of the synchronization
44 * is protecting things like owner thread priorities which aren't
45 * "part of" a single k_mutex. Should move those bits of the API
46 * under the scheduler lock so we can break this up.
47 */
48 static struct k_spinlock lock;
49
50 #ifdef CONFIG_OBJ_CORE_MUTEX
51 static struct k_obj_type obj_type_mutex;
52 #endif
53
z_impl_k_mutex_init(struct k_mutex * mutex)54 int z_impl_k_mutex_init(struct k_mutex *mutex)
55 {
56 mutex->owner = NULL;
57 mutex->lock_count = 0U;
58
59 z_waitq_init(&mutex->wait_q);
60
61 k_object_init(mutex);
62
63 #ifdef CONFIG_OBJ_CORE_MUTEX
64 k_obj_core_init_and_link(K_OBJ_CORE(mutex), &obj_type_mutex);
65 #endif
66
67 SYS_PORT_TRACING_OBJ_INIT(k_mutex, mutex, 0);
68
69 return 0;
70 }
71
72 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_init(struct k_mutex * mutex)73 static inline int z_vrfy_k_mutex_init(struct k_mutex *mutex)
74 {
75 K_OOPS(K_SYSCALL_OBJ_INIT(mutex, K_OBJ_MUTEX));
76 return z_impl_k_mutex_init(mutex);
77 }
78 #include <syscalls/k_mutex_init_mrsh.c>
79 #endif
80
new_prio_for_inheritance(int32_t target,int32_t limit)81 static int32_t new_prio_for_inheritance(int32_t target, int32_t limit)
82 {
83 int new_prio = z_is_prio_higher(target, limit) ? target : limit;
84
85 new_prio = z_get_new_prio_with_ceiling(new_prio);
86
87 return new_prio;
88 }
89
adjust_owner_prio(struct k_mutex * mutex,int32_t new_prio)90 static bool adjust_owner_prio(struct k_mutex *mutex, int32_t new_prio)
91 {
92 if (mutex->owner->base.prio != new_prio) {
93
94 LOG_DBG("%p (ready (y/n): %c) prio changed to %d (was %d)",
95 mutex->owner, z_is_thread_ready(mutex->owner) ?
96 'y' : 'n',
97 new_prio, mutex->owner->base.prio);
98
99 return z_set_prio(mutex->owner, new_prio);
100 }
101 return false;
102 }
103
z_impl_k_mutex_lock(struct k_mutex * mutex,k_timeout_t timeout)104 int z_impl_k_mutex_lock(struct k_mutex *mutex, k_timeout_t timeout)
105 {
106 int new_prio;
107 k_spinlock_key_t key;
108 bool resched = false;
109
110 __ASSERT(!arch_is_in_isr(), "mutexes cannot be used inside ISRs");
111
112 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mutex, lock, mutex, timeout);
113
114 key = k_spin_lock(&lock);
115
116 if (likely((mutex->lock_count == 0U) || (mutex->owner == _current))) {
117
118 mutex->owner_orig_prio = (mutex->lock_count == 0U) ?
119 _current->base.prio :
120 mutex->owner_orig_prio;
121
122 mutex->lock_count++;
123 mutex->owner = _current;
124
125 LOG_DBG("%p took mutex %p, count: %d, orig prio: %d",
126 _current, mutex, mutex->lock_count,
127 mutex->owner_orig_prio);
128
129 k_spin_unlock(&lock, key);
130
131 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, 0);
132
133 return 0;
134 }
135
136 if (unlikely(K_TIMEOUT_EQ(timeout, K_NO_WAIT))) {
137 k_spin_unlock(&lock, key);
138
139 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, -EBUSY);
140
141 return -EBUSY;
142 }
143
144 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_mutex, lock, mutex, timeout);
145
146 new_prio = new_prio_for_inheritance(_current->base.prio,
147 mutex->owner->base.prio);
148
149 LOG_DBG("adjusting prio up on mutex %p", mutex);
150
151 if (z_is_prio_higher(new_prio, mutex->owner->base.prio)) {
152 resched = adjust_owner_prio(mutex, new_prio);
153 }
154
155 int got_mutex = z_pend_curr(&lock, key, &mutex->wait_q, timeout);
156
157 LOG_DBG("on mutex %p got_mutex value: %d", mutex, got_mutex);
158
159 LOG_DBG("%p got mutex %p (y/n): %c", _current, mutex,
160 got_mutex ? 'y' : 'n');
161
162 if (got_mutex == 0) {
163 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, 0);
164 return 0;
165 }
166
167 /* timed out */
168
169 LOG_DBG("%p timeout on mutex %p", _current, mutex);
170
171 key = k_spin_lock(&lock);
172
173 /*
174 * Check if mutex was unlocked after this thread was unpended.
175 * If so, skip adjusting owner's priority down.
176 */
177 if (likely(mutex->owner != NULL)) {
178 struct k_thread *waiter = z_waitq_head(&mutex->wait_q);
179
180 new_prio = (waiter != NULL) ?
181 new_prio_for_inheritance(waiter->base.prio, mutex->owner_orig_prio) :
182 mutex->owner_orig_prio;
183
184 LOG_DBG("adjusting prio down on mutex %p", mutex);
185
186 resched = adjust_owner_prio(mutex, new_prio) || resched;
187 }
188
189 if (resched) {
190 z_reschedule(&lock, key);
191 } else {
192 k_spin_unlock(&lock, key);
193 }
194
195 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, -EAGAIN);
196
197 return -EAGAIN;
198 }
199 EXPORT_SYSCALL(k_mutex_lock);
200
201 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_lock(struct k_mutex * mutex,k_timeout_t timeout)202 static inline int z_vrfy_k_mutex_lock(struct k_mutex *mutex,
203 k_timeout_t timeout)
204 {
205 K_OOPS(K_SYSCALL_OBJ(mutex, K_OBJ_MUTEX));
206 return z_impl_k_mutex_lock(mutex, timeout);
207 }
208 #include <syscalls/k_mutex_lock_mrsh.c>
209 #endif
210
z_impl_k_mutex_unlock(struct k_mutex * mutex)211 int z_impl_k_mutex_unlock(struct k_mutex *mutex)
212 {
213 struct k_thread *new_owner;
214
215 __ASSERT(!arch_is_in_isr(), "mutexes cannot be used inside ISRs");
216
217 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mutex, unlock, mutex);
218
219 CHECKIF(mutex->owner == NULL) {
220 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, -EINVAL);
221
222 return -EINVAL;
223 }
224 /*
225 * The current thread does not own the mutex.
226 */
227 CHECKIF(mutex->owner != _current) {
228 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, -EPERM);
229
230 return -EPERM;
231 }
232
233 /*
234 * Attempt to unlock a mutex which is unlocked. mutex->lock_count
235 * cannot be zero if the current thread is equal to mutex->owner,
236 * therefore no underflow check is required. Use assert to catch
237 * undefined behavior.
238 */
239 __ASSERT_NO_MSG(mutex->lock_count > 0U);
240
241 LOG_DBG("mutex %p lock_count: %d", mutex, mutex->lock_count);
242
243 /*
244 * If we are the owner and count is greater than 1, then decrement
245 * the count and return and keep current thread as the owner.
246 */
247 if (mutex->lock_count > 1U) {
248 mutex->lock_count--;
249 goto k_mutex_unlock_return;
250 }
251
252 k_spinlock_key_t key = k_spin_lock(&lock);
253
254 adjust_owner_prio(mutex, mutex->owner_orig_prio);
255
256 /* Get the new owner, if any */
257 new_owner = z_unpend_first_thread(&mutex->wait_q);
258
259 mutex->owner = new_owner;
260
261 LOG_DBG("new owner of mutex %p: %p (prio: %d)",
262 mutex, new_owner, new_owner ? new_owner->base.prio : -1000);
263
264 if (new_owner != NULL) {
265 /*
266 * new owner is already of higher or equal prio than first
267 * waiter since the wait queue is priority-based: no need to
268 * adjust its priority
269 */
270 mutex->owner_orig_prio = new_owner->base.prio;
271 arch_thread_return_value_set(new_owner, 0);
272 z_ready_thread(new_owner);
273 z_reschedule(&lock, key);
274 } else {
275 mutex->lock_count = 0U;
276 k_spin_unlock(&lock, key);
277 }
278
279
280 k_mutex_unlock_return:
281 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, 0);
282
283 return 0;
284 }
285 EXPORT_SYSCALL(k_mutex_unlock);
286
287 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_unlock(struct k_mutex * mutex)288 static inline int z_vrfy_k_mutex_unlock(struct k_mutex *mutex)
289 {
290 K_OOPS(K_SYSCALL_OBJ(mutex, K_OBJ_MUTEX));
291 return z_impl_k_mutex_unlock(mutex);
292 }
293 #include <syscalls/k_mutex_unlock_mrsh.c>
294 #endif
295
296 #ifdef CONFIG_OBJ_CORE_MUTEX
init_mutex_obj_core_list(void)297 static int init_mutex_obj_core_list(void)
298 {
299 /* Initialize mutex object type */
300
301 z_obj_type_init(&obj_type_mutex, K_OBJ_TYPE_MUTEX_ID,
302 offsetof(struct k_mutex, obj_core));
303
304 /* Initialize and link statically defined mutexs */
305
306 STRUCT_SECTION_FOREACH(k_mutex, mutex) {
307 k_obj_core_init_and_link(K_OBJ_CORE(mutex), &obj_type_mutex);
308 }
309
310 return 0;
311 }
312
313 SYS_INIT(init_mutex_obj_core_list, PRE_KERNEL_1,
314 CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
315 #endif
316
