1 /*
2 * Copyright (c) 2016 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 /**
8 * @file @brief mutex kernel services
9 *
10 * This module contains routines for handling mutex locking and unlocking.
11 *
12 * Mutexes implement a priority inheritance algorithm that boosts the priority
13 * level of the owning thread to match the priority level of the highest
14 * priority thread waiting on the mutex.
15 *
16 * Each mutex that contributes to priority inheritance must be released in the
17 * reverse order in which it was acquired. Furthermore each subsequent mutex
18 * that contributes to raising the owning thread's priority level must be
19 * acquired at a point after the most recent "bumping" of the priority level.
20 *
21 * For example, if thread A has two mutexes contributing to the raising of its
22 * priority level, the second mutex M2 must be acquired by thread A after
23 * thread A's priority level was bumped due to owning the first mutex M1.
24 * When releasing the mutex, thread A must release M2 before it releases M1.
25 * Failure to follow this nested model may result in threads running at
26 * unexpected priority levels (too high, or too low).
27 */
28
29 #include <zephyr/kernel.h>
30 #include <zephyr/kernel_structs.h>
31 #include <zephyr/toolchain.h>
32 #include <ksched.h>
33 #include <wait_q.h>
34 #include <errno.h>
35 #include <zephyr/init.h>
36 #include <zephyr/syscall_handler.h>
37 #include <zephyr/tracing/tracing.h>
38 #include <zephyr/sys/check.h>
39 #include <zephyr/logging/log.h>
40 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
41
42 /* We use a global spinlock here because some of the synchronization
43 * is protecting things like owner thread priorities which aren't
44 * "part of" a single k_mutex. Should move those bits of the API
45 * under the scheduler lock so we can break this up.
46 */
47 static struct k_spinlock lock;
48
49 #ifdef CONFIG_OBJ_CORE_MUTEX
50 static struct k_obj_type obj_type_mutex;
51 #endif
52
z_impl_k_mutex_init(struct k_mutex * mutex)53 int z_impl_k_mutex_init(struct k_mutex *mutex)
54 {
55 mutex->owner = NULL;
56 mutex->lock_count = 0U;
57
58 z_waitq_init(&mutex->wait_q);
59
60 z_object_init(mutex);
61
62 #ifdef CONFIG_OBJ_CORE_MUTEX
63 k_obj_core_init_and_link(K_OBJ_CORE(mutex), &obj_type_mutex);
64 #endif
65
66 SYS_PORT_TRACING_OBJ_INIT(k_mutex, mutex, 0);
67
68 return 0;
69 }
70
71 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_init(struct k_mutex * mutex)72 static inline int z_vrfy_k_mutex_init(struct k_mutex *mutex)
73 {
74 Z_OOPS(Z_SYSCALL_OBJ_INIT(mutex, K_OBJ_MUTEX));
75 return z_impl_k_mutex_init(mutex);
76 }
77 #include <syscalls/k_mutex_init_mrsh.c>
78 #endif
79
new_prio_for_inheritance(int32_t target,int32_t limit)80 static int32_t new_prio_for_inheritance(int32_t target, int32_t limit)
81 {
82 int new_prio = z_is_prio_higher(target, limit) ? target : limit;
83
84 new_prio = z_get_new_prio_with_ceiling(new_prio);
85
86 return new_prio;
87 }
88
adjust_owner_prio(struct k_mutex * mutex,int32_t new_prio)89 static bool adjust_owner_prio(struct k_mutex *mutex, int32_t new_prio)
90 {
91 if (mutex->owner->base.prio != new_prio) {
92
93 LOG_DBG("%p (ready (y/n): %c) prio changed to %d (was %d)",
94 mutex->owner, z_is_thread_ready(mutex->owner) ?
95 'y' : 'n',
96 new_prio, mutex->owner->base.prio);
97
98 return z_set_prio(mutex->owner, new_prio);
99 }
100 return false;
101 }
102
z_impl_k_mutex_lock(struct k_mutex * mutex,k_timeout_t timeout)103 int z_impl_k_mutex_lock(struct k_mutex *mutex, k_timeout_t timeout)
104 {
105 int new_prio;
106 k_spinlock_key_t key;
107 bool resched = false;
108
109 __ASSERT(!arch_is_in_isr(), "mutexes cannot be used inside ISRs");
110
111 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mutex, lock, mutex, timeout);
112
113 key = k_spin_lock(&lock);
114
115 if (likely((mutex->lock_count == 0U) || (mutex->owner == _current))) {
116
117 mutex->owner_orig_prio = (mutex->lock_count == 0U) ?
118 _current->base.prio :
119 mutex->owner_orig_prio;
120
121 mutex->lock_count++;
122 mutex->owner = _current;
123
124 LOG_DBG("%p took mutex %p, count: %d, orig prio: %d",
125 _current, mutex, mutex->lock_count,
126 mutex->owner_orig_prio);
127
128 k_spin_unlock(&lock, key);
129
130 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, 0);
131
132 return 0;
133 }
134
135 if (unlikely(K_TIMEOUT_EQ(timeout, K_NO_WAIT))) {
136 k_spin_unlock(&lock, key);
137
138 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, -EBUSY);
139
140 return -EBUSY;
141 }
142
143 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_mutex, lock, mutex, timeout);
144
145 new_prio = new_prio_for_inheritance(_current->base.prio,
146 mutex->owner->base.prio);
147
148 LOG_DBG("adjusting prio up on mutex %p", mutex);
149
150 if (z_is_prio_higher(new_prio, mutex->owner->base.prio)) {
151 resched = adjust_owner_prio(mutex, new_prio);
152 }
153
154 int got_mutex = z_pend_curr(&lock, key, &mutex->wait_q, timeout);
155
156 LOG_DBG("on mutex %p got_mutex value: %d", mutex, got_mutex);
157
158 LOG_DBG("%p got mutex %p (y/n): %c", _current, mutex,
159 got_mutex ? 'y' : 'n');
160
161 if (got_mutex == 0) {
162 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, 0);
163 return 0;
164 }
165
166 /* timed out */
167
168 LOG_DBG("%p timeout on mutex %p", _current, mutex);
169
170 key = k_spin_lock(&lock);
171
172 /*
173 * Check if mutex was unlocked after this thread was unpended.
174 * If so, skip adjusting owner's priority down.
175 */
176 if (likely(mutex->owner != NULL)) {
177 struct k_thread *waiter = z_waitq_head(&mutex->wait_q);
178
179 new_prio = (waiter != NULL) ?
180 new_prio_for_inheritance(waiter->base.prio, mutex->owner_orig_prio) :
181 mutex->owner_orig_prio;
182
183 LOG_DBG("adjusting prio down on mutex %p", mutex);
184
185 resched = adjust_owner_prio(mutex, new_prio) || resched;
186 }
187
188 if (resched) {
189 z_reschedule(&lock, key);
190 } else {
191 k_spin_unlock(&lock, key);
192 }
193
194 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, lock, mutex, timeout, -EAGAIN);
195
196 return -EAGAIN;
197 }
198
199 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_lock(struct k_mutex * mutex,k_timeout_t timeout)200 static inline int z_vrfy_k_mutex_lock(struct k_mutex *mutex,
201 k_timeout_t timeout)
202 {
203 Z_OOPS(Z_SYSCALL_OBJ(mutex, K_OBJ_MUTEX));
204 return z_impl_k_mutex_lock(mutex, timeout);
205 }
206 #include <syscalls/k_mutex_lock_mrsh.c>
207 #endif
208
z_impl_k_mutex_unlock(struct k_mutex * mutex)209 int z_impl_k_mutex_unlock(struct k_mutex *mutex)
210 {
211 struct k_thread *new_owner;
212
213 __ASSERT(!arch_is_in_isr(), "mutexes cannot be used inside ISRs");
214
215 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mutex, unlock, mutex);
216
217 CHECKIF(mutex->owner == NULL) {
218 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, -EINVAL);
219
220 return -EINVAL;
221 }
222 /*
223 * The current thread does not own the mutex.
224 */
225 CHECKIF(mutex->owner != _current) {
226 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, -EPERM);
227
228 return -EPERM;
229 }
230
231 /*
232 * Attempt to unlock a mutex which is unlocked. mutex->lock_count
233 * cannot be zero if the current thread is equal to mutex->owner,
234 * therefore no underflow check is required. Use assert to catch
235 * undefined behavior.
236 */
237 __ASSERT_NO_MSG(mutex->lock_count > 0U);
238
239 LOG_DBG("mutex %p lock_count: %d", mutex, mutex->lock_count);
240
241 /*
242 * If we are the owner and count is greater than 1, then decrement
243 * the count and return and keep current thread as the owner.
244 */
245 if (mutex->lock_count > 1U) {
246 mutex->lock_count--;
247 goto k_mutex_unlock_return;
248 }
249
250 k_spinlock_key_t key = k_spin_lock(&lock);
251
252 adjust_owner_prio(mutex, mutex->owner_orig_prio);
253
254 /* Get the new owner, if any */
255 new_owner = z_unpend_first_thread(&mutex->wait_q);
256
257 mutex->owner = new_owner;
258
259 LOG_DBG("new owner of mutex %p: %p (prio: %d)",
260 mutex, new_owner, new_owner ? new_owner->base.prio : -1000);
261
262 if (new_owner != NULL) {
263 /*
264 * new owner is already of higher or equal prio than first
265 * waiter since the wait queue is priority-based: no need to
266 * adjust its priority
267 */
268 mutex->owner_orig_prio = new_owner->base.prio;
269 arch_thread_return_value_set(new_owner, 0);
270 z_ready_thread(new_owner);
271 z_reschedule(&lock, key);
272 } else {
273 mutex->lock_count = 0U;
274 k_spin_unlock(&lock, key);
275 }
276
277
278 k_mutex_unlock_return:
279 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mutex, unlock, mutex, 0);
280
281 return 0;
282 }
283
284 #ifdef CONFIG_USERSPACE
z_vrfy_k_mutex_unlock(struct k_mutex * mutex)285 static inline int z_vrfy_k_mutex_unlock(struct k_mutex *mutex)
286 {
287 Z_OOPS(Z_SYSCALL_OBJ(mutex, K_OBJ_MUTEX));
288 return z_impl_k_mutex_unlock(mutex);
289 }
290 #include <syscalls/k_mutex_unlock_mrsh.c>
291 #endif
292
293 #ifdef CONFIG_OBJ_CORE_MUTEX
init_mutex_obj_core_list(void)294 static int init_mutex_obj_core_list(void)
295 {
296 /* Initialize mutex object type */
297
298 z_obj_type_init(&obj_type_mutex, K_OBJ_TYPE_MUTEX_ID,
299 offsetof(struct k_mutex, obj_core));
300
301 /* Initialize and link statically defined mutexs */
302
303 STRUCT_SECTION_FOREACH(k_mutex, mutex) {
304 k_obj_core_init_and_link(K_OBJ_CORE(mutex), &obj_type_mutex);
305 }
306
307 return 0;
308 }
309
310 SYS_INIT(init_mutex_obj_core_list, PRE_KERNEL_1,
311 CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
312 #endif
313
