1 /**
2 * Copyright (c) 2024 Intel Corporation
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6 #include <zephyr/kernel.h>
7 #include <kswap.h>
8 #include <ksched.h>
9 #include <ipi.h>
10
11 #ifdef CONFIG_SCHED_IPI_SUPPORTED
12 static struct k_spinlock ipi_lock;
13 #endif
14
15 #ifdef CONFIG_TRACE_SCHED_IPI
16 extern void z_trace_sched_ipi(void);
17 #endif
18
flag_ipi(uint32_t ipi_mask)19 void flag_ipi(uint32_t ipi_mask)
20 {
21 #if defined(CONFIG_SCHED_IPI_SUPPORTED)
22 if (arch_num_cpus() > 1) {
23 atomic_or(&_kernel.pending_ipi, (atomic_val_t)ipi_mask);
24 }
25 #endif /* CONFIG_SCHED_IPI_SUPPORTED */
26 }
27
28 /* Create a bitmask of CPUs that need an IPI. Note: sched_spinlock is held. */
ipi_mask_create(struct k_thread * thread)29 atomic_val_t ipi_mask_create(struct k_thread *thread)
30 {
31 if (!IS_ENABLED(CONFIG_IPI_OPTIMIZE)) {
32 return (CONFIG_MP_MAX_NUM_CPUS > 1) ? IPI_ALL_CPUS_MASK : 0;
33 }
34
35 uint32_t ipi_mask = 0;
36 uint32_t num_cpus = (uint32_t)arch_num_cpus();
37 uint32_t id = _current_cpu->id;
38 struct k_thread *cpu_thread;
39 bool executable_on_cpu = true;
40
41 for (uint32_t i = 0; i < num_cpus; i++) {
42 if (id == i) {
43 continue;
44 }
45
46 /*
47 * An IPI absolutely does not need to be sent if ...
48 * 1. the CPU is not active, or
49 * 2. <thread> can not execute on the target CPU
50 * ... and might not need to be sent if ...
51 * 3. the target CPU's active thread is not preemptible, or
52 * 4. the target CPU's active thread has a higher priority
53 * (Items 3 & 4 may be overridden by a metaIRQ thread)
54 */
55
56 #if defined(CONFIG_SCHED_CPU_MASK)
57 executable_on_cpu = ((thread->base.cpu_mask & BIT(i)) != 0);
58 #endif
59
60 cpu_thread = _kernel.cpus[i].current;
61 if ((cpu_thread != NULL) &&
62 (((z_sched_prio_cmp(cpu_thread, thread) < 0) &&
63 (thread_is_preemptible(cpu_thread))) ||
64 thread_is_metairq(thread)) && executable_on_cpu) {
65 ipi_mask |= BIT(i);
66 }
67 }
68
69 return (atomic_val_t)ipi_mask;
70 }
71
signal_pending_ipi(void)72 void signal_pending_ipi(void)
73 {
74 /* Synchronization note: you might think we need to lock these
75 * two steps, but an IPI is idempotent. It's OK if we do it
76 * twice. All we require is that if a CPU sees the flag true,
77 * it is guaranteed to send the IPI, and if a core sets
78 * pending_ipi, the IPI will be sent the next time through
79 * this code.
80 */
81
82 #if defined(CONFIG_SCHED_IPI_SUPPORTED)
83 if (arch_num_cpus() > 1) {
84 uint32_t cpu_bitmap;
85
86 cpu_bitmap = (uint32_t)atomic_clear(&_kernel.pending_ipi);
87 if (cpu_bitmap != 0) {
88 #ifdef CONFIG_ARCH_HAS_DIRECTED_IPIS
89 arch_sched_directed_ipi(cpu_bitmap);
90 #else
91 arch_sched_broadcast_ipi();
92 #endif
93 }
94 }
95 #endif /* CONFIG_SCHED_IPI_SUPPORTED */
96 }
97
98 #ifdef CONFIG_SCHED_IPI_SUPPORTED
first_ipi_work(sys_dlist_t * list)99 static struct k_ipi_work *first_ipi_work(sys_dlist_t *list)
100 {
101 sys_dnode_t *work = sys_dlist_peek_head(list);
102 unsigned int cpu_id = _current_cpu->id;
103
104 return (work == NULL) ? NULL
105 : CONTAINER_OF(work, struct k_ipi_work, node[cpu_id]);
106 }
107
k_ipi_work_add(struct k_ipi_work * work,uint32_t cpu_bitmask,k_ipi_func_t func)108 int k_ipi_work_add(struct k_ipi_work *work, uint32_t cpu_bitmask,
109 k_ipi_func_t func)
110 {
111 __ASSERT(work != NULL, "");
112 __ASSERT(func != NULL, "");
113
114 k_spinlock_key_t key = k_spin_lock(&ipi_lock);
115
116 /* Verify the IPI work item is not currently in use */
117
118 if (k_event_wait_all(&work->event, work->bitmask,
119 false, K_NO_WAIT) != work->bitmask) {
120 k_spin_unlock(&ipi_lock, key);
121 return -EBUSY;
122 }
123
124 /*
125 * Add the IPI work item to the list(s)--but not for the current
126 * CPU as the architecture may not support sending an IPI to itself.
127 */
128
129 unsigned int cpu_id = _current_cpu->id;
130
131 cpu_bitmask &= (IPI_ALL_CPUS_MASK & ~BIT(cpu_id));
132
133 k_event_clear(&work->event, IPI_ALL_CPUS_MASK);
134 work->func = func;
135 work->bitmask = cpu_bitmask;
136
137 for (unsigned int id = 0; id < arch_num_cpus(); id++) {
138 if ((cpu_bitmask & BIT(id)) != 0) {
139 sys_dlist_append(&_kernel.cpus[id].ipi_workq, &work->node[id]);
140 }
141 }
142
143 flag_ipi(cpu_bitmask);
144
145 k_spin_unlock(&ipi_lock, key);
146
147 return 0;
148 }
149
k_ipi_work_wait(struct k_ipi_work * work,k_timeout_t timeout)150 int k_ipi_work_wait(struct k_ipi_work *work, k_timeout_t timeout)
151 {
152 uint32_t rv = k_event_wait_all(&work->event, work->bitmask,
153 false, timeout);
154
155 return (rv == 0) ? -EAGAIN : 0;
156 }
157
k_ipi_work_signal(void)158 void k_ipi_work_signal(void)
159 {
160 signal_pending_ipi();
161 }
162
ipi_work_process(sys_dlist_t * list)163 static void ipi_work_process(sys_dlist_t *list)
164 {
165 unsigned int cpu_id = _current_cpu->id;
166
167 k_spinlock_key_t key = k_spin_lock(&ipi_lock);
168
169 for (struct k_ipi_work *work = first_ipi_work(list);
170 work != NULL; work = first_ipi_work(list)) {
171 sys_dlist_remove(&work->node[cpu_id]);
172 k_spin_unlock(&ipi_lock, key);
173
174 work->func(work);
175
176 key = k_spin_lock(&ipi_lock);
177 k_event_post(&work->event, BIT(cpu_id));
178 }
179
180 k_spin_unlock(&ipi_lock, key);
181 }
182 #endif /* CONFIG_SCHED_IPI_SUPPORTED */
183
z_sched_ipi(void)184 void z_sched_ipi(void)
185 {
186 /* NOTE: When adding code to this, make sure this is called
187 * at appropriate location when !CONFIG_SCHED_IPI_SUPPORTED.
188 */
189 #ifdef CONFIG_TRACE_SCHED_IPI
190 z_trace_sched_ipi();
191 #endif /* CONFIG_TRACE_SCHED_IPI */
192
193 #ifdef CONFIG_TIMESLICING
194 if (thread_is_sliceable(_current)) {
195 z_time_slice();
196 }
197 #endif /* CONFIG_TIMESLICING */
198
199 #ifdef CONFIG_ARCH_IPI_LAZY_COPROCESSORS_SAVE
200 arch_ipi_lazy_coprocessors_save();
201 #endif
202
203 #ifdef CONFIG_SCHED_IPI_SUPPORTED
204 ipi_work_process(&_kernel.cpus[_current_cpu->id].ipi_workq);
205 #endif
206 }
207
