1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_STATIC_CALL_H
3 #define _LINUX_STATIC_CALL_H
4
5 /*
6 * Static call support
7 *
8 * Static calls use code patching to hard-code function pointers into direct
9 * branch instructions. They give the flexibility of function pointers, but
10 * with improved performance. This is especially important for cases where
11 * retpolines would otherwise be used, as retpolines can significantly impact
12 * performance.
13 *
14 *
15 * API overview:
16 *
17 * DECLARE_STATIC_CALL(name, func);
18 * DEFINE_STATIC_CALL(name, func);
19 * DEFINE_STATIC_CALL_NULL(name, typename);
20 * static_call(name)(args...);
21 * static_call_cond(name)(args...);
22 * static_call_update(name, func);
23 *
24 * Usage example:
25 *
26 * # Start with the following functions (with identical prototypes):
27 * int func_a(int arg1, int arg2);
28 * int func_b(int arg1, int arg2);
29 *
30 * # Define a 'my_name' reference, associated with func_a() by default
31 * DEFINE_STATIC_CALL(my_name, func_a);
32 *
33 * # Call func_a()
34 * static_call(my_name)(arg1, arg2);
35 *
36 * # Update 'my_name' to point to func_b()
37 * static_call_update(my_name, &func_b);
38 *
39 * # Call func_b()
40 * static_call(my_name)(arg1, arg2);
41 *
42 *
43 * Implementation details:
44 *
45 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
46 * Otherwise basic indirect calls are used (with function pointers).
47 *
48 * Each static_call() site calls into a trampoline associated with the name.
49 * The trampoline has a direct branch to the default function. Updates to a
50 * name will modify the trampoline's branch destination.
51 *
52 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
53 * themselves will be patched at runtime to call the functions directly,
54 * rather than calling through the trampoline. This requires objtool or a
55 * compiler plugin to detect all the static_call() sites and annotate them
56 * in the .static_call_sites section.
57 *
58 *
59 * Notes on NULL function pointers:
60 *
61 * Static_call()s support NULL functions, with many of the caveats that
62 * regular function pointers have.
63 *
64 * Clearly calling a NULL function pointer is 'BAD', so too for
65 * static_call()s (although when HAVE_STATIC_CALL it might not be immediately
66 * fatal). A NULL static_call can be the result of:
67 *
68 * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
69 *
70 * which is equivalent to declaring a NULL function pointer with just a
71 * typename:
72 *
73 * void (*my_func_ptr)(int arg1) = NULL;
74 *
75 * or using static_call_update() with a NULL function. In both cases the
76 * HAVE_STATIC_CALL implementation will patch the trampoline with a RET
77 * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
78 * architectures can patch the trampoline call to a NOP.
79 *
80 * In all cases, any argument evaluation is unconditional. Unlike a regular
81 * conditional function pointer call:
82 *
83 * if (my_func_ptr)
84 * my_func_ptr(arg1)
85 *
86 * where the argument evaludation also depends on the pointer value.
87 *
88 * When calling a static_call that can be NULL, use:
89 *
90 * static_call_cond(name)(arg1);
91 *
92 * which will include the required value tests to avoid NULL-pointer
93 * dereferences.
94 */
95
96 #include <linux/types.h>
97 #include <linux/cpu.h>
98 #include <linux/static_call_types.h>
99
100 #ifdef CONFIG_HAVE_STATIC_CALL
101 #include <asm/static_call.h>
102
103 /*
104 * Either @site or @tramp can be NULL.
105 */
106 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
107
108 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
109
110 /*
111 * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
112 * the symbol table so that objtool can reference it when it generates the
113 * .static_call_sites section.
114 */
115 #define __static_call(name) \
116 ({ \
117 __ADDRESSABLE(STATIC_CALL_KEY(name)); \
118 &STATIC_CALL_TRAMP(name); \
119 })
120
121 #else
122 #define STATIC_CALL_TRAMP_ADDR(name) NULL
123 #endif
124
125
126 #define DECLARE_STATIC_CALL(name, func) \
127 extern struct static_call_key STATIC_CALL_KEY(name); \
128 extern typeof(func) STATIC_CALL_TRAMP(name);
129
130 #define static_call_update(name, func) \
131 ({ \
132 BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \
133 __static_call_update(&STATIC_CALL_KEY(name), \
134 STATIC_CALL_TRAMP_ADDR(name), func); \
135 })
136
137 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
138
139 extern int __init static_call_init(void);
140
141 struct static_call_mod {
142 struct static_call_mod *next;
143 struct module *mod; /* for vmlinux, mod == NULL */
144 struct static_call_site *sites;
145 };
146
147 struct static_call_key {
148 void *func;
149 union {
150 /* bit 0: 0 = mods, 1 = sites */
151 unsigned long type;
152 struct static_call_mod *mods;
153 struct static_call_site *sites;
154 };
155 };
156
157 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
158 extern int static_call_mod_init(struct module *mod);
159 extern int static_call_text_reserved(void *start, void *end);
160
161 #define DEFINE_STATIC_CALL(name, _func) \
162 DECLARE_STATIC_CALL(name, _func); \
163 struct static_call_key STATIC_CALL_KEY(name) = { \
164 .func = _func, \
165 .type = 1, \
166 }; \
167 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
168
169 #define DEFINE_STATIC_CALL_NULL(name, _func) \
170 DECLARE_STATIC_CALL(name, _func); \
171 struct static_call_key STATIC_CALL_KEY(name) = { \
172 .func = NULL, \
173 .type = 1, \
174 }; \
175 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
176
177 #define static_call(name) __static_call(name)
178 #define static_call_cond(name) (void)__static_call(name)
179
180 #define EXPORT_STATIC_CALL(name) \
181 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
182 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
183
184 #define EXPORT_STATIC_CALL_GPL(name) \
185 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
186 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
187
188 #elif defined(CONFIG_HAVE_STATIC_CALL)
189
static_call_init(void)190 static inline int static_call_init(void) { return 0; }
191
192 struct static_call_key {
193 void *func;
194 };
195
196 #define DEFINE_STATIC_CALL(name, _func) \
197 DECLARE_STATIC_CALL(name, _func); \
198 struct static_call_key STATIC_CALL_KEY(name) = { \
199 .func = _func, \
200 }; \
201 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
202
203 #define DEFINE_STATIC_CALL_NULL(name, _func) \
204 DECLARE_STATIC_CALL(name, _func); \
205 struct static_call_key STATIC_CALL_KEY(name) = { \
206 .func = NULL, \
207 }; \
208 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
209
210 #define static_call(name) __static_call(name)
211 #define static_call_cond(name) (void)__static_call(name)
212
213 static inline
__static_call_update(struct static_call_key * key,void * tramp,void * func)214 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
215 {
216 cpus_read_lock();
217 WRITE_ONCE(key->func, func);
218 arch_static_call_transform(NULL, tramp, func, false);
219 cpus_read_unlock();
220 }
221
static_call_text_reserved(void * start,void * end)222 static inline int static_call_text_reserved(void *start, void *end)
223 {
224 return 0;
225 }
226
227 #define EXPORT_STATIC_CALL(name) \
228 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
229 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
230
231 #define EXPORT_STATIC_CALL_GPL(name) \
232 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
233 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
234
235 #else /* Generic implementation */
236
static_call_init(void)237 static inline int static_call_init(void) { return 0; }
238
239 struct static_call_key {
240 void *func;
241 };
242
243 #define DEFINE_STATIC_CALL(name, _func) \
244 DECLARE_STATIC_CALL(name, _func); \
245 struct static_call_key STATIC_CALL_KEY(name) = { \
246 .func = _func, \
247 }
248
249 #define DEFINE_STATIC_CALL_NULL(name, _func) \
250 DECLARE_STATIC_CALL(name, _func); \
251 struct static_call_key STATIC_CALL_KEY(name) = { \
252 .func = NULL, \
253 }
254
255 #define static_call(name) \
256 ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
257
__static_call_nop(void)258 static inline void __static_call_nop(void) { }
259
260 /*
261 * This horrific hack takes care of two things:
262 *
263 * - it ensures the compiler will only load the function pointer ONCE,
264 * which avoids a reload race.
265 *
266 * - it ensures the argument evaluation is unconditional, similar
267 * to the HAVE_STATIC_CALL variant.
268 *
269 * Sadly current GCC/Clang (10 for both) do not optimize this properly
270 * and will emit an indirect call for the NULL case :-(
271 */
272 #define __static_call_cond(name) \
273 ({ \
274 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \
275 if (!func) \
276 func = &__static_call_nop; \
277 (typeof(STATIC_CALL_TRAMP(name))*)func; \
278 })
279
280 #define static_call_cond(name) (void)__static_call_cond(name)
281
282 static inline
__static_call_update(struct static_call_key * key,void * tramp,void * func)283 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
284 {
285 WRITE_ONCE(key->func, func);
286 }
287
static_call_text_reserved(void * start,void * end)288 static inline int static_call_text_reserved(void *start, void *end)
289 {
290 return 0;
291 }
292
293 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))
294 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
295
296 #endif /* CONFIG_HAVE_STATIC_CALL */
297
298 #endif /* _LINUX_STATIC_CALL_H */
299
