1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
3
4 #include <errno.h>
5 #include <string.h>
6 #include <linux/bpf.h>
7 #include <bpf/bpf_helpers.h>
8 #include "bpf_misc.h"
9
10 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
11
12 int vals[] SEC(".data.vals") = {1, 2, 3, 4};
13
14 __naked __noinline __used
identity_subprog()15 static unsigned long identity_subprog()
16 {
17 /* the simplest *static* 64-bit identity function */
18 asm volatile (
19 "r0 = r1;"
20 "exit;"
21 );
22 }
23
24 __noinline __used
global_identity_subprog(__u64 x)25 unsigned long global_identity_subprog(__u64 x)
26 {
27 /* the simplest *global* 64-bit identity function */
28 return x;
29 }
30
31 __naked __noinline __used
callback_subprog()32 static unsigned long callback_subprog()
33 {
34 /* the simplest callback function */
35 asm volatile (
36 "r0 = 0;"
37 "exit;"
38 );
39 }
40
41 SEC("?raw_tp")
42 __success __log_level(2)
43 __msg("7: (0f) r1 += r0")
44 __msg("mark_precise: frame0: regs=r0 stack= before 6: (bf) r1 = r7")
45 __msg("mark_precise: frame0: regs=r0 stack= before 5: (27) r0 *= 4")
46 __msg("mark_precise: frame0: regs=r0 stack= before 11: (95) exit")
47 __msg("mark_precise: frame1: regs=r0 stack= before 10: (bf) r0 = r1")
48 __msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+5")
49 __msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6")
50 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3")
subprog_result_precise(void)51 __naked int subprog_result_precise(void)
52 {
53 asm volatile (
54 "r6 = 3;"
55 /* pass r6 through r1 into subprog to get it back as r0;
56 * this whole chain will have to be marked as precise later
57 */
58 "r1 = r6;"
59 "call identity_subprog;"
60 /* now use subprog's returned value (which is a
61 * r6 -> r1 -> r0 chain), as index into vals array, forcing
62 * all of that to be known precisely
63 */
64 "r0 *= 4;"
65 "r1 = %[vals];"
66 /* here r0->r1->r6 chain is forced to be precise and has to be
67 * propagated back to the beginning, including through the
68 * subprog call
69 */
70 "r1 += r0;"
71 "r0 = *(u32 *)(r1 + 0);"
72 "exit;"
73 :
74 : __imm_ptr(vals)
75 : __clobber_common, "r6"
76 );
77 }
78
79 SEC("?raw_tp")
80 __success __log_level(2)
81 __msg("9: (0f) r1 += r0")
82 __msg("mark_precise: frame0: last_idx 9 first_idx 0")
83 __msg("mark_precise: frame0: regs=r0 stack= before 8: (bf) r1 = r7")
84 __msg("mark_precise: frame0: regs=r0 stack= before 7: (27) r0 *= 4")
85 __msg("mark_precise: frame0: regs=r0 stack= before 5: (a5) if r0 < 0x4 goto pc+1")
86 __msg("mark_precise: frame0: regs=r0 stack= before 4: (85) call pc+7")
global_subprog_result_precise(void)87 __naked int global_subprog_result_precise(void)
88 {
89 asm volatile (
90 "r6 = 3;"
91 /* pass r6 through r1 into subprog to get it back as r0;
92 * given global_identity_subprog is global, precision won't
93 * propagate all the way back to r6
94 */
95 "r1 = r6;"
96 "call global_identity_subprog;"
97 /* now use subprog's returned value (which is unknown now, so
98 * we need to clamp it), as index into vals array, forcing r0
99 * to be marked precise (with no effect on r6, though)
100 */
101 "if r0 < %[vals_arr_sz] goto 1f;"
102 "r0 = %[vals_arr_sz] - 1;"
103 "1:"
104 "r0 *= 4;"
105 "r1 = %[vals];"
106 /* here r0 is forced to be precise and has to be
107 * propagated back to the global subprog call, but it
108 * shouldn't go all the way to mark r6 as precise
109 */
110 "r1 += r0;"
111 "r0 = *(u32 *)(r1 + 0);"
112 "exit;"
113 :
114 : __imm_ptr(vals),
115 __imm_const(vals_arr_sz, ARRAY_SIZE(vals))
116 : __clobber_common, "r6"
117 );
118 }
119
120 SEC("?raw_tp")
121 __success __log_level(2)
122 __msg("14: (0f) r1 += r6")
123 __msg("mark_precise: frame0: last_idx 14 first_idx 10")
124 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7")
125 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4")
126 __msg("mark_precise: frame0: regs=r6 stack= before 11: (25) if r6 > 0x3 goto pc+4")
127 __msg("mark_precise: frame0: regs=r6 stack= before 10: (bf) r6 = r0")
128 __msg("mark_precise: frame0: parent state regs=r0 stack=:")
129 __msg("mark_precise: frame0: last_idx 18 first_idx 0")
130 __msg("mark_precise: frame0: regs=r0 stack= before 18: (95) exit")
callback_result_precise(void)131 __naked int callback_result_precise(void)
132 {
133 asm volatile (
134 "r6 = 3;"
135
136 /* call subprog and use result; r0 shouldn't propagate back to
137 * callback_subprog
138 */
139 "r1 = r6;" /* nr_loops */
140 "r2 = %[callback_subprog];" /* callback_fn */
141 "r3 = 0;" /* callback_ctx */
142 "r4 = 0;" /* flags */
143 "call %[bpf_loop];"
144
145 "r6 = r0;"
146 "if r6 > 3 goto 1f;"
147 "r6 *= 4;"
148 "r1 = %[vals];"
149 /* here r6 is forced to be precise and has to be propagated
150 * back to the bpf_loop() call, but not beyond
151 */
152 "r1 += r6;"
153 "r0 = *(u32 *)(r1 + 0);"
154 "1:"
155 "exit;"
156 :
157 : __imm_ptr(vals),
158 __imm_ptr(callback_subprog),
159 __imm(bpf_loop)
160 : __clobber_common, "r6"
161 );
162 }
163
164 SEC("?raw_tp")
165 __success __log_level(2)
166 __msg("7: (0f) r1 += r6")
167 __msg("mark_precise: frame0: last_idx 7 first_idx 0")
168 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7")
169 __msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4")
170 __msg("mark_precise: frame0: regs=r6 stack= before 11: (95) exit")
171 __msg("mark_precise: frame1: regs= stack= before 10: (bf) r0 = r1")
172 __msg("mark_precise: frame1: regs= stack= before 4: (85) call pc+5")
173 __msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0")
174 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3")
parent_callee_saved_reg_precise(void)175 __naked int parent_callee_saved_reg_precise(void)
176 {
177 asm volatile (
178 "r6 = 3;"
179
180 /* call subprog and ignore result; we need this call only to
181 * complicate jump history
182 */
183 "r1 = 0;"
184 "call identity_subprog;"
185
186 "r6 *= 4;"
187 "r1 = %[vals];"
188 /* here r6 is forced to be precise and has to be propagated
189 * back to the beginning, handling (and ignoring) subprog call
190 */
191 "r1 += r6;"
192 "r0 = *(u32 *)(r1 + 0);"
193 "exit;"
194 :
195 : __imm_ptr(vals)
196 : __clobber_common, "r6"
197 );
198 }
199
200 SEC("?raw_tp")
201 __success __log_level(2)
202 __msg("7: (0f) r1 += r6")
203 __msg("mark_precise: frame0: last_idx 7 first_idx 0")
204 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7")
205 __msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4")
206 __msg("mark_precise: frame0: regs=r6 stack= before 4: (85) call pc+5")
207 __msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0")
208 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3")
parent_callee_saved_reg_precise_global(void)209 __naked int parent_callee_saved_reg_precise_global(void)
210 {
211 asm volatile (
212 "r6 = 3;"
213
214 /* call subprog and ignore result; we need this call only to
215 * complicate jump history
216 */
217 "r1 = 0;"
218 "call global_identity_subprog;"
219
220 "r6 *= 4;"
221 "r1 = %[vals];"
222 /* here r6 is forced to be precise and has to be propagated
223 * back to the beginning, handling (and ignoring) subprog call
224 */
225 "r1 += r6;"
226 "r0 = *(u32 *)(r1 + 0);"
227 "exit;"
228 :
229 : __imm_ptr(vals)
230 : __clobber_common, "r6"
231 );
232 }
233
234 SEC("?raw_tp")
235 __success __log_level(2)
236 __msg("12: (0f) r1 += r6")
237 __msg("mark_precise: frame0: last_idx 12 first_idx 10")
238 __msg("mark_precise: frame0: regs=r6 stack= before 11: (bf) r1 = r7")
239 __msg("mark_precise: frame0: regs=r6 stack= before 10: (27) r6 *= 4")
240 __msg("mark_precise: frame0: parent state regs=r6 stack=:")
241 __msg("mark_precise: frame0: last_idx 16 first_idx 0")
242 __msg("mark_precise: frame0: regs=r6 stack= before 16: (95) exit")
243 __msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0")
244 __msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop#181")
245 __msg("mark_precise: frame0: regs=r6 stack= before 8: (b7) r4 = 0")
246 __msg("mark_precise: frame0: regs=r6 stack= before 7: (b7) r3 = 0")
247 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r2 = r8")
248 __msg("mark_precise: frame0: regs=r6 stack= before 5: (b7) r1 = 1")
249 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3")
parent_callee_saved_reg_precise_with_callback(void)250 __naked int parent_callee_saved_reg_precise_with_callback(void)
251 {
252 asm volatile (
253 "r6 = 3;"
254
255 /* call subprog and ignore result; we need this call only to
256 * complicate jump history
257 */
258 "r1 = 1;" /* nr_loops */
259 "r2 = %[callback_subprog];" /* callback_fn */
260 "r3 = 0;" /* callback_ctx */
261 "r4 = 0;" /* flags */
262 "call %[bpf_loop];"
263
264 "r6 *= 4;"
265 "r1 = %[vals];"
266 /* here r6 is forced to be precise and has to be propagated
267 * back to the beginning, handling (and ignoring) callback call
268 */
269 "r1 += r6;"
270 "r0 = *(u32 *)(r1 + 0);"
271 "exit;"
272 :
273 : __imm_ptr(vals),
274 __imm_ptr(callback_subprog),
275 __imm(bpf_loop)
276 : __clobber_common, "r6"
277 );
278 }
279
280 SEC("?raw_tp")
281 __success __log_level(2)
282 __msg("9: (0f) r1 += r6")
283 __msg("mark_precise: frame0: last_idx 9 first_idx 6")
284 __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7")
285 __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4")
286 __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)")
287 __msg("mark_precise: frame0: parent state regs= stack=-8:")
288 __msg("mark_precise: frame0: last_idx 13 first_idx 0")
289 __msg("mark_precise: frame0: regs= stack=-8 before 13: (95) exit")
290 __msg("mark_precise: frame1: regs= stack= before 12: (bf) r0 = r1")
291 __msg("mark_precise: frame1: regs= stack= before 5: (85) call pc+6")
292 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0")
293 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6")
294 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3")
parent_stack_slot_precise(void)295 __naked int parent_stack_slot_precise(void)
296 {
297 asm volatile (
298 /* spill reg */
299 "r6 = 3;"
300 "*(u64 *)(r10 - 8) = r6;"
301
302 /* call subprog and ignore result; we need this call only to
303 * complicate jump history
304 */
305 "r1 = 0;"
306 "call identity_subprog;"
307
308 /* restore reg from stack; in this case we'll be carrying
309 * stack mask when going back into subprog through jump
310 * history
311 */
312 "r6 = *(u64 *)(r10 - 8);"
313
314 "r6 *= 4;"
315 "r1 = %[vals];"
316 /* here r6 is forced to be precise and has to be propagated
317 * back to the beginning, handling (and ignoring) subprog call
318 */
319 "r1 += r6;"
320 "r0 = *(u32 *)(r1 + 0);"
321 "exit;"
322 :
323 : __imm_ptr(vals)
324 : __clobber_common, "r6"
325 );
326 }
327
328 SEC("?raw_tp")
329 __success __log_level(2)
330 __msg("9: (0f) r1 += r6")
331 __msg("mark_precise: frame0: last_idx 9 first_idx 6")
332 __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7")
333 __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4")
334 __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)")
335 __msg("mark_precise: frame0: parent state regs= stack=-8:")
336 __msg("mark_precise: frame0: last_idx 5 first_idx 0")
337 __msg("mark_precise: frame0: regs= stack=-8 before 5: (85) call pc+6")
338 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0")
339 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6")
340 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3")
parent_stack_slot_precise_global(void)341 __naked int parent_stack_slot_precise_global(void)
342 {
343 asm volatile (
344 /* spill reg */
345 "r6 = 3;"
346 "*(u64 *)(r10 - 8) = r6;"
347
348 /* call subprog and ignore result; we need this call only to
349 * complicate jump history
350 */
351 "r1 = 0;"
352 "call global_identity_subprog;"
353
354 /* restore reg from stack; in this case we'll be carrying
355 * stack mask when going back into subprog through jump
356 * history
357 */
358 "r6 = *(u64 *)(r10 - 8);"
359
360 "r6 *= 4;"
361 "r1 = %[vals];"
362 /* here r6 is forced to be precise and has to be propagated
363 * back to the beginning, handling (and ignoring) subprog call
364 */
365 "r1 += r6;"
366 "r0 = *(u32 *)(r1 + 0);"
367 "exit;"
368 :
369 : __imm_ptr(vals)
370 : __clobber_common, "r6"
371 );
372 }
373
374 SEC("?raw_tp")
375 __success __log_level(2)
376 __msg("14: (0f) r1 += r6")
377 __msg("mark_precise: frame0: last_idx 14 first_idx 11")
378 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7")
379 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4")
380 __msg("mark_precise: frame0: regs=r6 stack= before 11: (79) r6 = *(u64 *)(r10 -8)")
381 __msg("mark_precise: frame0: parent state regs= stack=-8:")
382 __msg("mark_precise: frame0: last_idx 18 first_idx 0")
383 __msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit")
384 __msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0")
385 __msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181")
386 __msg("mark_precise: frame0: regs= stack=-8 before 9: (b7) r4 = 0")
387 __msg("mark_precise: frame0: regs= stack=-8 before 8: (b7) r3 = 0")
388 __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r2 = r8")
389 __msg("mark_precise: frame0: regs= stack=-8 before 6: (bf) r1 = r6")
390 __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -8) = r6")
391 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3")
parent_stack_slot_precise_with_callback(void)392 __naked int parent_stack_slot_precise_with_callback(void)
393 {
394 asm volatile (
395 /* spill reg */
396 "r6 = 3;"
397 "*(u64 *)(r10 - 8) = r6;"
398
399 /* ensure we have callback frame in jump history */
400 "r1 = r6;" /* nr_loops */
401 "r2 = %[callback_subprog];" /* callback_fn */
402 "r3 = 0;" /* callback_ctx */
403 "r4 = 0;" /* flags */
404 "call %[bpf_loop];"
405
406 /* restore reg from stack; in this case we'll be carrying
407 * stack mask when going back into subprog through jump
408 * history
409 */
410 "r6 = *(u64 *)(r10 - 8);"
411
412 "r6 *= 4;"
413 "r1 = %[vals];"
414 /* here r6 is forced to be precise and has to be propagated
415 * back to the beginning, handling (and ignoring) subprog call
416 */
417 "r1 += r6;"
418 "r0 = *(u32 *)(r1 + 0);"
419 "exit;"
420 :
421 : __imm_ptr(vals),
422 __imm_ptr(callback_subprog),
423 __imm(bpf_loop)
424 : __clobber_common, "r6"
425 );
426 }
427
428 __noinline __used
subprog_with_precise_arg(__u64 x)429 static __u64 subprog_with_precise_arg(__u64 x)
430 {
431 return vals[x]; /* x is forced to be precise */
432 }
433
434 SEC("?raw_tp")
435 __success __log_level(2)
436 __msg("8: (0f) r2 += r1")
437 __msg("mark_precise: frame1: last_idx 8 first_idx 0")
438 __msg("mark_precise: frame1: regs=r1 stack= before 6: (18) r2 = ")
439 __msg("mark_precise: frame1: regs=r1 stack= before 5: (67) r1 <<= 2")
440 __msg("mark_precise: frame1: regs=r1 stack= before 2: (85) call pc+2")
441 __msg("mark_precise: frame0: regs=r1 stack= before 1: (bf) r1 = r6")
442 __msg("mark_precise: frame0: regs=r6 stack= before 0: (b7) r6 = 3")
subprog_arg_precise(void)443 __naked int subprog_arg_precise(void)
444 {
445 asm volatile (
446 "r6 = 3;"
447 "r1 = r6;"
448 /* subprog_with_precise_arg expects its argument to be
449 * precise, so r1->r6 will be marked precise from inside the
450 * subprog
451 */
452 "call subprog_with_precise_arg;"
453 "r0 += r6;"
454 "exit;"
455 :
456 :
457 : __clobber_common, "r6"
458 );
459 }
460
461 /* r1 is pointer to stack slot;
462 * r2 is a register to spill into that slot
463 * subprog also spills r2 into its own stack slot
464 */
465 __naked __noinline __used
subprog_spill_reg_precise(void)466 static __u64 subprog_spill_reg_precise(void)
467 {
468 asm volatile (
469 /* spill to parent stack */
470 "*(u64 *)(r1 + 0) = r2;"
471 /* spill to subprog stack (we use -16 offset to avoid
472 * accidental confusion with parent's -8 stack slot in
473 * verifier log output)
474 */
475 "*(u64 *)(r10 - 16) = r2;"
476 /* use both spills as return result to propagete precision everywhere */
477 "r0 = *(u64 *)(r10 - 16);"
478 "r2 = *(u64 *)(r1 + 0);"
479 "r0 += r2;"
480 "exit;"
481 );
482 }
483
484 SEC("?raw_tp")
485 __success __log_level(2)
486 /* precision backtracking can't currently handle stack access not through r10,
487 * so we won't be able to mark stack slot fp-8 as precise, and so will
488 * fallback to forcing all as precise
489 */
490 __msg("mark_precise: frame0: falling back to forcing all scalars precise")
subprog_spill_into_parent_stack_slot_precise(void)491 __naked int subprog_spill_into_parent_stack_slot_precise(void)
492 {
493 asm volatile (
494 "r6 = 1;"
495
496 /* pass pointer to stack slot and r6 to subprog;
497 * r6 will be marked precise and spilled into fp-8 slot, which
498 * also should be marked precise
499 */
500 "r1 = r10;"
501 "r1 += -8;"
502 "r2 = r6;"
503 "call subprog_spill_reg_precise;"
504
505 /* restore reg from stack; in this case we'll be carrying
506 * stack mask when going back into subprog through jump
507 * history
508 */
509 "r7 = *(u64 *)(r10 - 8);"
510
511 "r7 *= 4;"
512 "r1 = %[vals];"
513 /* here r7 is forced to be precise and has to be propagated
514 * back to the beginning, handling subprog call and logic
515 */
516 "r1 += r7;"
517 "r0 = *(u32 *)(r1 + 0);"
518 "exit;"
519 :
520 : __imm_ptr(vals)
521 : __clobber_common, "r6", "r7"
522 );
523 }
524
525 __naked __noinline __used
subprog_with_checkpoint(void)526 static __u64 subprog_with_checkpoint(void)
527 {
528 asm volatile (
529 "r0 = 0;"
530 /* guaranteed checkpoint if BPF_F_TEST_STATE_FREQ is used */
531 "goto +0;"
532 "exit;"
533 );
534 }
535
536 char _license[] SEC("license") = "GPL";
537
