Lines Matching +full:0 +full:x40
64 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
73 do { EMIT1(b1); EMIT(off, 4); } while (0)
75 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
77 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
79 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
94 #define TCALL_CNT (MAX_BPF_JIT_REG + 0) /* Tail Call Count */
96 #define IA32_EAX (0x0)
97 #define IA32_EBX (0x3)
98 #define IA32_ECX (0x1)
99 #define IA32_EDX (0x2)
100 #define IA32_ESI (0x6)
101 #define IA32_EDI (0x7)
102 #define IA32_EBP (0x5)
103 #define IA32_ESP (0x4)
107 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
109 #define IA32_JB 0x72
110 #define IA32_JAE 0x73
111 #define IA32_JE 0x74
112 #define IA32_JNE 0x75
113 #define IA32_JBE 0x76
114 #define IA32_JA 0x77
115 #define IA32_JL 0x7C
116 #define IA32_JGE 0x7D
117 #define IA32_JLE 0x7E
118 #define IA32_JG 0x7F
120 #define COND_JMP_OPCODE_INVALID (0xFF)
139 [BPF_REG_0] = {STACK_OFFSET(0), STACK_OFFSET(4)},
166 #define dst_lo dst[0]
168 #define src_lo src[0]
202 memset(area, 0xcc, size); in jit_fill_hole()
209 int cnt = 0; in emit_ia32_mov_i()
212 if (val == 0) { in emit_ia32_mov_i()
214 EMIT2(0x33, add_2reg(0xC0, IA32_EAX, IA32_EAX)); in emit_ia32_mov_i()
216 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mov_i()
219 EMIT3_off32(0xC7, add_1reg(0x40, IA32_EBP), in emit_ia32_mov_i()
223 if (val == 0) in emit_ia32_mov_i()
224 EMIT2(0x33, add_2reg(0xC0, dst, dst)); in emit_ia32_mov_i()
226 EMIT2_off32(0xC7, add_1reg(0xC0, dst), in emit_ia32_mov_i()
237 int cnt = 0; in emit_ia32_mov_r()
242 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src)); in emit_ia32_mov_r()
245 EMIT3(0x89, add_2reg(0x40, IA32_EBP, sreg), STACK_VAR(dst)); in emit_ia32_mov_r()
248 EMIT2(0x89, add_2reg(0xC0, dst, sreg)); in emit_ia32_mov_r()
265 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); in emit_ia32_mov_r64()
272 u32 hi = 0; in emit_ia32_mov_i64()
275 hi = (u32)~0; in emit_ia32_mov_i64()
288 int cnt = 0; in emit_ia32_mul_r()
293 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src)); in emit_ia32_mul_r()
297 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_mul_r()
300 EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_mul_r()
303 EMIT2(0xF7, add_1reg(0xE0, sreg)); in emit_ia32_mul_r()
307 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r()
311 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_mul_r()
321 int cnt = 0; in emit_ia32_to_le_r64()
326 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_to_le_r64()
328 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_to_le_r64()
337 EMIT2(0x0F, 0xB7); in emit_ia32_to_le_r64()
338 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_to_le_r64()
341 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_le_r64()
346 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_le_r64()
355 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_to_le_r64()
358 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_to_le_r64()
369 int cnt = 0; in emit_ia32_to_be_r64()
374 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_to_be_r64()
376 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_to_be_r64()
382 EMIT1(0x66); in emit_ia32_to_be_r64()
383 EMIT3(0xC1, add_1reg(0xC8, dreg_lo), 8); in emit_ia32_to_be_r64()
385 EMIT2(0x0F, 0xB7); in emit_ia32_to_be_r64()
386 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_to_be_r64()
390 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_be_r64()
394 EMIT1(0x0F); in emit_ia32_to_be_r64()
395 EMIT1(add_1reg(0xC8, dreg_lo)); in emit_ia32_to_be_r64()
399 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_be_r64()
403 EMIT1(0x0F); in emit_ia32_to_be_r64()
404 EMIT1(add_1reg(0xC8, dreg_lo)); in emit_ia32_to_be_r64()
407 EMIT1(0x0F); in emit_ia32_to_be_r64()
408 EMIT1(add_1reg(0xC8, dreg_hi)); in emit_ia32_to_be_r64()
411 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, dreg_hi)); in emit_ia32_to_be_r64()
413 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_to_be_r64()
415 EMIT2(0x89, add_2reg(0xC0, dreg_lo, IA32_ECX)); in emit_ia32_to_be_r64()
421 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_to_be_r64()
424 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_to_be_r64()
438 int cnt = 0; in emit_ia32_div_mod_r()
442 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_div_mod_r()
446 EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX)); in emit_ia32_div_mod_r()
450 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_div_mod_r()
454 EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_div_mod_r()
457 EMIT2(0x31, add_2reg(0xC0, IA32_EDX, IA32_EDX)); in emit_ia32_div_mod_r()
459 EMIT2(0xF7, add_1reg(0xF0, IA32_ECX)); in emit_ia32_div_mod_r()
463 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_div_mod_r()
466 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EDX)); in emit_ia32_div_mod_r()
469 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_div_mod_r()
472 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_div_mod_r()
485 int cnt = 0; in emit_ia32_shift_r()
491 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_shift_r()
495 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src)); in emit_ia32_shift_r()
498 EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX)); in emit_ia32_shift_r()
502 b2 = 0xE0; break; in emit_ia32_shift_r()
504 b2 = 0xE8; break; in emit_ia32_shift_r()
506 b2 = 0xF8; break; in emit_ia32_shift_r()
510 EMIT2(0xD3, add_1reg(b2, dreg)); in emit_ia32_shift_r()
514 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), STACK_VAR(dst)); in emit_ia32_shift_r()
527 int cnt = 0; in emit_ia32_alu_r()
533 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src)); in emit_ia32_alu_r()
537 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(dst)); in emit_ia32_alu_r()
543 EMIT2(0x11, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
545 EMIT2(0x01, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
550 EMIT2(0x19, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
552 EMIT2(0x29, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
556 EMIT2(0x09, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
560 EMIT2(0x21, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
564 EMIT2(0x31, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
570 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), in emit_ia32_alu_r()
588 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in emit_ia32_alu_r64()
601 int cnt = 0; in emit_ia32_alu_i()
607 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_alu_i()
611 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EDX), val); in emit_ia32_alu_i()
618 EMIT3(0x83, add_1reg(0xD0, dreg), val); in emit_ia32_alu_i()
620 EMIT2(0x11, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
623 EMIT3(0x83, add_1reg(0xC0, dreg), val); in emit_ia32_alu_i()
625 EMIT2(0x01, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
632 EMIT3(0x83, add_1reg(0xD8, dreg), val); in emit_ia32_alu_i()
634 EMIT2(0x19, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
637 EMIT3(0x83, add_1reg(0xE8, dreg), val); in emit_ia32_alu_i()
639 EMIT2(0x29, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
645 EMIT3(0x83, add_1reg(0xC8, dreg), val); in emit_ia32_alu_i()
647 EMIT2(0x09, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
652 EMIT3(0x83, add_1reg(0xE0, dreg), val); in emit_ia32_alu_i()
654 EMIT2(0x21, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
659 EMIT3(0x83, add_1reg(0xF0, dreg), val); in emit_ia32_alu_i()
661 EMIT2(0x31, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
664 EMIT2(0xF7, add_1reg(0xD8, dreg)); in emit_ia32_alu_i()
670 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), in emit_ia32_alu_i()
682 u32 hi = 0; in emit_ia32_alu_i64()
685 hi = (u32)~0; in emit_ia32_alu_i64()
691 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in emit_ia32_alu_i64()
700 int cnt = 0; in emit_ia32_neg64()
705 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_neg64()
707 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_neg64()
712 EMIT2(0xF7, add_1reg(0xD8, dreg_lo)); in emit_ia32_neg64()
713 /* adc dreg_hi,0x0 */ in emit_ia32_neg64()
714 EMIT3(0x83, add_1reg(0xD0, dreg_hi), 0x00); in emit_ia32_neg64()
716 EMIT2(0xF7, add_1reg(0xD8, dreg_hi)); in emit_ia32_neg64()
720 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_neg64()
723 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_neg64()
734 int cnt = 0; in emit_ia32_lsh_r64()
739 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_lsh_r64()
741 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_lsh_r64()
747 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_lsh_r64()
751 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_lsh_r64()
754 EMIT3(0x0F, 0xA5, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_r64()
756 EMIT2(0xD3, add_1reg(0xE0, dreg_lo)); in emit_ia32_lsh_r64()
761 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_lsh_r64()
766 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_r64()
768 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_r64()
772 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_lsh_r64()
775 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_lsh_r64()
787 int cnt = 0; in emit_ia32_arsh_r64()
792 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_arsh_r64()
794 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_arsh_r64()
800 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_arsh_r64()
804 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_arsh_r64()
807 EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_r64()
809 EMIT2(0xD3, add_1reg(0xF8, dreg_hi)); in emit_ia32_arsh_r64()
814 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_arsh_r64()
819 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_r64()
821 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_r64()
825 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_arsh_r64()
828 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_arsh_r64()
840 int cnt = 0; in emit_ia32_rsh_r64()
845 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_rsh_r64()
847 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_rsh_r64()
853 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_rsh_r64()
857 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_rsh_r64()
860 EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_r64()
862 EMIT2(0xD3, add_1reg(0xE8, dreg_hi)); in emit_ia32_rsh_r64()
867 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_rsh_r64()
872 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_r64()
874 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_r64()
878 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_rsh_r64()
881 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_rsh_r64()
893 int cnt = 0; in emit_ia32_lsh_i64()
898 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_lsh_i64()
900 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_lsh_i64()
906 EMIT4(0x0F, 0xA4, add_2reg(0xC0, dreg_hi, dreg_lo), val); in emit_ia32_lsh_i64()
908 EMIT3(0xC1, add_1reg(0xE0, dreg_lo), val); in emit_ia32_lsh_i64()
913 EMIT3(0xC1, add_1reg(0xE0, dreg_lo), value); in emit_ia32_lsh_i64()
915 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_i64()
917 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_i64()
920 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_i64()
922 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_lsh_i64()
927 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_lsh_i64()
930 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_lsh_i64()
941 int cnt = 0; in emit_ia32_rsh_i64()
946 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_rsh_i64()
948 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_rsh_i64()
955 EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val); in emit_ia32_rsh_i64()
957 EMIT3(0xC1, add_1reg(0xE8, dreg_hi), val); in emit_ia32_rsh_i64()
962 EMIT3(0xC1, add_1reg(0xE8, dreg_hi), value); in emit_ia32_rsh_i64()
964 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_i64()
966 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_i64()
969 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_rsh_i64()
971 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_i64()
976 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_rsh_i64()
979 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_rsh_i64()
990 int cnt = 0; in emit_ia32_arsh_i64()
995 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_arsh_i64()
997 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_arsh_i64()
1003 EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val); in emit_ia32_arsh_i64()
1005 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), val); in emit_ia32_arsh_i64()
1010 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), value); in emit_ia32_arsh_i64()
1012 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_i64()
1015 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_i64()
1018 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_i64()
1020 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_i64()
1025 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_arsh_i64()
1028 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_arsh_i64()
1038 int cnt = 0; in emit_ia32_mul_r64()
1042 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1046 EMIT2(0x8B, add_2reg(0xC0, dst_hi, IA32_EAX)); in emit_ia32_mul_r64()
1050 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo)); in emit_ia32_mul_r64()
1053 EMIT2(0xF7, add_1reg(0xE0, src_lo)); in emit_ia32_mul_r64()
1056 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_r64()
1060 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1064 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1068 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_hi)); in emit_ia32_mul_r64()
1071 EMIT2(0xF7, add_1reg(0xE0, src_hi)); in emit_ia32_mul_r64()
1074 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_r64()
1078 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1082 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1086 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo)); in emit_ia32_mul_r64()
1089 EMIT2(0xF7, add_1reg(0xE0, src_lo)); in emit_ia32_mul_r64()
1092 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX)); in emit_ia32_mul_r64()
1096 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1099 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_mul_r64()
1103 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1105 EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX)); in emit_ia32_mul_r64()
1115 int cnt = 0; in emit_ia32_mul_i64()
1118 hi = val & (1<<31) ? (u32)~0 : 0; in emit_ia32_mul_i64()
1120 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val); in emit_ia32_mul_i64()
1123 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_hi)); in emit_ia32_mul_i64()
1126 EMIT2(0xF7, add_1reg(0xE0, dst_hi)); in emit_ia32_mul_i64()
1129 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_i64()
1132 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), hi); in emit_ia32_mul_i64()
1135 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo)); in emit_ia32_mul_i64()
1138 EMIT2(0xF7, add_1reg(0xE0, dst_lo)); in emit_ia32_mul_i64()
1140 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_i64()
1143 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val); in emit_ia32_mul_i64()
1146 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo)); in emit_ia32_mul_i64()
1149 EMIT2(0xF7, add_1reg(0xE0, dst_lo)); in emit_ia32_mul_i64()
1152 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX)); in emit_ia32_mul_i64()
1156 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_i64()
1159 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_mul_i64()
1163 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_i64()
1165 EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX)); in emit_ia32_mul_i64()
1182 return 0; in bpf_size_to_x86_bytes()
1202 int cnt = 0; in emit_prologue()
1204 const u8 fplo = bpf2ia32[BPF_REG_FP][0]; in emit_prologue()
1209 EMIT1(0x55); in emit_prologue()
1211 EMIT2(0x89, 0xE5); in emit_prologue()
1213 EMIT1(0x57); in emit_prologue()
1215 EMIT1(0x56); in emit_prologue()
1217 EMIT1(0x53); in emit_prologue()
1220 EMIT2_off32(0x81, 0xEC, STACK_SIZE); in emit_prologue()
1222 EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 12); in emit_prologue()
1224 EMIT2(0x31, add_2reg(0xC0, IA32_EBX, IA32_EBX)); in emit_prologue()
1227 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBP), STACK_VAR(fplo)); in emit_prologue()
1228 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(fphi)); in emit_prologue()
1232 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0])); in emit_prologue()
1233 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(r1[1])); in emit_prologue()
1236 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[0])); in emit_prologue()
1237 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_prologue()
1248 int cnt = 0; in emit_epilogue()
1251 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r0[0])); in emit_epilogue()
1253 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1])); in emit_epilogue()
1256 EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 12); in emit_epilogue()
1259 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12); in emit_epilogue()
1261 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8); in emit_epilogue()
1263 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4); in emit_epilogue()
1265 EMIT1(0xC9); /* leave */ in emit_epilogue()
1266 EMIT1(0xC3); /* ret */ in emit_epilogue()
1286 int cnt = 0; in emit_bpf_tail_call()
1299 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r2[0])); in emit_bpf_tail_call()
1301 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r3[0])); in emit_bpf_tail_call()
1304 EMIT3(0x39, add_2reg(0x40, IA32_EAX, IA32_EDX), in emit_bpf_tail_call()
1315 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0])); in emit_bpf_tail_call()
1316 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_bpf_tail_call()
1319 EMIT3(0x83, add_1reg(0xF8, IA32_EBX), hi); in emit_bpf_tail_call()
1322 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo); in emit_bpf_tail_call()
1327 /* add eax,0x1 */ in emit_bpf_tail_call()
1328 EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 0x01); in emit_bpf_tail_call()
1329 /* adc ebx,0x0 */ in emit_bpf_tail_call()
1330 EMIT3(0x83, add_1reg(0xD0, IA32_EBX), 0x00); in emit_bpf_tail_call()
1333 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0])); in emit_bpf_tail_call()
1335 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_bpf_tail_call()
1339 EMIT3_off32(0x8B, 0x94, 0x90, offsetof(struct bpf_array, ptrs)); in emit_bpf_tail_call()
1346 EMIT2(0x85, add_2reg(0xC0, IA32_EDX, IA32_EDX)); in emit_bpf_tail_call()
1352 EMIT3(0x8B, add_2reg(0x40, IA32_EDX, IA32_EDX), in emit_bpf_tail_call()
1355 EMIT3(0x83, add_1reg(0xC0, IA32_EDX), PROLOGUE_SIZE); in emit_bpf_tail_call()
1358 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0])); in emit_bpf_tail_call()
1378 int cnt = 0; in emit_push_r64()
1381 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_hi)); in emit_push_r64()
1383 EMIT1(0x51); in emit_push_r64()
1386 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo)); in emit_push_r64()
1388 EMIT1(0x51); in emit_push_r64()
1396 int cnt = 0; in emit_push_r32()
1399 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo)); in emit_push_r32()
1401 EMIT1(0x51); in emit_push_r32()
1541 * When emitting a call (0xE8), it needs to figure out
1543 * following the call (0xE8) instruction. At this point, it knows
1548 * the call (0xE8) and the end_addr:
1549 * - 0-1 jit-insn (3 bytes each) to restore the esp pointer if there
1551 * - 0-2 jit-insns (3 bytes each) to handle the return value.
1557 int i, cnt = 0, first_stack_regno, last_stack_regno; in emit_kfunc_call()
1560 int bytes_in_stack = 0; in emit_kfunc_call()
1570 for (i = 0; i < fm->nr_args; i++) { in emit_kfunc_call()
1592 cur_arg_reg = &arg_regs[0]; in emit_kfunc_call()
1595 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++), in emit_kfunc_call()
1596 STACK_VAR(bpf2ia32[i][0])); in emit_kfunc_call()
1599 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++), in emit_kfunc_call()
1622 EMIT1_off32(0xE8, jmp_offset); in emit_kfunc_call()
1626 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_kfunc_call()
1627 STACK_VAR(bpf2ia32[BPF_REG_0][0])); in emit_kfunc_call()
1631 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_kfunc_call()
1636 EMIT3(0x83, add_1reg(0xC0, IA32_ESP), bytes_in_stack); in emit_kfunc_call()
1640 return 0; in emit_kfunc_call()
1650 int i, cnt = 0; in do_jit()
1651 int proglen = 0; in do_jit()
1656 for (i = 0; i < insn_cnt; i++, insn++) { in do_jit()
1681 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1744 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1751 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1764 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1772 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1787 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1795 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1809 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
1813 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1848 dst_lo, 0, dstk, &prog); in do_jit()
1850 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1893 EMIT3(0x0F, 0xAE, 0xE8); in do_jit()
1902 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1906 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in do_jit()
1910 EMIT(0xC6, 1); break; in do_jit()
1912 EMIT2(0x66, 0xC7); break; in do_jit()
1915 EMIT(0xC7, 1); break; in do_jit()
1919 EMIT2(add_1reg(0x40, IA32_EAX), insn->off); in do_jit()
1921 EMIT1_off32(add_1reg(0x80, IA32_EAX), in do_jit()
1928 hi = imm32 & (1<<31) ? (u32)~0 : 0; in do_jit()
1929 EMIT2_off32(0xC7, add_1reg(0x80, IA32_EAX), in do_jit()
1942 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1946 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in do_jit()
1950 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
1954 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EDX)); in do_jit()
1958 EMIT(0x88, 1); break; in do_jit()
1960 EMIT2(0x66, 0x89); break; in do_jit()
1963 EMIT(0x89, 1); break; in do_jit()
1967 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX), in do_jit()
1970 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
1976 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, in do_jit()
1981 EMIT2(0x8B, add_2reg(0xC0, src_hi, in do_jit()
1983 EMIT1(0x89); in do_jit()
1985 EMIT2(add_2reg(0x40, IA32_EAX, in do_jit()
1989 EMIT1(add_2reg(0x80, IA32_EAX, in do_jit()
2003 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2007 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EAX)); in do_jit()
2011 EMIT2(0x0F, 0xB6); break; in do_jit()
2013 EMIT2(0x0F, 0xB7); break; in do_jit()
2016 EMIT(0x8B, 1); break; in do_jit()
2020 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX), in do_jit()
2023 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
2028 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
2032 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EDX)); in do_jit()
2040 EMIT3(0xC7, add_1reg(0x40, IA32_EBP), in do_jit()
2042 EMIT(0x0, 4); in do_jit()
2045 EMIT2(0x33, in do_jit()
2046 add_2reg(0xC0, dst_hi, dst_hi)); in do_jit()
2050 EMIT2_off32(0x8B, in do_jit()
2051 add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
2054 EMIT3(0x89, in do_jit()
2055 add_2reg(0x40, IA32_EBP, in do_jit()
2059 EMIT2(0x89, in do_jit()
2060 add_2reg(0xC0, dst_hi, IA32_EDX)); in do_jit()
2100 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2101 STACK_VAR(r1[0])); in do_jit()
2103 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
2111 EMIT1_off32(0xE8, jmp_offset + 9); in do_jit()
2114 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2115 STACK_VAR(r0[0])); in do_jit()
2117 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
2121 EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 32); in do_jit()
2152 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2155 EMIT3(0x8B, in do_jit()
2156 add_2reg(0x40, IA32_EBP, in do_jit()
2162 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2165 EMIT3(0x8B, in do_jit()
2166 add_2reg(0x40, IA32_EBP, in do_jit()
2173 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2177 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2190 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2192 EMIT3(0x8B, in do_jit()
2193 add_2reg(0x40, IA32_EBP, in do_jit()
2199 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2201 EMIT3(0x8B, in do_jit()
2202 add_2reg(0x40, IA32_EBP, in do_jit()
2208 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2211 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2223 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2226 EMIT3(0x8B, in do_jit()
2227 add_2reg(0x40, IA32_EBP, in do_jit()
2232 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo)); in do_jit()
2235 EMIT2(0x89, in do_jit()
2236 add_2reg(0xC0, dreg_hi, dst_hi)); in do_jit()
2240 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2243 EMIT3(0x8B, in do_jit()
2244 add_2reg(0x40, IA32_EBP, in do_jit()
2249 EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); in do_jit()
2252 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); in do_jit()
2254 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); in do_jit()
2268 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2271 EMIT3(0x8B, in do_jit()
2272 add_2reg(0x40, IA32_EBP, in do_jit()
2277 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo)); in do_jit()
2280 EMIT2(0x89, in do_jit()
2281 add_2reg(0xC0, dreg_hi, dst_hi)); in do_jit()
2285 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_lo), imm32); in do_jit()
2288 EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); in do_jit()
2290 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2292 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_hi), hi); in do_jit()
2294 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); in do_jit()
2296 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); in do_jit()
2324 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2327 EMIT3(0x8B, in do_jit()
2328 add_2reg(0x40, IA32_EBP, in do_jit()
2334 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
2336 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2338 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi); in do_jit()
2340 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2344 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2353 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2371 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2373 EMIT3(0x8B, in do_jit()
2374 add_2reg(0x40, IA32_EBP, in do_jit()
2380 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
2381 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2383 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi); in do_jit()
2385 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2388 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2400 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2405 EMIT2(0xEB, 6); in do_jit()
2413 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2437 EMIT2(0xEB, jmp_offset); in do_jit()
2439 EMIT1_off32(0xE9, jmp_offset); in do_jit()
2509 int proglen, oldproglen = 0; in bpf_int_jit_compile()
2542 for (proglen = 0, i = 0; i < prog->len; i++) { in bpf_int_jit_compile()
2554 for (pass = 0; pass < 20 || image; pass++) { in bpf_int_jit_compile()
2556 if (proglen <= 0) { in bpf_int_jit_compile()