Lines Matching +full:0 +full:x89
65 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
74 do { EMIT1(b1); EMIT(off, 4); } while (0)
76 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
78 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
80 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
95 #define TCALL_CNT (MAX_BPF_JIT_REG + 0) /* Tail Call Count */
97 #define IA32_EAX (0x0)
98 #define IA32_EBX (0x3)
99 #define IA32_ECX (0x1)
100 #define IA32_EDX (0x2)
101 #define IA32_ESI (0x6)
102 #define IA32_EDI (0x7)
103 #define IA32_EBP (0x5)
104 #define IA32_ESP (0x4)
108 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
110 #define IA32_JB 0x72
111 #define IA32_JAE 0x73
112 #define IA32_JE 0x74
113 #define IA32_JNE 0x75
114 #define IA32_JBE 0x76
115 #define IA32_JA 0x77
116 #define IA32_JL 0x7C
117 #define IA32_JGE 0x7D
118 #define IA32_JLE 0x7E
119 #define IA32_JG 0x7F
121 #define COND_JMP_OPCODE_INVALID (0xFF)
140 [BPF_REG_0] = {STACK_OFFSET(0), STACK_OFFSET(4)},
167 #define dst_lo dst[0]
169 #define src_lo src[0]
203 memset(area, 0xcc, size); in jit_fill_hole()
210 int cnt = 0; in emit_ia32_mov_i()
213 if (val == 0) { in emit_ia32_mov_i()
215 EMIT2(0x33, add_2reg(0xC0, IA32_EAX, IA32_EAX)); in emit_ia32_mov_i()
217 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mov_i()
220 EMIT3_off32(0xC7, add_1reg(0x40, IA32_EBP), in emit_ia32_mov_i()
224 if (val == 0) in emit_ia32_mov_i()
225 EMIT2(0x33, add_2reg(0xC0, dst, dst)); in emit_ia32_mov_i()
227 EMIT2_off32(0xC7, add_1reg(0xC0, dst), in emit_ia32_mov_i()
238 int cnt = 0; in emit_ia32_mov_r()
243 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src)); in emit_ia32_mov_r()
246 EMIT3(0x89, add_2reg(0x40, IA32_EBP, sreg), STACK_VAR(dst)); in emit_ia32_mov_r()
249 EMIT2(0x89, add_2reg(0xC0, dst, sreg)); in emit_ia32_mov_r()
266 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); in emit_ia32_mov_r64()
273 u32 hi = 0; in emit_ia32_mov_i64()
276 hi = (u32)~0; in emit_ia32_mov_i64()
289 int cnt = 0; in emit_ia32_mul_r()
294 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src)); in emit_ia32_mul_r()
298 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_mul_r()
301 EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_mul_r()
304 EMIT2(0xF7, add_1reg(0xE0, sreg)); in emit_ia32_mul_r()
308 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r()
312 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_mul_r()
322 int cnt = 0; in emit_ia32_to_le_r64()
327 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_to_le_r64()
329 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_to_le_r64()
338 EMIT2(0x0F, 0xB7); in emit_ia32_to_le_r64()
339 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_to_le_r64()
342 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_le_r64()
347 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_le_r64()
356 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_to_le_r64()
359 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_to_le_r64()
370 int cnt = 0; in emit_ia32_to_be_r64()
375 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_to_be_r64()
377 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_to_be_r64()
383 EMIT1(0x66); in emit_ia32_to_be_r64()
384 EMIT3(0xC1, add_1reg(0xC8, dreg_lo), 8); in emit_ia32_to_be_r64()
386 EMIT2(0x0F, 0xB7); in emit_ia32_to_be_r64()
387 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_to_be_r64()
391 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_be_r64()
395 EMIT1(0x0F); in emit_ia32_to_be_r64()
396 EMIT1(add_1reg(0xC8, dreg_lo)); in emit_ia32_to_be_r64()
400 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_be_r64()
404 EMIT1(0x0F); in emit_ia32_to_be_r64()
405 EMIT1(add_1reg(0xC8, dreg_lo)); in emit_ia32_to_be_r64()
408 EMIT1(0x0F); in emit_ia32_to_be_r64()
409 EMIT1(add_1reg(0xC8, dreg_hi)); in emit_ia32_to_be_r64()
412 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, dreg_hi)); in emit_ia32_to_be_r64()
414 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_to_be_r64()
416 EMIT2(0x89, add_2reg(0xC0, dreg_lo, IA32_ECX)); in emit_ia32_to_be_r64()
422 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_to_be_r64()
425 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_to_be_r64()
439 int cnt = 0; in emit_ia32_div_mod_r()
443 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_div_mod_r()
447 EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX)); in emit_ia32_div_mod_r()
451 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_div_mod_r()
455 EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_div_mod_r()
458 EMIT2(0x31, add_2reg(0xC0, IA32_EDX, IA32_EDX)); in emit_ia32_div_mod_r()
460 EMIT2(0xF7, add_1reg(0xF0, IA32_ECX)); in emit_ia32_div_mod_r()
464 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_div_mod_r()
467 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EDX)); in emit_ia32_div_mod_r()
470 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_div_mod_r()
473 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_div_mod_r()
486 int cnt = 0; in emit_ia32_shift_r()
492 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_shift_r()
496 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src)); in emit_ia32_shift_r()
499 EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX)); in emit_ia32_shift_r()
503 b2 = 0xE0; break; in emit_ia32_shift_r()
505 b2 = 0xE8; break; in emit_ia32_shift_r()
507 b2 = 0xF8; break; in emit_ia32_shift_r()
511 EMIT2(0xD3, add_1reg(b2, dreg)); in emit_ia32_shift_r()
515 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), STACK_VAR(dst)); in emit_ia32_shift_r()
528 int cnt = 0; in emit_ia32_alu_r()
534 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src)); in emit_ia32_alu_r()
538 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(dst)); in emit_ia32_alu_r()
544 EMIT2(0x11, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
546 EMIT2(0x01, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
551 EMIT2(0x19, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
553 EMIT2(0x29, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
557 EMIT2(0x09, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
561 EMIT2(0x21, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
565 EMIT2(0x31, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
571 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), in emit_ia32_alu_r()
589 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in emit_ia32_alu_r64()
602 int cnt = 0; in emit_ia32_alu_i()
608 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_alu_i()
612 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EDX), val); in emit_ia32_alu_i()
619 EMIT3(0x83, add_1reg(0xD0, dreg), val); in emit_ia32_alu_i()
621 EMIT2(0x11, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
624 EMIT3(0x83, add_1reg(0xC0, dreg), val); in emit_ia32_alu_i()
626 EMIT2(0x01, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
633 EMIT3(0x83, add_1reg(0xD8, dreg), val); in emit_ia32_alu_i()
635 EMIT2(0x19, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
638 EMIT3(0x83, add_1reg(0xE8, dreg), val); in emit_ia32_alu_i()
640 EMIT2(0x29, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
646 EMIT3(0x83, add_1reg(0xC8, dreg), val); in emit_ia32_alu_i()
648 EMIT2(0x09, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
653 EMIT3(0x83, add_1reg(0xE0, dreg), val); in emit_ia32_alu_i()
655 EMIT2(0x21, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
660 EMIT3(0x83, add_1reg(0xF0, dreg), val); in emit_ia32_alu_i()
662 EMIT2(0x31, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
665 EMIT2(0xF7, add_1reg(0xD8, dreg)); in emit_ia32_alu_i()
671 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), in emit_ia32_alu_i()
683 u32 hi = 0; in emit_ia32_alu_i64()
686 hi = (u32)~0; in emit_ia32_alu_i64()
692 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in emit_ia32_alu_i64()
701 int cnt = 0; in emit_ia32_neg64()
706 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_neg64()
708 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_neg64()
713 EMIT2(0xF7, add_1reg(0xD8, dreg_lo)); in emit_ia32_neg64()
714 /* adc dreg_hi,0x0 */ in emit_ia32_neg64()
715 EMIT3(0x83, add_1reg(0xD0, dreg_hi), 0x00); in emit_ia32_neg64()
717 EMIT2(0xF7, add_1reg(0xD8, dreg_hi)); in emit_ia32_neg64()
721 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_neg64()
724 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_neg64()
735 int cnt = 0; in emit_ia32_lsh_r64()
740 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_lsh_r64()
742 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_lsh_r64()
748 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_lsh_r64()
752 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_lsh_r64()
755 EMIT3(0x0F, 0xA5, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_r64()
757 EMIT2(0xD3, add_1reg(0xE0, dreg_lo)); in emit_ia32_lsh_r64()
762 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_lsh_r64()
767 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_r64()
769 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_r64()
773 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_lsh_r64()
776 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_lsh_r64()
788 int cnt = 0; in emit_ia32_arsh_r64()
793 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_arsh_r64()
795 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_arsh_r64()
801 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_arsh_r64()
805 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_arsh_r64()
808 EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_r64()
810 EMIT2(0xD3, add_1reg(0xF8, dreg_hi)); in emit_ia32_arsh_r64()
815 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_arsh_r64()
820 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_r64()
822 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_r64()
826 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_arsh_r64()
829 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_arsh_r64()
841 int cnt = 0; in emit_ia32_rsh_r64()
846 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_rsh_r64()
848 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_rsh_r64()
854 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_rsh_r64()
858 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_rsh_r64()
861 EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_r64()
863 EMIT2(0xD3, add_1reg(0xE8, dreg_hi)); in emit_ia32_rsh_r64()
868 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_rsh_r64()
873 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_r64()
875 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_r64()
879 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_rsh_r64()
882 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_rsh_r64()
894 int cnt = 0; in emit_ia32_lsh_i64()
899 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_lsh_i64()
901 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_lsh_i64()
907 EMIT4(0x0F, 0xA4, add_2reg(0xC0, dreg_hi, dreg_lo), val); in emit_ia32_lsh_i64()
909 EMIT3(0xC1, add_1reg(0xE0, dreg_lo), val); in emit_ia32_lsh_i64()
914 EMIT3(0xC1, add_1reg(0xE0, dreg_lo), value); in emit_ia32_lsh_i64()
916 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_i64()
918 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_i64()
921 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_i64()
923 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_lsh_i64()
928 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_lsh_i64()
931 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_lsh_i64()
942 int cnt = 0; in emit_ia32_rsh_i64()
947 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_rsh_i64()
949 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_rsh_i64()
956 EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val); in emit_ia32_rsh_i64()
958 EMIT3(0xC1, add_1reg(0xE8, dreg_hi), val); in emit_ia32_rsh_i64()
963 EMIT3(0xC1, add_1reg(0xE8, dreg_hi), value); in emit_ia32_rsh_i64()
965 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_i64()
967 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_i64()
970 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_rsh_i64()
972 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_i64()
977 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_rsh_i64()
980 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_rsh_i64()
991 int cnt = 0; in emit_ia32_arsh_i64()
996 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_arsh_i64()
998 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_arsh_i64()
1004 EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val); in emit_ia32_arsh_i64()
1006 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), val); in emit_ia32_arsh_i64()
1011 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), value); in emit_ia32_arsh_i64()
1013 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_i64()
1016 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_i64()
1019 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_i64()
1021 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_i64()
1026 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_arsh_i64()
1029 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_arsh_i64()
1039 int cnt = 0; in emit_ia32_mul_r64()
1043 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1047 EMIT2(0x8B, add_2reg(0xC0, dst_hi, IA32_EAX)); in emit_ia32_mul_r64()
1051 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo)); in emit_ia32_mul_r64()
1054 EMIT2(0xF7, add_1reg(0xE0, src_lo)); in emit_ia32_mul_r64()
1057 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_r64()
1061 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1065 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1069 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_hi)); in emit_ia32_mul_r64()
1072 EMIT2(0xF7, add_1reg(0xE0, src_hi)); in emit_ia32_mul_r64()
1075 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_r64()
1079 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1083 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1087 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo)); in emit_ia32_mul_r64()
1090 EMIT2(0xF7, add_1reg(0xE0, src_lo)); in emit_ia32_mul_r64()
1093 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX)); in emit_ia32_mul_r64()
1097 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1100 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_mul_r64()
1104 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1106 EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX)); in emit_ia32_mul_r64()
1116 int cnt = 0; in emit_ia32_mul_i64()
1119 hi = val & (1<<31) ? (u32)~0 : 0; in emit_ia32_mul_i64()
1121 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val); in emit_ia32_mul_i64()
1124 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_hi)); in emit_ia32_mul_i64()
1127 EMIT2(0xF7, add_1reg(0xE0, dst_hi)); in emit_ia32_mul_i64()
1130 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_i64()
1133 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), hi); in emit_ia32_mul_i64()
1136 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo)); in emit_ia32_mul_i64()
1139 EMIT2(0xF7, add_1reg(0xE0, dst_lo)); in emit_ia32_mul_i64()
1141 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_i64()
1144 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val); in emit_ia32_mul_i64()
1147 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo)); in emit_ia32_mul_i64()
1150 EMIT2(0xF7, add_1reg(0xE0, dst_lo)); in emit_ia32_mul_i64()
1153 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX)); in emit_ia32_mul_i64()
1157 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_i64()
1160 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_mul_i64()
1164 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_i64()
1166 EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX)); in emit_ia32_mul_i64()
1183 return 0; in bpf_size_to_x86_bytes()
1203 int cnt = 0; in emit_prologue()
1205 const u8 fplo = bpf2ia32[BPF_REG_FP][0]; in emit_prologue()
1210 EMIT1(0x55); in emit_prologue()
1212 EMIT2(0x89, 0xE5); in emit_prologue()
1214 EMIT1(0x57); in emit_prologue()
1216 EMIT1(0x56); in emit_prologue()
1218 EMIT1(0x53); in emit_prologue()
1221 EMIT2_off32(0x81, 0xEC, STACK_SIZE); in emit_prologue()
1223 EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 12); in emit_prologue()
1225 EMIT2(0x31, add_2reg(0xC0, IA32_EBX, IA32_EBX)); in emit_prologue()
1228 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBP), STACK_VAR(fplo)); in emit_prologue()
1229 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(fphi)); in emit_prologue()
1233 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0])); in emit_prologue()
1234 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(r1[1])); in emit_prologue()
1237 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[0])); in emit_prologue()
1238 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_prologue()
1249 int cnt = 0; in emit_epilogue()
1252 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r0[0])); in emit_epilogue()
1254 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1])); in emit_epilogue()
1257 EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 12); in emit_epilogue()
1260 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12); in emit_epilogue()
1262 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8); in emit_epilogue()
1264 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4); in emit_epilogue()
1266 EMIT1(0xC9); /* leave */ in emit_epilogue()
1267 EMIT1(0xC3); /* ret */ in emit_epilogue()
1274 int cnt = 0; in emit_jmp_edx()
1277 EMIT1_off32(0xE9, (u8 *)__x86_indirect_thunk_edx - (ip + 5)); in emit_jmp_edx()
1279 EMIT2(0xFF, 0xE2); in emit_jmp_edx()
1302 int cnt = 0; in emit_bpf_tail_call()
1315 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r2[0])); in emit_bpf_tail_call()
1317 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r3[0])); in emit_bpf_tail_call()
1320 EMIT3(0x39, add_2reg(0x40, IA32_EAX, IA32_EDX), in emit_bpf_tail_call()
1331 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0])); in emit_bpf_tail_call()
1332 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_bpf_tail_call()
1335 EMIT3(0x83, add_1reg(0xF8, IA32_EBX), hi); in emit_bpf_tail_call()
1338 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo); in emit_bpf_tail_call()
1343 /* add eax,0x1 */ in emit_bpf_tail_call()
1344 EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 0x01); in emit_bpf_tail_call()
1345 /* adc ebx,0x0 */ in emit_bpf_tail_call()
1346 EMIT3(0x83, add_1reg(0xD0, IA32_EBX), 0x00); in emit_bpf_tail_call()
1349 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0])); in emit_bpf_tail_call()
1351 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_bpf_tail_call()
1355 EMIT3_off32(0x8B, 0x94, 0x90, offsetof(struct bpf_array, ptrs)); in emit_bpf_tail_call()
1362 EMIT2(0x85, add_2reg(0xC0, IA32_EDX, IA32_EDX)); in emit_bpf_tail_call()
1368 EMIT3(0x8B, add_2reg(0x40, IA32_EDX, IA32_EDX), in emit_bpf_tail_call()
1371 EMIT3(0x83, add_1reg(0xC0, IA32_EDX), PROLOGUE_SIZE); in emit_bpf_tail_call()
1374 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0])); in emit_bpf_tail_call()
1394 int cnt = 0; in emit_push_r64()
1397 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_hi)); in emit_push_r64()
1399 EMIT1(0x51); in emit_push_r64()
1402 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo)); in emit_push_r64()
1404 EMIT1(0x51); in emit_push_r64()
1412 int cnt = 0; in emit_push_r32()
1415 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo)); in emit_push_r32()
1417 EMIT1(0x51); in emit_push_r32()
1557 * When emitting a call (0xE8), it needs to figure out
1559 * following the call (0xE8) instruction. At this point, it knows
1564 * the call (0xE8) and the end_addr:
1565 * - 0-1 jit-insn (3 bytes each) to restore the esp pointer if there
1567 * - 0-2 jit-insns (3 bytes each) to handle the return value.
1573 int i, cnt = 0, first_stack_regno, last_stack_regno; in emit_kfunc_call()
1576 int bytes_in_stack = 0; in emit_kfunc_call()
1586 for (i = 0; i < fm->nr_args; i++) { in emit_kfunc_call()
1608 cur_arg_reg = &arg_regs[0]; in emit_kfunc_call()
1611 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++), in emit_kfunc_call()
1612 STACK_VAR(bpf2ia32[i][0])); in emit_kfunc_call()
1615 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++), in emit_kfunc_call()
1638 EMIT1_off32(0xE8, jmp_offset); in emit_kfunc_call()
1642 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_kfunc_call()
1643 STACK_VAR(bpf2ia32[BPF_REG_0][0])); in emit_kfunc_call()
1647 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_kfunc_call()
1652 EMIT3(0x83, add_1reg(0xC0, IA32_ESP), bytes_in_stack); in emit_kfunc_call()
1656 return 0; in emit_kfunc_call()
1666 int i, cnt = 0; in do_jit()
1667 int proglen = 0; in do_jit()
1672 for (i = 0; i < insn_cnt; i++, insn++) { in do_jit()
1697 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1760 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1767 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1780 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1788 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1803 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1811 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1825 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
1829 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1864 dst_lo, 0, dstk, &prog); in do_jit()
1866 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1909 EMIT3(0x0F, 0xAE, 0xE8); in do_jit()
1918 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1922 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in do_jit()
1926 EMIT(0xC6, 1); break; in do_jit()
1928 EMIT2(0x66, 0xC7); break; in do_jit()
1931 EMIT(0xC7, 1); break; in do_jit()
1935 EMIT2(add_1reg(0x40, IA32_EAX), insn->off); in do_jit()
1937 EMIT1_off32(add_1reg(0x80, IA32_EAX), in do_jit()
1944 hi = imm32 & (1<<31) ? (u32)~0 : 0; in do_jit()
1945 EMIT2_off32(0xC7, add_1reg(0x80, IA32_EAX), in do_jit()
1958 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1962 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in do_jit()
1966 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
1970 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EDX)); in do_jit()
1974 EMIT(0x88, 1); break; in do_jit()
1976 EMIT2(0x66, 0x89); break; in do_jit()
1979 EMIT(0x89, 1); break; in do_jit()
1983 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX), in do_jit()
1986 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
1992 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, in do_jit()
1997 EMIT2(0x8B, add_2reg(0xC0, src_hi, in do_jit()
1999 EMIT1(0x89); in do_jit()
2001 EMIT2(add_2reg(0x40, IA32_EAX, in do_jit()
2005 EMIT1(add_2reg(0x80, IA32_EAX, in do_jit()
2019 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2023 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EAX)); in do_jit()
2027 EMIT2(0x0F, 0xB6); break; in do_jit()
2029 EMIT2(0x0F, 0xB7); break; in do_jit()
2032 EMIT(0x8B, 1); break; in do_jit()
2036 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX), in do_jit()
2039 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
2044 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
2048 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EDX)); in do_jit()
2056 EMIT3(0xC7, add_1reg(0x40, IA32_EBP), in do_jit()
2058 EMIT(0x0, 4); in do_jit()
2061 EMIT2(0x33, in do_jit()
2062 add_2reg(0xC0, dst_hi, dst_hi)); in do_jit()
2066 EMIT2_off32(0x8B, in do_jit()
2067 add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
2070 EMIT3(0x89, in do_jit()
2071 add_2reg(0x40, IA32_EBP, in do_jit()
2075 EMIT2(0x89, in do_jit()
2076 add_2reg(0xC0, dst_hi, IA32_EDX)); in do_jit()
2116 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2117 STACK_VAR(r1[0])); in do_jit()
2119 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
2127 EMIT1_off32(0xE8, jmp_offset + 9); in do_jit()
2130 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2131 STACK_VAR(r0[0])); in do_jit()
2133 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
2137 EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 32); in do_jit()
2168 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2171 EMIT3(0x8B, in do_jit()
2172 add_2reg(0x40, IA32_EBP, in do_jit()
2178 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2181 EMIT3(0x8B, in do_jit()
2182 add_2reg(0x40, IA32_EBP, in do_jit()
2189 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2193 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2206 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2208 EMIT3(0x8B, in do_jit()
2209 add_2reg(0x40, IA32_EBP, in do_jit()
2215 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2217 EMIT3(0x8B, in do_jit()
2218 add_2reg(0x40, IA32_EBP, in do_jit()
2224 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2227 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2239 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2242 EMIT3(0x8B, in do_jit()
2243 add_2reg(0x40, IA32_EBP, in do_jit()
2248 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo)); in do_jit()
2251 EMIT2(0x89, in do_jit()
2252 add_2reg(0xC0, dreg_hi, dst_hi)); in do_jit()
2256 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2259 EMIT3(0x8B, in do_jit()
2260 add_2reg(0x40, IA32_EBP, in do_jit()
2265 EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); in do_jit()
2268 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); in do_jit()
2270 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); in do_jit()
2284 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2287 EMIT3(0x8B, in do_jit()
2288 add_2reg(0x40, IA32_EBP, in do_jit()
2293 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo)); in do_jit()
2296 EMIT2(0x89, in do_jit()
2297 add_2reg(0xC0, dreg_hi, dst_hi)); in do_jit()
2301 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_lo), imm32); in do_jit()
2304 EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); in do_jit()
2306 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2308 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_hi), hi); in do_jit()
2310 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); in do_jit()
2312 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); in do_jit()
2340 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2343 EMIT3(0x8B, in do_jit()
2344 add_2reg(0x40, IA32_EBP, in do_jit()
2350 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
2352 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2354 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi); in do_jit()
2356 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2360 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2369 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2387 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2389 EMIT3(0x8B, in do_jit()
2390 add_2reg(0x40, IA32_EBP, in do_jit()
2396 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
2397 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2399 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi); in do_jit()
2401 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2404 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2416 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2421 EMIT2(0xEB, 6); in do_jit()
2429 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2453 EMIT2(0xEB, jmp_offset); in do_jit()
2455 EMIT1_off32(0xE9, jmp_offset); in do_jit()
2525 int proglen, oldproglen = 0; in bpf_int_jit_compile()
2558 for (proglen = 0, i = 0; i < prog->len; i++) { in bpf_int_jit_compile()
2570 for (pass = 0; pass < 20 || image; pass++) { in bpf_int_jit_compile()
2572 if (proglen <= 0) { in bpf_int_jit_compile()