Lines Matching +full:0 +full:x48
33 do { prog = emit_code(prog, bytes, len); } while (0)
41 do { EMIT1(b1); EMIT(off, 4); } while (0)
43 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
45 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
47 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
74 EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
75 } while (0)
88 return 0; in bpf_size_to_x86_bytes()
93 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
95 #define X86_JB 0x72
96 #define X86_JAE 0x73
97 #define X86_JE 0x74
98 #define X86_JNE 0x75
99 #define X86_JBE 0x76
100 #define X86_JA 0x77
101 #define X86_JL 0x7C
102 #define X86_JGE 0x7D
103 #define X86_JLE 0x7E
104 #define X86_JG 0x7F
121 [BPF_REG_0] = 0, /* RAX */
126 [BPF_REG_5] = 0, /* R8 */
215 [BPF_ADD] = 0x01,
216 [BPF_SUB] = 0x29,
217 [BPF_AND] = 0x21,
218 [BPF_OR] = 0x09,
219 [BPF_XOR] = 0x31,
220 [BPF_LSH] = 0xE0,
221 [BPF_RSH] = 0xE8,
222 [BPF_ARSH] = 0xF8,
228 memset(area, 0xcc, size); in jit_fill_hole()
233 return IS_ERR_OR_NULL(text_poke_set(dst, 0xcc, len)); in bpf_arch_text_invalidate()
261 if (callee_regs_used[0]) in push_callee_regs()
262 EMIT1(0x53); /* push rbx */ in push_callee_regs()
264 EMIT2(0x41, 0x55); /* push r13 */ in push_callee_regs()
266 EMIT2(0x41, 0x56); /* push r14 */ in push_callee_regs()
268 EMIT2(0x41, 0x57); /* push r15 */ in push_callee_regs()
277 EMIT2(0x41, 0x5F); /* pop r15 */ in pop_callee_regs()
279 EMIT2(0x41, 0x5E); /* pop r14 */ in pop_callee_regs()
281 EMIT2(0x41, 0x5D); /* pop r13 */ in pop_callee_regs()
282 if (callee_regs_used[0]) in pop_callee_regs()
283 EMIT1(0x5B); /* pop rbx */ in pop_callee_regs()
305 EMIT2(0x31, 0xC0); /* xor eax, eax */ in emit_prologue()
307 EMIT2(0x66, 0x90); /* nop2 */ in emit_prologue()
309 EMIT1(0x55); /* push rbp */ in emit_prologue()
310 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in emit_prologue()
317 EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); in emit_prologue()
319 EMIT1(0x50); /* push rax */ in emit_prologue()
335 return 0; in emit_patch()
340 return emit_patch(pprog, func, ip, 0xE8); in emit_call()
345 return emit_patch(pprog, func, ip, 0xE9); in emit_jump()
384 ret = 0; in __bpf_arch_text_poke()
409 #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8)
417 EMIT2(0xFF, 0xE0 + reg); in emit_indirect_jump()
422 EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ in emit_indirect_jump()
424 EMIT1(0xCC); /* int3 */ in emit_indirect_jump()
437 EMIT1(0xC3); /* ret */ in emit_return()
439 EMIT1(0xCC); /* int3 */ in emit_return()
477 EMIT2(0x89, 0xD2); /* mov edx, edx */ in emit_bpf_tail_call_indirect()
478 EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ in emit_bpf_tail_call_indirect()
488 EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ in emit_bpf_tail_call_indirect()
489 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call_indirect()
493 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ in emit_bpf_tail_call_indirect()
494 EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ in emit_bpf_tail_call_indirect()
497 EMIT4_off32(0x48, 0x8B, 0x8C, 0xD6, /* mov rcx, [rsi + rdx * 8 + offsetof(...)] */ in emit_bpf_tail_call_indirect()
504 EMIT3(0x48, 0x85, 0xC9); /* test rcx,rcx */ in emit_bpf_tail_call_indirect()
511 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_indirect()
513 EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ in emit_bpf_tail_call_indirect()
517 EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ in emit_bpf_tail_call_indirect()
519 EMIT4(0x48, 0x83, 0xC1, /* add rcx, X86_TAIL_CALL_OFFSET */ in emit_bpf_tail_call_indirect()
546 EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ in emit_bpf_tail_call_direct()
547 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call_direct()
551 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ in emit_bpf_tail_call_direct()
552 EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ in emit_bpf_tail_call_direct()
563 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_direct()
565 EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); in emit_bpf_tail_call_direct()
583 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_tail_call_direct_fixup()
601 BUG_ON(ret < 0); in bpf_tail_call_direct_fixup()
606 BUG_ON(ret < 0); in bpf_tail_call_direct_fixup()
623 if (sign_propagate && (s32)imm32 < 0) { in emit_mov_imm32()
625 b1 = add_1mod(0x48, dst_reg); in emit_mov_imm32()
626 b2 = 0xC7; in emit_mov_imm32()
627 b3 = 0xC0; in emit_mov_imm32()
636 if (imm32 == 0) { in emit_mov_imm32()
638 EMIT1(add_2mod(0x40, dst_reg, dst_reg)); in emit_mov_imm32()
639 b2 = 0x31; /* xor */ in emit_mov_imm32()
640 b3 = 0xC0; in emit_mov_imm32()
647 EMIT1(add_1mod(0x40, dst_reg)); in emit_mov_imm32()
648 EMIT1_off32(add_1reg(0xB8, dst_reg), imm32); in emit_mov_imm32()
668 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); in emit_mov_imm64()
686 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in emit_mov_reg()
687 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg)); in emit_mov_reg()
701 * If off == 0 we could skip this and save one extra byte, but in emit_insn_suffix()
705 EMIT2(add_2reg(0x40, ptr_reg, val_reg), off); in emit_insn_suffix()
708 EMIT1_off32(add_2reg(0x80, ptr_reg, val_reg), off); in emit_insn_suffix()
721 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in maybe_emit_mod()
723 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in maybe_emit_mod()
735 EMIT1(add_1mod(0x48, reg)); in maybe_emit_1mod()
737 EMIT1(add_1mod(0x40, reg)); in maybe_emit_1mod()
749 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); in emit_ldx()
753 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); in emit_ldx()
756 /* Emit 'mov eax, dword ptr [rax+0x14]' */ in emit_ldx()
758 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); in emit_ldx()
760 EMIT1(0x8B); in emit_ldx()
763 /* Emit 'mov rax, qword ptr [rax+0x14]' */ in emit_ldx()
764 EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); in emit_ldx()
781 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); in emit_stx()
783 EMIT1(0x88); in emit_stx()
787 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89); in emit_stx()
789 EMIT2(0x66, 0x89); in emit_stx()
793 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89); in emit_stx()
795 EMIT1(0x89); in emit_stx()
798 EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89); in emit_stx()
810 EMIT1(0xF0); /* lock prefix */ in emit_atomic()
825 EMIT2(0x0F, 0xC1); in emit_atomic()
829 EMIT1(0x87); in emit_atomic()
833 EMIT2(0x0F, 0xB1); in emit_atomic()
843 return 0; in emit_atomic()
851 *(unsigned long *)((void *)regs + reg) = 0; in ex_handler_bpf()
852 regs->ip += x->fixup & 0xff; in ex_handler_bpf()
865 regs_used[0] = true; in detect_reg_usage()
880 while (len > 0) { in emit_nops()
886 for (i = 0; i < noplen; i++) in emit_nops()
906 int i, excnt = 0; in do_jit()
907 int ilen, proglen = 0; in do_jit()
919 bpf_prog->aux->func_idx != 0); in do_jit()
926 addrs[0] = proglen; in do_jit()
933 u8 b2 = 0, b3 = 0; in do_jit()
955 EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
970 EMIT2(0xF7, add_1reg(0xD8, dst_reg)); in do_jit()
992 b3 = 0xC0; in do_jit()
993 b2 = 0x05; in do_jit()
996 b3 = 0xE8; in do_jit()
997 b2 = 0x2D; in do_jit()
1000 b3 = 0xE0; in do_jit()
1001 b2 = 0x25; in do_jit()
1004 b3 = 0xC8; in do_jit()
1005 b2 = 0x0D; in do_jit()
1008 b3 = 0xF0; in do_jit()
1009 b2 = 0x35; in do_jit()
1014 EMIT3(0x83, add_1reg(b3, dst_reg), imm32); in do_jit()
1018 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); in do_jit()
1028 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
1045 EMIT1(0x50); /* push rax */ in do_jit()
1047 EMIT1(0x52); /* push rdx */ in do_jit()
1058 EMIT3_off32(0x49, 0xC7, 0xC3, imm32); in do_jit()
1070 EMIT2(0x31, 0xd2); in do_jit()
1074 EMIT2(0xF7, add_1reg(0xF0, src_reg)); in do_jit()
1086 EMIT1(0x5A); /* pop rdx */ in do_jit()
1088 EMIT1(0x58); /* pop rax */ in do_jit()
1099 EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg), in do_jit()
1103 EMIT2_off32(0x69, in do_jit()
1104 add_2reg(0xC0, dst_reg, dst_reg), in do_jit()
1114 EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg)); in do_jit()
1129 EMIT2(0xD1, add_1reg(b3, dst_reg)); in do_jit()
1131 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); in do_jit()
1149 EMIT1(0x51); /* push rcx */ in do_jit()
1160 EMIT2(0xD3, add_1reg(b3, dst_reg)); in do_jit()
1163 EMIT1(0x59); /* pop rcx */ in do_jit()
1174 EMIT1(0x66); in do_jit()
1176 EMIT1(0x41); in do_jit()
1177 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); in do_jit()
1181 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
1183 EMIT2(0x0F, 0xB7); in do_jit()
1184 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1189 EMIT2(0x41, 0x0F); in do_jit()
1191 EMIT1(0x0F); in do_jit()
1192 EMIT1(add_1reg(0xC8, dst_reg)); in do_jit()
1196 EMIT3(add_1mod(0x48, dst_reg), 0x0F, in do_jit()
1197 add_1reg(0xC8, dst_reg)); in do_jit()
1210 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
1212 EMIT2(0x0F, 0xB7); in do_jit()
1213 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1218 EMIT1(0x45); in do_jit()
1219 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1236 EMIT2(0x41, 0xC6); in do_jit()
1238 EMIT1(0xC6); in do_jit()
1242 EMIT3(0x66, 0x41, 0xC7); in do_jit()
1244 EMIT2(0x66, 0xC7); in do_jit()
1248 EMIT2(0x41, 0xC7); in do_jit()
1250 EMIT1(0xC7); in do_jit()
1253 EMIT2(add_1mod(0x48, dst_reg), 0xC7); in do_jit()
1256 EMIT2(add_1reg(0x40, dst_reg), insn->off); in do_jit()
1258 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off); in do_jit()
1297 EMIT2(add_1mod(0x48, AUX_REG), add_1reg(0xB8, AUX_REG)); in do_jit()
1302 EMIT2(0x39, add_2reg(0xC0, src_reg, AUX_REG)); in do_jit()
1304 EMIT2(X86_JB, 0); in do_jit()
1311 EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off); in do_jit()
1316 EMIT2(0x73 /* JNC */, 0); in do_jit()
1320 emit_mov_imm32(&prog, false, dst_reg, 0); in do_jit()
1322 EMIT2(0xEB, 0); in do_jit()
1369 * End result: x86 insn "mov rbx, qword ptr [rax+0x14]" in do_jit()
1409 add_2reg(0xC0, AUX_REG, real_src_reg)); in do_jit()
1440 EMIT3_off32(0x48, 0x8B, 0x85, in do_jit()
1489 EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
1497 EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
1505 EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); in do_jit()
1529 if (imm32 == 0) { in do_jit()
1532 EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1541 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); in do_jit()
1543 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); in do_jit()
1598 * "nops" is 0. in do_jit()
1608 if (nops != 0 && nops != 4) { in do_jit()
1617 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
1646 * to pad any extra byte (0 byte). in do_jit()
1654 if (nops != 0 && nops != 2 && nops != 5) { in do_jit()
1671 * jmp, there is nothing to pad (0 byte). in do_jit()
1679 if (nops != 0 && nops != 3) { in do_jit()
1686 EMIT2(0xEB, jmp_offset); in do_jit()
1688 EMIT1_off32(0xE9, jmp_offset); in do_jit()
1704 EMIT1(0xC9); /* leave */ in do_jit()
1759 * mov QWORD PTR [rbp-0x10],rdi in save_regs()
1760 * mov QWORD PTR [rbp-0x8],rsi in save_regs()
1762 for (i = 0, j = 0; i < min(nr_args, 6); i++) { in save_regs()
1789 * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10] in restore_regs()
1790 * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8] in restore_regs()
1792 for (i = 0, j = 0; i < min(nr_args, 6); i++) { in restore_regs()
1852 EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); in invoke_bpf_prog()
1859 /* if (__bpf_prog_enter*(prog) == 0) in invoke_bpf_prog()
1862 EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ in invoke_bpf_prog()
1868 EMIT4(0x48, 0x8D, 0x7D, -stack_size); in invoke_bpf_prog()
1890 jmp_insn[0] = X86_JE; in invoke_bpf_prog()
1898 EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); in invoke_bpf_prog()
1903 return 0; in invoke_bpf_prog()
1927 EMIT2_off32(0x0F, jmp_cond + 0x10, offset); in emit_cond_near_jump()
1929 return 0; in emit_cond_near_jump()
1939 for (i = 0; i < tl->nr_links; i++) { in invoke_bpf()
1945 return 0; in invoke_bpf()
1956 * Set this to 0 to avoid confusing the program. in invoke_bpf_mod_ret()
1958 emit_mov_imm32(&prog, false, BPF_REG_0, 0); in invoke_bpf_mod_ret()
1960 for (i = 0; i < tl->nr_links; i++) { in invoke_bpf_mod_ret()
1965 * if (*(u64 *)(rbp - 8) != 0) in invoke_bpf_mod_ret()
1968 /* cmp QWORD PTR [rbp - 0x8], 0x0 */ in invoke_bpf_mod_ret()
1969 EMIT4(0x48, 0x83, 0x7d, 0xf8); EMIT1(0x00); in invoke_bpf_mod_ret()
1981 return 0; in invoke_bpf_mod_ret()
2049 int ret, i, nr_args = m->nr_args, extra_nregs = 0; in arch_prepare_bpf_trampoline()
2063 for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { in arch_prepare_bpf_trampoline()
2074 * RBP + 0 [ RBP ] in arch_prepare_bpf_trampoline()
2106 stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; in arch_prepare_bpf_trampoline()
2121 EMIT1(0x55); /* push rbp */ in arch_prepare_bpf_trampoline()
2122 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in arch_prepare_bpf_trampoline()
2123 EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */ in arch_prepare_bpf_trampoline()
2124 EMIT1(0x53); /* push rbx */ in arch_prepare_bpf_trampoline()
2130 emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args + extra_nregs); in arch_prepare_bpf_trampoline()
2176 EMIT2(0xff, 0xd0); /* call *rax */ in arch_prepare_bpf_trampoline()
2201 for (i = 0; i < fmod_ret->nr_links; i++) in arch_prepare_bpf_trampoline()
2232 EMIT1(0x5B); /* pop rbx */ in arch_prepare_bpf_trampoline()
2233 EMIT1(0xC9); /* leave */ in arch_prepare_bpf_trampoline()
2236 EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ in arch_prepare_bpf_trampoline()
2260 EMIT1(add_1mod(0x48, BPF_REG_3)); /* cmp rdx,func */ in emit_bpf_dispatcher()
2263 EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), in emit_bpf_dispatcher()
2274 return 0; in emit_bpf_dispatcher()
2281 EMIT1(add_1mod(0x48, BPF_REG_3)); /* cmp rdx,func */ in emit_bpf_dispatcher()
2284 EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), progs[a + pivot]); in emit_bpf_dispatcher()
2289 EMIT2_off32(0x0F, X86_JG + 0x10, 0); in emit_bpf_dispatcher()
2291 EMIT2(X86_JG, 0); in emit_bpf_dispatcher()
2315 return 0; in emit_bpf_dispatcher()
2327 return 0; in cmp_ips()
2334 sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL); in arch_prepare_bpf_dispatcher()
2335 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf); in arch_prepare_bpf_dispatcher()
2356 int proglen, oldproglen = 0; in bpf_int_jit_compile()
2413 for (proglen = 0, i = 0; i <= prog->len; i++) { in bpf_int_jit_compile()
2426 for (pass = 0; pass < MAX_PASSES || image; pass++) { in bpf_int_jit_compile()
2430 if (proglen <= 0) { in bpf_int_jit_compile()
2442 prog->jited = 0; in bpf_int_jit_compile()
2443 prog->jited_len = 0; in bpf_int_jit_compile()