Lines Matching +full:gen +full:- +full:2
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
25 * ctx - bpf program context
26 * stack - bpf program stack
27 * blob - bpf_attr-s, strings, insns, map data.
38 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
42 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) in realloc_insn_buf() argument
44 size_t off = gen->insn_cur - gen->insn_start; in realloc_insn_buf()
47 if (gen->error) in realloc_insn_buf()
48 return gen->error; in realloc_insn_buf()
50 gen->error = -ERANGE; in realloc_insn_buf()
51 return -ERANGE; in realloc_insn_buf()
53 insn_start = realloc(gen->insn_start, off + size); in realloc_insn_buf()
55 gen->error = -ENOMEM; in realloc_insn_buf()
56 free(gen->insn_start); in realloc_insn_buf()
57 gen->insn_start = NULL; in realloc_insn_buf()
58 return -ENOMEM; in realloc_insn_buf()
60 gen->insn_start = insn_start; in realloc_insn_buf()
61 gen->insn_cur = insn_start + off; in realloc_insn_buf()
65 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) in realloc_data_buf() argument
67 size_t off = gen->data_cur - gen->data_start; in realloc_data_buf()
70 if (gen->error) in realloc_data_buf()
71 return gen->error; in realloc_data_buf()
73 gen->error = -ERANGE; in realloc_data_buf()
74 return -ERANGE; in realloc_data_buf()
76 data_start = realloc(gen->data_start, off + size); in realloc_data_buf()
78 gen->error = -ENOMEM; in realloc_data_buf()
79 free(gen->data_start); in realloc_data_buf()
80 gen->data_start = NULL; in realloc_data_buf()
81 return -ENOMEM; in realloc_data_buf()
83 gen->data_start = data_start; in realloc_data_buf()
84 gen->data_cur = data_start + off; in realloc_data_buf()
88 static void emit(struct bpf_gen *gen, struct bpf_insn insn) in emit() argument
90 if (realloc_insn_buf(gen, sizeof(insn))) in emit()
92 memcpy(gen->insn_cur, &insn, sizeof(insn)); in emit()
93 gen->insn_cur += sizeof(insn); in emit()
96 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) in emit2() argument
98 emit(gen, insn1); in emit2()
99 emit(gen, insn2); in emit2()
102 void bpf_gen__init(struct bpf_gen *gen, int log_level) in bpf_gen__init() argument
107 gen->log_level = log_level; in bpf_gen__init()
109 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); in bpf_gen__init()
112 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); in bpf_gen__init()
113 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); in bpf_gen__init()
114 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); in bpf_gen__init()
115 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in bpf_gen__init()
116 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); in bpf_gen__init()
119 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, in bpf_gen__init()
121 (stack_sz / 4) * 3 + 2)); in bpf_gen__init()
124 gen->cleanup_label = gen->insn_cur - gen->insn_start; in bpf_gen__init()
127 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); in bpf_gen__init()
128 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); in bpf_gen__init()
129 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in bpf_gen__init()
132 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); in bpf_gen__init()
133 emit(gen, BPF_EXIT_INSN()); in bpf_gen__init()
136 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) in add_data() argument
140 if (realloc_data_buf(gen, size)) in add_data()
142 prev = gen->data_cur; in add_data()
143 memcpy(gen->data_cur, data, size); in add_data()
144 gen->data_cur += size; in add_data()
145 return prev - gen->data_start; in add_data()
153 case 2: return BPF_H; in insn_bytes_to_bpf_size()
155 default: return -1; in insn_bytes_to_bpf_size()
160 static void emit_rel_store(struct bpf_gen *gen, int off, int data) in emit_rel_store() argument
162 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
164 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
166 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); in emit_rel_store()
170 static void emit_rel_store_sp(struct bpf_gen *gen, int off, int stack_off) in emit_rel_store_sp() argument
172 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_10)); in emit_rel_store_sp()
173 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, stack_off)); in emit_rel_store_sp()
174 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store_sp()
176 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); in emit_rel_store_sp()
179 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, in move_ctx2blob() argument
182 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); in move_ctx2blob()
185 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c in move_ctx2blob()
187 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); in move_ctx2blob()
188 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_ctx2blob()
190 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_ctx2blob()
193 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) in move_stack2blob() argument
195 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2blob()
196 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_stack2blob()
198 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_stack2blob()
201 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) in move_stack2ctx() argument
203 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2ctx()
204 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_stack2ctx()
207 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) in emit_sys_bpf() argument
209 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); in emit_sys_bpf()
210 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_bpf()
212 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); in emit_sys_bpf()
213 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); in emit_sys_bpf()
215 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_sys_bpf()
223 static void emit_check_err(struct bpf_gen *gen) in emit_check_err() argument
225 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; in emit_check_err()
231 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); in emit_check_err()
233 gen->error = -ERANGE; in emit_check_err()
234 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); in emit_check_err()
238 /* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
239 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, in emit_debug() argument
245 if (!gen->log_level) in emit_debug()
248 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0) in emit_debug()
255 addr = add_data(gen, buf, len); in emit_debug()
257 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_debug()
259 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_debug()
261 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); in emit_debug()
263 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); in emit_debug()
264 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); in emit_debug()
267 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) in debug_regs() argument
272 emit_debug(gen, reg1, reg2, fmt, args); in debug_regs()
276 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) in debug_ret() argument
281 emit_debug(gen, BPF_REG_7, -1, fmt, args); in debug_ret()
285 static void __emit_sys_close(struct bpf_gen *gen) in __emit_sys_close() argument
287 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, in __emit_sys_close()
288 /* 2 is the number of the following insns in __emit_sys_close()
291 2 + (gen->log_level ? 6 : 0))); in __emit_sys_close()
292 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); in __emit_sys_close()
293 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in __emit_sys_close()
294 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); in __emit_sys_close()
297 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) in emit_sys_close_stack() argument
299 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); in emit_sys_close_stack()
300 __emit_sys_close(gen); in emit_sys_close_stack()
303 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) in emit_sys_close_blob() argument
305 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_close_blob()
307 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); in emit_sys_close_blob()
308 __emit_sys_close(gen); in emit_sys_close_blob()
311 int bpf_gen__finish(struct bpf_gen *gen) in bpf_gen__finish() argument
315 emit_sys_close_stack(gen, stack_off(btf_fd)); in bpf_gen__finish()
316 for (i = 0; i < gen->nr_progs; i++) in bpf_gen__finish()
317 move_stack2ctx(gen, in bpf_gen__finish()
319 sizeof(struct bpf_map_desc) * gen->nr_maps + in bpf_gen__finish()
323 for (i = 0; i < gen->nr_maps; i++) in bpf_gen__finish()
324 move_stack2ctx(gen, in bpf_gen__finish()
329 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); in bpf_gen__finish()
330 emit(gen, BPF_EXIT_INSN()); in bpf_gen__finish()
331 pr_debug("gen: finish %d\n", gen->error); in bpf_gen__finish()
332 if (!gen->error) { in bpf_gen__finish()
333 struct gen_loader_opts *opts = gen->opts; in bpf_gen__finish()
335 opts->insns = gen->insn_start; in bpf_gen__finish()
336 opts->insns_sz = gen->insn_cur - gen->insn_start; in bpf_gen__finish()
337 opts->data = gen->data_start; in bpf_gen__finish()
338 opts->data_sz = gen->data_cur - gen->data_start; in bpf_gen__finish()
340 return gen->error; in bpf_gen__finish()
343 void bpf_gen__free(struct bpf_gen *gen) in bpf_gen__free() argument
345 if (!gen) in bpf_gen__free()
347 free(gen->data_start); in bpf_gen__free()
348 free(gen->insn_start); in bpf_gen__free()
349 free(gen); in bpf_gen__free()
352 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, in bpf_gen__load_btf() argument
360 pr_debug("gen: load_btf: size %d\n", btf_raw_size); in bpf_gen__load_btf()
361 btf_data = add_data(gen, btf_raw_data, btf_raw_size); in bpf_gen__load_btf()
364 btf_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__load_btf()
367 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, in bpf_gen__load_btf()
369 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, in bpf_gen__load_btf()
371 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, in bpf_gen__load_btf()
374 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); in bpf_gen__load_btf()
376 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); in bpf_gen__load_btf()
377 debug_ret(gen, "btf_load size %d", btf_raw_size); in bpf_gen__load_btf()
378 emit_check_err(gen); in bpf_gen__load_btf()
380 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); in bpf_gen__load_btf()
383 void bpf_gen__map_create(struct bpf_gen *gen, in bpf_gen__map_create() argument
392 attr.map_type = map_attr->map_type; in bpf_gen__map_create()
393 attr.key_size = map_attr->key_size; in bpf_gen__map_create()
394 attr.value_size = map_attr->value_size; in bpf_gen__map_create()
395 attr.map_flags = map_attr->map_flags; in bpf_gen__map_create()
396 memcpy(attr.map_name, map_attr->name, in bpf_gen__map_create()
397 min((unsigned)strlen(map_attr->name), BPF_OBJ_NAME_LEN - 1)); in bpf_gen__map_create()
398 attr.numa_node = map_attr->numa_node; in bpf_gen__map_create()
399 attr.map_ifindex = map_attr->map_ifindex; in bpf_gen__map_create()
400 attr.max_entries = map_attr->max_entries; in bpf_gen__map_create()
418 attr.btf_key_type_id = map_attr->btf_key_type_id; in bpf_gen__map_create()
419 attr.btf_value_type_id = map_attr->btf_value_type_id; in bpf_gen__map_create()
422 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n", in bpf_gen__map_create()
423 attr.map_name, map_idx, map_attr->map_type, attr.btf_value_type_id); in bpf_gen__map_create()
425 map_create_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_create()
428 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, in bpf_gen__map_create()
433 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, in bpf_gen__map_create()
442 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, in bpf_gen__map_create()
448 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); in bpf_gen__map_create()
449 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", in bpf_gen__map_create()
450 attr.map_name, map_idx, map_attr->map_type, attr.value_size, in bpf_gen__map_create()
452 emit_check_err(gen); in bpf_gen__map_create()
457 * It's called with -1 to create an inner map. in bpf_gen__map_create()
459 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__map_create()
461 } else if (map_idx != gen->nr_maps) { in bpf_gen__map_create()
462 gen->error = -EDOM; /* internal bug */ in bpf_gen__map_create()
465 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__map_create()
467 gen->nr_maps++; in bpf_gen__map_create()
470 emit_sys_close_stack(gen, stack_off(inner_map_fd)); in bpf_gen__map_create()
473 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, in bpf_gen__record_attach_target() argument
480 gen->attach_kind = kind; in bpf_gen__record_attach_target()
481 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", in bpf_gen__record_attach_target()
483 if (ret == sizeof(gen->attach_target)) in bpf_gen__record_attach_target()
484 gen->error = -ENOSPC; in bpf_gen__record_attach_target()
487 static void emit_find_attach_target(struct bpf_gen *gen) in emit_find_attach_target() argument
489 int name, len = strlen(gen->attach_target) + 1; in emit_find_attach_target()
491 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); in emit_find_attach_target()
492 name = add_data(gen, gen->attach_target, len); in emit_find_attach_target()
494 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_find_attach_target()
496 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_find_attach_target()
497 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); in emit_find_attach_target()
498 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_find_attach_target()
499 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_find_attach_target()
500 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_find_attach_target()
501 debug_ret(gen, "find_by_name_kind(%s,%d)", in emit_find_attach_target()
502 gen->attach_target, gen->attach_kind); in emit_find_attach_target()
503 emit_check_err(gen); in emit_find_attach_target()
504 /* if successful, btf_id is in lower 32-bit of R7 and in emit_find_attach_target()
505 * btf_obj_fd is in upper 32-bit in emit_find_attach_target()
509 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind, in bpf_gen__record_extern() argument
514 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); in bpf_gen__record_extern()
516 gen->error = -ENOMEM; in bpf_gen__record_extern()
519 gen->relos = relo; in bpf_gen__record_extern()
520 relo += gen->relo_cnt; in bpf_gen__record_extern()
521 relo->name = name; in bpf_gen__record_extern()
522 relo->kind = kind; in bpf_gen__record_extern()
523 relo->insn_idx = insn_idx; in bpf_gen__record_extern()
524 gen->relo_cnt++; in bpf_gen__record_extern()
527 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) in emit_relo() argument
529 int name, insn, len = strlen(relo->name) + 1; in emit_relo()
531 pr_debug("gen: emit_relo: %s at %d\n", relo->name, relo->insn_idx); in emit_relo()
532 name = add_data(gen, relo->name, len); in emit_relo()
534 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo()
536 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_relo()
537 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); in emit_relo()
538 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_relo()
539 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_relo()
540 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_relo()
541 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); in emit_relo()
542 emit_check_err(gen); in emit_relo()
544 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx + in emit_relo()
546 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo()
548 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 0)); in emit_relo()
549 if (relo->kind == BTF_KIND_VAR) { in emit_relo()
551 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in emit_relo()
552 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in emit_relo()
557 static void emit_relos(struct bpf_gen *gen, int insns) in emit_relos() argument
561 for (i = 0; i < gen->relo_cnt; i++) in emit_relos()
562 emit_relo(gen, gen->relos + i, insns); in emit_relos()
565 static void cleanup_relos(struct bpf_gen *gen, int insns) in cleanup_relos() argument
569 for (i = 0; i < gen->relo_cnt; i++) { in cleanup_relos()
570 if (gen->relos[i].kind != BTF_KIND_VAR) in cleanup_relos()
574 sizeof(struct bpf_insn) * (gen->relos[i].insn_idx + 1) + in cleanup_relos()
576 emit_sys_close_blob(gen, insn); in cleanup_relos()
578 if (gen->relo_cnt) { in cleanup_relos()
579 free(gen->relos); in cleanup_relos()
580 gen->relo_cnt = 0; in cleanup_relos()
581 gen->relos = NULL; in cleanup_relos()
585 void bpf_gen__prog_load(struct bpf_gen *gen, in bpf_gen__prog_load() argument
593 pr_debug("gen: prog_load: type %d insns_cnt %zd\n", in bpf_gen__prog_load()
594 load_attr->prog_type, load_attr->insn_cnt); in bpf_gen__prog_load()
596 license = add_data(gen, load_attr->license, strlen(load_attr->license) + 1); in bpf_gen__prog_load()
598 insns = add_data(gen, load_attr->insns, in bpf_gen__prog_load()
599 load_attr->insn_cnt * sizeof(struct bpf_insn)); in bpf_gen__prog_load()
601 attr.prog_type = load_attr->prog_type; in bpf_gen__prog_load()
602 attr.expected_attach_type = load_attr->expected_attach_type; in bpf_gen__prog_load()
603 attr.attach_btf_id = load_attr->attach_btf_id; in bpf_gen__prog_load()
604 attr.prog_ifindex = load_attr->prog_ifindex; in bpf_gen__prog_load()
606 attr.insn_cnt = (__u32)load_attr->insn_cnt; in bpf_gen__prog_load()
607 attr.prog_flags = load_attr->prog_flags; in bpf_gen__prog_load()
609 attr.func_info_rec_size = load_attr->func_info_rec_size; in bpf_gen__prog_load()
610 attr.func_info_cnt = load_attr->func_info_cnt; in bpf_gen__prog_load()
611 func_info = add_data(gen, load_attr->func_info, in bpf_gen__prog_load()
614 attr.line_info_rec_size = load_attr->line_info_rec_size; in bpf_gen__prog_load()
615 attr.line_info_cnt = load_attr->line_info_cnt; in bpf_gen__prog_load()
616 line_info = add_data(gen, load_attr->line_info, in bpf_gen__prog_load()
619 memcpy(attr.prog_name, load_attr->name, in bpf_gen__prog_load()
620 min((unsigned)strlen(load_attr->name), BPF_OBJ_NAME_LEN - 1)); in bpf_gen__prog_load()
621 prog_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__prog_load()
624 emit_rel_store(gen, attr_field(prog_load_attr, license), license); in bpf_gen__prog_load()
627 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns); in bpf_gen__prog_load()
630 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); in bpf_gen__prog_load()
633 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); in bpf_gen__prog_load()
636 emit_rel_store_sp(gen, attr_field(prog_load_attr, fd_array), in bpf_gen__prog_load()
640 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, in bpf_gen__prog_load()
642 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, in bpf_gen__prog_load()
644 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, in bpf_gen__prog_load()
647 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, in bpf_gen__prog_load()
649 if (gen->attach_kind) { in bpf_gen__prog_load()
650 emit_find_attach_target(gen); in bpf_gen__prog_load()
652 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__prog_load()
654 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
656 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in bpf_gen__prog_load()
657 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
660 emit_relos(gen, insns); in bpf_gen__prog_load()
662 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); in bpf_gen__prog_load()
663 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); in bpf_gen__prog_load()
665 cleanup_relos(gen, insns); in bpf_gen__prog_load()
666 if (gen->attach_kind) in bpf_gen__prog_load()
667 emit_sys_close_blob(gen, in bpf_gen__prog_load()
669 emit_check_err(gen); in bpf_gen__prog_load()
671 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__prog_load()
672 stack_off(prog_fd[gen->nr_progs]))); in bpf_gen__prog_load()
673 gen->nr_progs++; in bpf_gen__prog_load()
676 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, in bpf_gen__map_update_elem() argument
685 pr_debug("gen: map_update_elem: idx %d\n", map_idx); in bpf_gen__map_update_elem()
687 value = add_data(gen, pvalue, value_size); in bpf_gen__map_update_elem()
688 key = add_data(gen, &zero, sizeof(zero)); in bpf_gen__map_update_elem()
693 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, in bpf_gen__map_update_elem()
697 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 4)); in bpf_gen__map_update_elem()
698 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_update_elem()
700 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); in bpf_gen__map_update_elem()
701 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); in bpf_gen__map_update_elem()
703 map_update_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_update_elem()
704 move_stack2blob(gen, attr_field(map_update_attr, map_fd), 4, in bpf_gen__map_update_elem()
706 emit_rel_store(gen, attr_field(map_update_attr, key), key); in bpf_gen__map_update_elem()
707 emit_rel_store(gen, attr_field(map_update_attr, value), value); in bpf_gen__map_update_elem()
709 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); in bpf_gen__map_update_elem()
710 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); in bpf_gen__map_update_elem()
711 emit_check_err(gen); in bpf_gen__map_update_elem()
714 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) in bpf_gen__map_freeze() argument
721 pr_debug("gen: map_freeze: idx %d\n", map_idx); in bpf_gen__map_freeze()
722 map_freeze_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_freeze()
723 move_stack2blob(gen, attr_field(map_freeze_attr, map_fd), 4, in bpf_gen__map_freeze()
726 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); in bpf_gen__map_freeze()
727 debug_ret(gen, "map_freeze"); in bpf_gen__map_freeze()
728 emit_check_err(gen); in bpf_gen__map_freeze()