Lines Matching refs:btf
244 struct btf { struct
263 struct btf *base_btf; argument
303 struct btf *btf; member
409 const struct btf *btf; member
448 void (*show)(const struct btf *btf, const struct btf_type *t,
524 u32 btf_nr_types(const struct btf *btf) in btf_nr_types() argument
528 while (btf) { in btf_nr_types()
529 total += btf->nr_types; in btf_nr_types()
530 btf = btf->base_btf; in btf_nr_types()
536 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind) in btf_find_by_name_kind() argument
542 total = btf_nr_types(btf); in btf_find_by_name_kind()
544 t = btf_type_by_id(btf, i); in btf_find_by_name_kind()
548 tname = btf_name_by_offset(btf, t->name_off); in btf_find_by_name_kind()
556 s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p) in bpf_find_btf_id()
558 struct btf *btf; in bpf_find_btf_id() local
562 btf = bpf_get_btf_vmlinux(); in bpf_find_btf_id()
563 if (IS_ERR(btf)) in bpf_find_btf_id()
564 return PTR_ERR(btf); in bpf_find_btf_id()
565 if (!btf) in bpf_find_btf_id()
568 ret = btf_find_by_name_kind(btf, name, kind); in bpf_find_btf_id()
573 btf_get(btf); in bpf_find_btf_id()
574 *btf_p = btf; in bpf_find_btf_id()
580 idr_for_each_entry(&btf_idr, btf, id) { in bpf_find_btf_id()
581 if (!btf_is_module(btf)) in bpf_find_btf_id()
586 btf_get(btf); in bpf_find_btf_id()
588 ret = btf_find_by_name_kind(btf, name, kind); in bpf_find_btf_id()
590 *btf_p = btf; in bpf_find_btf_id()
593 btf_put(btf); in bpf_find_btf_id()
600 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf, in btf_type_skip_modifiers() argument
603 const struct btf_type *t = btf_type_by_id(btf, id); in btf_type_skip_modifiers()
607 t = btf_type_by_id(btf, t->type); in btf_type_skip_modifiers()
616 const struct btf_type *btf_type_resolve_ptr(const struct btf *btf, in btf_type_resolve_ptr() argument
621 t = btf_type_skip_modifiers(btf, id, NULL); in btf_type_resolve_ptr()
625 return btf_type_skip_modifiers(btf, t->type, res_id); in btf_type_resolve_ptr()
628 const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, in btf_type_resolve_func_ptr() argument
633 ptype = btf_type_resolve_ptr(btf, id, res_id); in btf_type_resolve_func_ptr()
744 static bool btf_name_offset_valid(const struct btf *btf, u32 offset) in btf_name_offset_valid() argument
749 while (offset < btf->start_str_off) in btf_name_offset_valid()
750 btf = btf->base_btf; in btf_name_offset_valid()
752 offset -= btf->start_str_off; in btf_name_offset_valid()
753 return offset < btf->hdr.str_len; in btf_name_offset_valid()
766 static const char *btf_str_by_offset(const struct btf *btf, u32 offset) in btf_str_by_offset() argument
768 while (offset < btf->start_str_off) in btf_str_by_offset()
769 btf = btf->base_btf; in btf_str_by_offset()
771 offset -= btf->start_str_off; in btf_str_by_offset()
772 if (offset < btf->hdr.str_len) in btf_str_by_offset()
773 return &btf->strings[offset]; in btf_str_by_offset()
778 static bool __btf_name_valid(const struct btf *btf, u32 offset) in __btf_name_valid() argument
781 const char *src = btf_str_by_offset(btf, offset); in __btf_name_valid()
799 static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) in btf_name_valid_identifier() argument
801 return __btf_name_valid(btf, offset); in btf_name_valid_identifier()
804 static bool btf_name_valid_section(const struct btf *btf, u32 offset) in btf_name_valid_section() argument
806 return __btf_name_valid(btf, offset); in btf_name_valid_section()
809 static const char *__btf_name_by_offset(const struct btf *btf, u32 offset) in __btf_name_by_offset() argument
816 name = btf_str_by_offset(btf, offset); in __btf_name_by_offset()
820 const char *btf_name_by_offset(const struct btf *btf, u32 offset) in btf_name_by_offset() argument
822 return btf_str_by_offset(btf, offset); in btf_name_by_offset()
825 const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) in btf_type_by_id() argument
827 while (type_id < btf->start_id) in btf_type_by_id()
828 btf = btf->base_btf; in btf_type_by_id()
830 type_id -= btf->start_id; in btf_type_by_id()
831 if (type_id >= btf->nr_types) in btf_type_by_id()
833 return btf->types[type_id]; in btf_type_by_id()
864 bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, in btf_member_is_reg_int() argument
873 t = btf_type_id_size(btf, &id, NULL); in btf_member_is_reg_int()
902 static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, in btf_type_skip_qualifiers() argument
905 const struct btf_type *t = btf_type_by_id(btf, id); in btf_type_skip_qualifiers()
909 t = btf_type_by_id(btf, t->type); in btf_type_skip_qualifiers()
955 member = btf_name_by_offset(show->btf, m->name_off); in btf_show_name()
967 t = btf_type_by_id(show->btf, id); in btf_show_name()
998 name = btf_name_by_offset(show->btf, in btf_show_name()
1025 t = btf_type_skip_qualifiers(show->btf, id); in btf_show_name()
1032 name = btf_name_by_offset(show->btf, t->name_off); in btf_show_name()
1222 rt = btf_resolve_size(show->btf, t, &size); in btf_show_obj_safe()
1414 struct btf *btf = env->btf; in __btf_verifier_log_type() local
1429 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in __btf_verifier_log_type()
1436 __btf_name_by_offset(btf, t->name_off), in __btf_verifier_log_type()
1464 struct btf *btf = env->btf; in btf_verifier_log_member() local
1475 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in btf_verifier_log_member()
1491 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1497 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1542 const struct btf *btf = env->btf; in btf_verifier_log_hdr() local
1550 hdr = &btf->hdr; in btf_verifier_log_hdr()
1564 struct btf *btf = env->btf; in btf_add_type() local
1566 if (btf->types_size == btf->nr_types) { in btf_add_type()
1572 if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { in btf_add_type()
1577 expand_by = max_t(u32, btf->types_size >> 2, 16); in btf_add_type()
1579 btf->types_size + expand_by); in btf_add_type()
1586 if (btf->nr_types == 0) { in btf_add_type()
1587 if (!btf->base_btf) { in btf_add_type()
1590 btf->nr_types++; in btf_add_type()
1593 memcpy(new_types, btf->types, in btf_add_type()
1594 sizeof(*btf->types) * btf->nr_types); in btf_add_type()
1597 kvfree(btf->types); in btf_add_type()
1598 btf->types = new_types; in btf_add_type()
1599 btf->types_size = new_size; in btf_add_type()
1602 btf->types[btf->nr_types++] = t; in btf_add_type()
1607 static int btf_alloc_id(struct btf *btf) in btf_alloc_id() argument
1613 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC); in btf_alloc_id()
1615 btf->id = id; in btf_alloc_id()
1625 static void btf_free_id(struct btf *btf) in btf_free_id() argument
1639 idr_remove(&btf_idr, btf->id); in btf_free_id()
1643 static void btf_free_kfunc_set_tab(struct btf *btf) in btf_free_kfunc_set_tab() argument
1645 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; in btf_free_kfunc_set_tab()
1653 if (btf_is_module(btf)) in btf_free_kfunc_set_tab()
1659 btf->kfunc_set_tab = NULL; in btf_free_kfunc_set_tab()
1662 static void btf_free_dtor_kfunc_tab(struct btf *btf) in btf_free_dtor_kfunc_tab() argument
1664 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_free_dtor_kfunc_tab()
1669 btf->dtor_kfunc_tab = NULL; in btf_free_dtor_kfunc_tab()
1683 static void btf_free_struct_meta_tab(struct btf *btf) in btf_free_struct_meta_tab() argument
1685 struct btf_struct_metas *tab = btf->struct_meta_tab; in btf_free_struct_meta_tab()
1688 btf->struct_meta_tab = NULL; in btf_free_struct_meta_tab()
1691 static void btf_free(struct btf *btf) in btf_free() argument
1693 btf_free_struct_meta_tab(btf); in btf_free()
1694 btf_free_dtor_kfunc_tab(btf); in btf_free()
1695 btf_free_kfunc_set_tab(btf); in btf_free()
1696 kvfree(btf->types); in btf_free()
1697 kvfree(btf->resolved_sizes); in btf_free()
1698 kvfree(btf->resolved_ids); in btf_free()
1699 kvfree(btf->data); in btf_free()
1700 kfree(btf); in btf_free()
1705 struct btf *btf = container_of(rcu, struct btf, rcu); in btf_free_rcu() local
1707 btf_free(btf); in btf_free_rcu()
1710 void btf_get(struct btf *btf) in btf_get() argument
1712 refcount_inc(&btf->refcnt); in btf_get()
1715 void btf_put(struct btf *btf) in btf_put() argument
1717 if (btf && refcount_dec_and_test(&btf->refcnt)) { in btf_put()
1718 btf_free_id(btf); in btf_put()
1719 call_rcu(&btf->rcu, btf_free_rcu); in btf_put()
1725 struct btf *btf = env->btf; in env_resolve_init() local
1726 u32 nr_types = btf->nr_types; in env_resolve_init()
1746 btf->resolved_sizes = resolved_sizes; in env_resolve_init()
1747 btf->resolved_ids = resolved_ids; in env_resolve_init()
1794 if (type_id < env->btf->start_id) in env_type_is_resolved()
1797 return env->visit_states[type_id - env->btf->start_id] == RESOLVED; in env_type_is_resolved()
1803 const struct btf *btf = env->btf; in env_stack_push() local
1809 if (type_id < btf->start_id in env_stack_push()
1810 || env->visit_states[type_id - btf->start_id] != NOT_VISITED) in env_stack_push()
1813 env->visit_states[type_id - btf->start_id] = VISITED; in env_stack_push()
1841 struct btf *btf = env->btf; in env_stack_pop_resolved() local
1843 type_id -= btf->start_id; /* adjust to local type id */ in env_stack_pop_resolved()
1844 btf->resolved_sizes[type_id] = resolved_size; in env_stack_pop_resolved()
1845 btf->resolved_ids[type_id] = resolved_type_id; in env_stack_pop_resolved()
1875 __btf_resolve_size(const struct btf *btf, const struct btf_type *type, in __btf_resolve_size() argument
1906 type = btf_type_by_id(btf, type->type); in __btf_resolve_size()
1916 type = btf_type_by_id(btf, array->type); in __btf_resolve_size()
1945 btf_resolve_size(const struct btf *btf, const struct btf_type *type, in btf_resolve_size() argument
1948 return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL); in btf_resolve_size()
1951 static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id) in btf_resolved_type_id() argument
1953 while (type_id < btf->start_id) in btf_resolved_type_id()
1954 btf = btf->base_btf; in btf_resolved_type_id()
1956 return btf->resolved_ids[type_id - btf->start_id]; in btf_resolved_type_id()
1960 static const struct btf_type *btf_type_id_resolve(const struct btf *btf, in btf_type_id_resolve() argument
1963 *type_id = btf_resolved_type_id(btf, *type_id); in btf_type_id_resolve()
1964 return btf_type_by_id(btf, *type_id); in btf_type_id_resolve()
1967 static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id) in btf_resolved_type_size() argument
1969 while (type_id < btf->start_id) in btf_resolved_type_size()
1970 btf = btf->base_btf; in btf_resolved_type_size()
1972 return btf->resolved_sizes[type_id - btf->start_id]; in btf_resolved_type_size()
1975 const struct btf_type *btf_type_id_size(const struct btf *btf, in btf_type_id_size() argument
1982 size_type = btf_type_by_id(btf, size_type_id); in btf_type_id_size()
1989 size = btf_resolved_type_size(btf, size_type_id); in btf_type_id_size()
1997 size_type_id = btf_resolved_type_id(btf, size_type_id); in btf_type_id_size()
1998 size_type = btf_type_by_id(btf, size_type_id); in btf_type_id_size()
2004 size = btf_resolved_type_size(btf, size_type_id); in btf_type_id_size()
2067 static void btf_df_show(const struct btf *btf, const struct btf_type *t, in btf_df_show() argument
2335 static void btf_int_bits_show(const struct btf *btf, in btf_int_bits_show() argument
2354 static void btf_int_show(const struct btf *btf, const struct btf_type *t, in btf_int_show() argument
2370 btf_int_bits_show(btf, t, safe_data, bits_offset, show); in btf_int_show()
2417 btf_int_bits_show(btf, t, safe_data, bits_offset, show); in btf_int_show()
2441 struct btf *btf = env->btf; in btf_modifier_check_member() local
2443 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); in btf_modifier_check_member()
2466 struct btf *btf = env->btf; in btf_modifier_check_kflag_member() local
2468 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); in btf_modifier_check_kflag_member()
2535 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_ref_type_check_meta()
2540 value = btf_name_by_offset(env->btf, t->name_off); in btf_ref_type_check_meta()
2563 struct btf *btf = env->btf; in btf_modifier_resolve() local
2565 next_type = btf_type_by_id(btf, next_type_id); in btf_modifier_resolve()
2581 if (!btf_type_id_size(btf, &next_type_id, NULL)) { in btf_modifier_resolve()
2583 next_type = btf_type_id_resolve(btf, &next_type_id); in btf_modifier_resolve()
2605 struct btf *btf = env->btf; in btf_var_resolve() local
2607 next_type = btf_type_by_id(btf, next_type_id); in btf_var_resolve()
2622 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); in btf_var_resolve()
2635 if (!btf_type_id_size(btf, &next_type_id, NULL)) { in btf_var_resolve()
2651 struct btf *btf = env->btf; in btf_ptr_resolve() local
2653 next_type = btf_type_by_id(btf, next_type_id); in btf_ptr_resolve()
2676 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); in btf_ptr_resolve()
2685 if (!btf_type_id_size(btf, &next_type_id, NULL)) { in btf_ptr_resolve()
2687 next_type = btf_type_id_resolve(btf, &next_type_id); in btf_ptr_resolve()
2702 static void btf_modifier_show(const struct btf *btf, in btf_modifier_show() argument
2707 if (btf->resolved_ids) in btf_modifier_show()
2708 t = btf_type_id_resolve(btf, &type_id); in btf_modifier_show()
2710 t = btf_type_skip_modifiers(btf, type_id, NULL); in btf_modifier_show()
2712 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_modifier_show()
2715 static void btf_var_show(const struct btf *btf, const struct btf_type *t, in btf_var_show() argument
2719 t = btf_type_id_resolve(btf, &type_id); in btf_var_show()
2721 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_var_show()
2724 static void btf_ptr_show(const struct btf *btf, const struct btf_type *t, in btf_ptr_show() argument
2782 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_fwd_check_meta()
2815 struct btf *btf = env->btf; in btf_array_check_member() local
2824 btf_type_id_size(btf, &array_type_id, &array_size); in btf_array_check_member()
2895 struct btf *btf = env->btf; in btf_array_resolve() local
2900 index_type = btf_type_by_id(btf, index_type_id); in btf_array_resolve()
2911 index_type = btf_type_id_size(btf, &index_type_id, NULL); in btf_array_resolve()
2920 elem_type = btf_type_by_id(btf, elem_type_id); in btf_array_resolve()
2932 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); in btf_array_resolve()
2963 static void __btf_array_show(const struct btf *btf, const struct btf_type *t, in __btf_array_show() argument
2974 elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL); in __btf_array_show()
3003 elem_ops->show(btf, elem_type, elem_type_id, data, in __btf_array_show()
3016 static void btf_array_show(const struct btf *btf, const struct btf_type *t, in btf_array_show() argument
3032 __btf_array_show(btf, t, type_id, data, bits_offset, show); in btf_array_show()
3046 __btf_array_show(btf, t, type_id, data, bits_offset, show); in btf_array_show()
3090 struct btf *btf = env->btf; in btf_struct_check_meta() local
3105 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_struct_check_meta()
3114 if (!btf_name_offset_valid(btf, member->name_off)) { in btf_struct_check_meta()
3123 !btf_name_valid_identifier(btf, member->name_off)) { in btf_struct_check_meta()
3186 last_member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3202 const struct btf_type *member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3260 static int btf_find_struct(const struct btf *btf, const struct btf_type *t, in btf_find_struct() argument
3273 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, in btf_find_kptr() argument
3281 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3285 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3290 if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) in btf_find_kptr()
3292 if (!strcmp("kptr_untrusted", __btf_name_by_offset(btf, t->name_off))) in btf_find_kptr()
3294 else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off))) in btf_find_kptr()
3300 t = btf_type_skip_modifiers(btf, t->type, &res_id); in btf_find_kptr()
3311 static const char *btf_find_decl_tag_value(const struct btf *btf, in btf_find_decl_tag_value() argument
3317 for (i = 1; i < btf_nr_types(btf); i++) { in btf_find_decl_tag_value()
3318 const struct btf_type *t = btf_type_by_id(btf, i); in btf_find_decl_tag_value()
3323 if (pt != btf_type_by_id(btf, t->type) || in btf_find_decl_tag_value()
3326 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) in btf_find_decl_tag_value()
3328 return __btf_name_by_offset(btf, t->name_off) + len; in btf_find_decl_tag_value()
3334 btf_find_graph_root(const struct btf *btf, const struct btf_type *pt, in btf_find_graph_root() argument
3347 value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:"); in btf_find_graph_root()
3356 id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT); in btf_find_graph_root()
3419 static int btf_find_struct_field(const struct btf *btf, in btf_find_struct_field() argument
3429 const struct btf_type *member_type = btf_type_by_id(btf, in btf_find_struct_field()
3432 field_type = btf_get_field_type(__btf_name_by_offset(btf, member_type->name_off), in btf_find_struct_field()
3453 ret = btf_find_struct(btf, member_type, off, sz, field_type, in btf_find_struct_field()
3460 ret = btf_find_kptr(btf, member_type, off, sz, in btf_find_struct_field()
3467 ret = btf_find_graph_root(btf, t, member_type, in btf_find_struct_field()
3487 static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, in btf_find_datasec_var() argument
3497 const struct btf_type *var = btf_type_by_id(btf, vsi->type); in btf_find_datasec_var()
3498 const struct btf_type *var_type = btf_type_by_id(btf, var->type); in btf_find_datasec_var()
3500 field_type = btf_get_field_type(__btf_name_by_offset(btf, var_type->name_off), in btf_find_datasec_var()
3519 ret = btf_find_struct(btf, var_type, off, sz, field_type, in btf_find_datasec_var()
3526 ret = btf_find_kptr(btf, var_type, off, sz, in btf_find_datasec_var()
3533 ret = btf_find_graph_root(btf, var, var_type, in btf_find_datasec_var()
3553 static int btf_find_field(const struct btf *btf, const struct btf_type *t, in btf_find_field() argument
3558 return btf_find_struct_field(btf, t, field_mask, info, info_cnt); in btf_find_field()
3560 return btf_find_datasec_var(btf, t, field_mask, info, info_cnt); in btf_find_field()
3564 static int btf_parse_kptr(const struct btf *btf, struct btf_field *field, in btf_parse_kptr() argument
3572 struct btf *kptr_btf; in btf_parse_kptr()
3579 t = btf_type_by_id(btf, info->kptr.type_id); in btf_parse_kptr()
3580 id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info), in btf_parse_kptr()
3584 WARN_ON_ONCE(btf_is_kernel(btf)); in btf_parse_kptr()
3591 kptr_btf = (struct btf *)btf; in btf_parse_kptr()
3645 field->kptr.btf = kptr_btf; in btf_parse_kptr()
3655 static int btf_parse_graph_root(const struct btf *btf, in btf_parse_graph_root() argument
3666 t = btf_type_by_id(btf, info->graph_root.value_btf_id); in btf_parse_graph_root()
3673 __btf_name_by_offset(btf, member->name_off))) in btf_parse_graph_root()
3678 n = btf_type_by_id(btf, member->type); in btf_parse_graph_root()
3681 if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off))) in btf_parse_graph_root()
3690 field->graph_root.btf = (struct btf *)btf; in btf_parse_graph_root()
3699 static int btf_parse_list_head(const struct btf *btf, struct btf_field *field, in btf_parse_list_head() argument
3702 return btf_parse_graph_root(btf, field, info, "bpf_list_node", in btf_parse_list_head()
3706 static int btf_parse_rb_root(const struct btf *btf, struct btf_field *field, in btf_parse_rb_root() argument
3709 return btf_parse_graph_root(btf, field, info, "bpf_rb_node", in btf_parse_rb_root()
3725 struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t, in btf_parse_fields() argument
3733 ret = btf_find_field(btf, t, field_mask, info_arr, ARRAY_SIZE(info_arr)); in btf_parse_fields()
3786 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
3791 ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
3796 ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
3836 int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) in btf_check_and_fixup_fields() argument
3857 meta = btf_find_struct_meta(btf, btf_id); in btf_check_and_fixup_fields()
3908 static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, in __btf_struct_show() argument
3921 const struct btf_type *member_type = btf_type_by_id(btf, in __btf_struct_show()
3945 ops->show(btf, member_type, member->type, in __btf_struct_show()
3955 static void btf_struct_show(const struct btf *btf, const struct btf_type *t, in btf_struct_show() argument
3971 __btf_struct_show(btf, t, type_id, data, bits_offset, show); in btf_struct_show()
3986 __btf_struct_show(btf, t, type_id, data, bits_offset, show); in btf_struct_show()
4063 struct btf *btf = env->btf; in btf_enum_check_meta() local
4085 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum_check_meta()
4093 if (!btf_name_offset_valid(btf, enums[i].name_off)) { in btf_enum_check_meta()
4101 !btf_name_valid_identifier(btf, enums[i].name_off)) { in btf_enum_check_meta()
4110 __btf_name_by_offset(btf, enums[i].name_off), in btf_enum_check_meta()
4123 static void btf_enum_show(const struct btf *btf, const struct btf_type *t, in btf_enum_show() argument
4143 __btf_name_by_offset(btf, in btf_enum_show()
4171 struct btf *btf = env->btf; in btf_enum64_check_meta() local
4193 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum64_check_meta()
4201 if (!btf_name_offset_valid(btf, enums[i].name_off)) { in btf_enum64_check_meta()
4209 !btf_name_valid_identifier(btf, enums[i].name_off)) { in btf_enum64_check_meta()
4219 __btf_name_by_offset(btf, enums[i].name_off), in btf_enum64_check_meta()
4226 static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, in btf_enum64_show() argument
4246 __btf_name_by_offset(btf, in btf_enum64_show()
4316 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4320 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4328 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4361 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_func_check_meta()
4430 !__btf_name_valid(env->btf, t->name_off)) { in btf_var_check_meta()
4496 !btf_name_valid_section(env->btf, t->name_off)) { in btf_datasec_check_meta()
4546 struct btf *btf = env->btf; in btf_datasec_resolve() local
4552 const struct btf_type *var_type = btf_type_by_id(env->btf, in btf_datasec_resolve()
4567 if (!btf_type_id_size(btf, &type_id, &type_size)) { in btf_datasec_resolve()
4588 static void btf_datasec_show(const struct btf *btf, in btf_datasec_show() argument
4601 __btf_name_by_offset(btf, t->name_off)); in btf_datasec_show()
4603 var = btf_type_by_id(btf, vsi->type); in btf_datasec_show()
4606 btf_type_ops(var)->show(btf, var, vsi->type, in btf_datasec_show()
4712 value = btf_name_by_offset(env->btf, t->name_off); in btf_decl_tag_check_meta()
4745 struct btf *btf = env->btf; in btf_decl_tag_resolve() local
4749 next_type = btf_type_by_id(btf, next_type_id); in btf_decl_tag_resolve()
4770 next_type = btf_type_by_id(btf, next_type->type); in btf_decl_tag_resolve()
4805 const struct btf *btf; in btf_func_proto_check() local
4809 btf = env->btf; in btf_func_proto_check()
4817 ret_type = btf_type_by_id(btf, ret_type_id); in btf_func_proto_check()
4836 if (!btf_type_id_size(btf, &ret_type_id, NULL)) { in btf_func_proto_check()
4860 arg_type = btf_type_by_id(btf, arg_type_id); in btf_func_proto_check()
4872 (!btf_name_offset_valid(btf, args[i].name_off) || in btf_func_proto_check()
4873 !btf_name_valid_identifier(btf, args[i].name_off))) { in btf_func_proto_check()
4886 if (!btf_type_id_size(btf, &arg_type_id, NULL)) { in btf_func_proto_check()
4900 const struct btf *btf; in btf_func_check() local
4903 btf = env->btf; in btf_func_check()
4904 proto_type = btf_type_by_id(btf, t->type); in btf_func_check()
4972 if (!btf_name_offset_valid(env->btf, t->name_off)) { in btf_check_meta()
4989 struct btf *btf = env->btf; in btf_check_all_metas() local
4993 hdr = &btf->hdr; in btf_check_all_metas()
4994 cur = btf->nohdr_data + hdr->type_off; in btf_check_all_metas()
4997 env->log_type_id = btf->base_btf ? btf->start_id : 1; in btf_check_all_metas()
5018 struct btf *btf = env->btf; in btf_resolve_valid() local
5024 return !btf_resolved_type_id(btf, type_id) && in btf_resolve_valid()
5025 !btf_resolved_type_size(btf, type_id); in btf_resolve_valid()
5028 return btf_resolved_type_id(btf, type_id) && in btf_resolve_valid()
5029 !btf_resolved_type_size(btf, type_id); in btf_resolve_valid()
5033 t = btf_type_id_resolve(btf, &type_id); in btf_resolve_valid()
5046 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); in btf_resolve_valid()
5049 btf_resolved_type_size(btf, type_id)); in btf_resolve_valid()
5090 struct btf *btf = env->btf; in btf_check_all_types() local
5100 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) { in btf_check_all_types()
5101 type_id = btf->start_id + i; in btf_check_all_types()
5102 t = btf_type_by_id(btf, type_id); in btf_check_all_types()
5124 const struct btf_header *hdr = &env->btf->hdr; in btf_parse_type_sec()
5133 if (!env->btf->base_btf && !hdr->type_len) { in btf_parse_type_sec()
5148 struct btf *btf = env->btf; in btf_parse_str_sec() local
5151 hdr = &btf->hdr; in btf_parse_str_sec()
5152 start = btf->nohdr_data + hdr->str_off; in btf_parse_str_sec()
5155 if (end != btf->data + btf->data_size) { in btf_parse_str_sec()
5160 btf->strings = start; in btf_parse_str_sec()
5162 if (btf->base_btf && !hdr->str_len) in btf_parse_str_sec()
5168 if (!btf->base_btf && start[0]) { in btf_parse_str_sec()
5195 const struct btf *btf; in btf_check_sec_info() local
5197 btf = env->btf; in btf_check_sec_info()
5198 hdr = &btf->hdr; in btf_check_sec_info()
5246 struct btf *btf; in btf_parse_hdr() local
5248 btf = env->btf; in btf_parse_hdr()
5249 btf_data_size = btf->data_size; in btf_parse_hdr()
5256 hdr = btf->data; in btf_parse_hdr()
5264 if (hdr_len > sizeof(btf->hdr)) { in btf_parse_hdr()
5265 u8 *expected_zero = btf->data + sizeof(btf->hdr); in btf_parse_hdr()
5266 u8 *end = btf->data + hdr_len; in btf_parse_hdr()
5276 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); in btf_parse_hdr()
5277 memcpy(&btf->hdr, btf->data, hdr_copy); in btf_parse_hdr()
5279 hdr = &btf->hdr; in btf_parse_hdr()
5298 if (!btf->base_btf && btf_data_size == hdr->hdr_len) { in btf_parse_hdr()
5316 btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) in btf_parse_struct_metas() argument
5337 id = btf_find_by_name_kind(btf, alloc_obj_fields[i], BTF_KIND_STRUCT); in btf_parse_struct_metas()
5347 n = btf_nr_types(btf); in btf_parse_struct_metas()
5356 t = btf_type_by_id(btf, i); in btf_parse_struct_metas()
5385 record = btf_parse_fields(btf, t, BPF_SPIN_LOCK | BPF_LIST_HEAD | BPF_LIST_NODE | in btf_parse_struct_metas()
5401 struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id) in btf_find_struct_meta() argument
5406 tab = btf->struct_meta_tab; in btf_find_struct_meta()
5413 struct btf *btf, int start_id) in btf_check_type_tags() argument
5418 n = btf_nr_types(btf); in btf_check_type_tags()
5424 t = btf_type_by_id(btf, i); in btf_check_type_tags()
5450 t = btf_type_by_id(btf, cur_id); in btf_check_type_tags()
5474 static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) in btf_parse()
5476 bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); in btf_parse()
5480 struct btf *btf = NULL; in btf_parse() local
5499 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); in btf_parse()
5500 if (!btf) { in btf_parse()
5504 env->btf = btf; in btf_parse()
5512 btf->data = data; in btf_parse()
5513 btf->data_size = attr->btf_size; in btf_parse()
5524 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse()
5534 err = btf_check_type_tags(env, btf, 1); in btf_parse()
5538 struct_meta_tab = btf_parse_struct_metas(&env->log, btf); in btf_parse()
5543 btf->struct_meta_tab = struct_meta_tab; in btf_parse()
5549 err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record); in btf_parse()
5560 refcount_set(&btf->refcnt, 1); in btf_parse()
5561 return btf; in btf_parse()
5564 btf_free_struct_meta_tab(btf); in btf_parse()
5572 if (btf) in btf_parse()
5573 btf_free(btf); in btf_parse()
5579 extern struct btf *btf_vmlinux;
5612 btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, in btf_get_prog_ctx_type() argument
5626 t = btf_type_by_id(btf, t->type); in btf_get_prog_ctx_type()
5628 t = btf_type_by_id(btf, t->type); in btf_get_prog_ctx_type()
5637 tname = btf_name_by_offset(btf, t->name_off); in btf_get_prog_ctx_type()
5683 struct btf *btf, in btf_translate_to_vmlinux() argument
5690 prog_ctx_type = btf_get_prog_ctx_type(log, btf, t, prog_type, arg); in btf_translate_to_vmlinux()
5720 struct btf *btf_parse_vmlinux(void) in BTF_ID()
5724 struct btf *btf = NULL; in BTF_ID() local
5734 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); in BTF_ID()
5735 if (!btf) { in BTF_ID()
5739 env->btf = btf; in BTF_ID()
5741 btf->data = __start_BTF; in BTF_ID()
5742 btf->data_size = __stop_BTF - __start_BTF; in BTF_ID()
5743 btf->kernel_btf = true; in BTF_ID()
5744 snprintf(btf->name, sizeof(btf->name), "vmlinux"); in BTF_ID()
5750 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in BTF_ID()
5760 err = btf_check_type_tags(env, btf, 1); in BTF_ID()
5765 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); in BTF_ID()
5767 bpf_struct_ops_init(btf, log); in BTF_ID()
5769 refcount_set(&btf->refcnt, 1); in BTF_ID()
5771 err = btf_alloc_id(btf); in BTF_ID()
5776 return btf; in BTF_ID()
5780 if (btf) { in BTF_ID()
5781 kvfree(btf->types); in BTF_ID()
5782 kfree(btf); in BTF_ID()
5789 static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_si… in btf_parse_module()
5793 struct btf *btf = NULL, *base_btf; in btf_parse_module() local
5809 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); in btf_parse_module()
5810 if (!btf) { in btf_parse_module()
5814 env->btf = btf; in btf_parse_module()
5816 btf->base_btf = base_btf; in btf_parse_module()
5817 btf->start_id = base_btf->nr_types; in btf_parse_module()
5818 btf->start_str_off = base_btf->hdr.str_len; in btf_parse_module()
5819 btf->kernel_btf = true; in btf_parse_module()
5820 snprintf(btf->name, sizeof(btf->name), "%s", module_name); in btf_parse_module()
5822 btf->data = kvmalloc(data_size, GFP_KERNEL | __GFP_NOWARN); in btf_parse_module()
5823 if (!btf->data) { in btf_parse_module()
5827 memcpy(btf->data, data, data_size); in btf_parse_module()
5828 btf->data_size = data_size; in btf_parse_module()
5834 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse_module()
5844 err = btf_check_type_tags(env, btf, btf_nr_types(base_btf)); in btf_parse_module()
5849 refcount_set(&btf->refcnt, 1); in btf_parse_module()
5850 return btf; in btf_parse_module()
5854 if (btf) { in btf_parse_module()
5855 kvfree(btf->data); in btf_parse_module()
5856 kvfree(btf->types); in btf_parse_module()
5857 kfree(btf); in btf_parse_module()
5864 struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) in bpf_prog_get_target_btf()
5869 return tgt_prog->aux->btf; in bpf_prog_get_target_btf()
5874 static bool is_int_ptr(struct btf *btf, const struct btf_type *t) in is_int_ptr() argument
5877 t = btf_type_skip_modifiers(btf, t->type, NULL); in is_int_ptr()
5882 static u32 get_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto, in get_ctx_arg_idx() argument
5896 t = btf_type_skip_modifiers(btf, args[i].type, NULL); in get_ctx_arg_idx()
5902 t = btf_type_skip_modifiers(btf, func_proto->type, NULL); in get_ctx_arg_idx()
5932 struct btf *btf = bpf_prog_get_target_btf(prog); in btf_ctx_access() local
5945 arg = get_ctx_arg_idx(btf, t, off); in btf_ctx_access()
5983 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
5992 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_ctx_access()
6009 t = btf_type_by_id(btf, args[arg].type); in btf_ctx_access()
6014 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6022 __btf_name_by_offset(btf, t->name_off), in btf_ctx_access()
6048 if (is_int_ptr(btf, t)) in btf_ctx_access()
6062 info->btf = btf_vmlinux; in btf_ctx_access()
6080 ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg); in btf_ctx_access()
6082 info->btf = btf_vmlinux; in btf_ctx_access()
6090 info->btf = btf; in btf_ctx_access()
6092 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6095 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6105 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6115 __btf_name_by_offset(btf, t->name_off)); in btf_ctx_access()
6126 static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, in btf_struct_walk() argument
6139 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_struct_walk()
6140 tname = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
6165 mtype = btf_type_skip_modifiers(btf, member->type, in btf_struct_walk()
6179 t = btf_type_skip_modifiers(btf, array_elem->type, in btf_struct_walk()
6241 mtype = btf_type_by_id(btf, member->type); in btf_struct_walk()
6242 mname = __btf_name_by_offset(btf, member->name_off); in btf_struct_walk()
6244 mtype = __btf_resolve_size(btf, mtype, &msize, in btf_struct_walk()
6342 t = btf_type_by_id(btf, mtype->type); in btf_struct_walk()
6344 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
6356 stype = btf_type_skip_modifiers(btf, mtype->type, &id); in btf_struct_walk()
6391 const struct btf *btf = reg->btf; in btf_struct_access() local
6402 meta = btf_find_struct_meta(btf, id); in btf_struct_access()
6419 t = btf_type_by_id(btf, id); in btf_struct_access()
6421 err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); in btf_struct_access()
6443 t = btf_type_by_id(btf, id); in btf_struct_access()
6464 bool btf_types_are_same(const struct btf *btf1, u32 id1, in btf_types_are_same()
6465 const struct btf *btf2, u32 id2) in btf_types_are_same()
6475 const struct btf *btf, u32 id, int off, in btf_struct_ids_match() argument
6476 const struct btf *need_btf, u32 need_type_id, in btf_struct_ids_match()
6484 if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id)) in btf_struct_ids_match()
6493 type = btf_type_by_id(btf, id); in btf_struct_ids_match()
6496 err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); in btf_struct_ids_match()
6505 if (!btf_types_are_same(btf, id, need_btf, need_type_id)) { in btf_struct_ids_match()
6513 static int __get_type_size(struct btf *btf, u32 btf_id, in __get_type_size() argument
6518 *ret_type = btf_type_by_id(btf, 0); in __get_type_size()
6522 t = btf_type_by_id(btf, btf_id); in __get_type_size()
6524 t = btf_type_by_id(btf, t->type); in __get_type_size()
6549 struct btf *btf, in btf_distill_func_proto() argument
6580 ret = __get_type_size(btf, func->type, &t); in btf_distill_func_proto()
6597 ret = __get_type_size(btf, args[i].type, &t); in btf_distill_func_proto()
6628 struct btf *btf1, const struct btf_type *t1, in btf_check_func_type_match()
6629 struct btf *btf2, const struct btf_type *t2) in btf_check_func_type_match()
6739 struct btf *btf2, const struct btf_type *t2) in btf_check_type_match()
6741 struct btf *btf1 = prog->aux->btf; in btf_check_type_match()
6762 const struct btf *btf, u32 func_id, in btf_check_func_arg_match() argument
6775 t = btf_type_by_id(btf, func_id); in btf_check_func_arg_match()
6784 func_name = btf_name_by_offset(btf, t->name_off); in btf_check_func_arg_match()
6786 t = btf_type_by_id(btf, t->type); in btf_check_func_arg_match()
6807 t = btf_type_skip_modifiers(btf, args[i].type, NULL); in btf_check_func_arg_match()
6821 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); in btf_check_func_arg_match()
6822 ref_tname = btf_name_by_offset(btf, ref_t->name_off); in btf_check_func_arg_match()
6828 if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { in btf_check_func_arg_match()
6842 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); in btf_check_func_arg_match()
6874 struct btf *btf = prog->aux->btf; in btf_check_subprog_arg_match() local
6890 err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, false); in btf_check_subprog_arg_match()
6917 struct btf *btf = prog->aux->btf; in btf_check_subprog_call() local
6933 err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, true); in btf_check_subprog_call()
6957 struct btf *btf = prog->aux->btf; in btf_prepare_func_args() local
6975 t = btf_type_by_id(btf, btf_id); in btf_prepare_func_args()
6984 tname = btf_name_by_offset(btf, t->name_off); in btf_prepare_func_args()
6997 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7010 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7012 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7025 t = btf_type_by_id(btf, args[i].type); in btf_prepare_func_args()
7027 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7033 if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { in btf_prepare_func_args()
7038 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_prepare_func_args()
7040 ref_t = btf_resolve_size(btf, t, ®->mem_size); in btf_prepare_func_args()
7044 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), in btf_prepare_func_args()
7061 static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, in btf_type_show() argument
7064 const struct btf_type *t = btf_type_by_id(btf, type_id); in btf_type_show()
7066 show->btf = btf; in btf_type_show()
7070 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); in btf_type_show()
7079 int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, in btf_type_seq_show_flags() argument
7088 btf_type_show(btf, type_id, obj, &sseq); in btf_type_seq_show_flags()
7093 void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, in btf_type_seq_show() argument
7096 (void) btf_type_seq_show_flags(btf, type_id, obj, m, in btf_type_seq_show()
7129 int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, in btf_type_snprintf_show() argument
7140 btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf); in btf_type_snprintf_show()
7153 const struct btf *btf = filp->private_data; in bpf_btf_show_fdinfo() local
7155 seq_printf(m, "btf_id:\t%u\n", btf->id); in bpf_btf_show_fdinfo()
7172 static int __btf_new_fd(struct btf *btf) in __btf_new_fd() argument
7174 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); in __btf_new_fd()
7179 struct btf *btf; in btf_new_fd() local
7182 btf = btf_parse(attr, uattr, uattr_size); in btf_new_fd()
7183 if (IS_ERR(btf)) in btf_new_fd()
7184 return PTR_ERR(btf); in btf_new_fd()
7186 ret = btf_alloc_id(btf); in btf_new_fd()
7188 btf_free(btf); in btf_new_fd()
7198 ret = __btf_new_fd(btf); in btf_new_fd()
7200 btf_put(btf); in btf_new_fd()
7205 struct btf *btf_get_by_fd(int fd) in btf_get_by_fd()
7207 struct btf *btf; in btf_get_by_fd() local
7220 btf = f.file->private_data; in btf_get_by_fd()
7221 refcount_inc(&btf->refcnt); in btf_get_by_fd()
7224 return btf; in btf_get_by_fd()
7227 int btf_get_info_by_fd(const struct btf *btf, in btf_get_info_by_fd() argument
7247 info.id = btf->id; in btf_get_info_by_fd()
7248 ubtf = u64_to_user_ptr(info.btf); in btf_get_info_by_fd()
7249 btf_copy = min_t(u32, btf->data_size, info.btf_size); in btf_get_info_by_fd()
7250 if (copy_to_user(ubtf, btf->data, btf_copy)) in btf_get_info_by_fd()
7252 info.btf_size = btf->data_size; in btf_get_info_by_fd()
7254 info.kernel_btf = btf->kernel_btf; in btf_get_info_by_fd()
7261 name_len = strlen(btf->name); in btf_get_info_by_fd()
7266 if (copy_to_user(uname, btf->name, name_len + 1)) in btf_get_info_by_fd()
7271 if (copy_to_user(uname, btf->name, uname_len - 1)) in btf_get_info_by_fd()
7289 struct btf *btf; in btf_get_fd_by_id() local
7293 btf = idr_find(&btf_idr, id); in btf_get_fd_by_id()
7294 if (!btf || !refcount_inc_not_zero(&btf->refcnt)) in btf_get_fd_by_id()
7295 btf = ERR_PTR(-ENOENT); in btf_get_fd_by_id()
7298 if (IS_ERR(btf)) in btf_get_fd_by_id()
7299 return PTR_ERR(btf); in btf_get_fd_by_id()
7301 fd = __btf_new_fd(btf); in btf_get_fd_by_id()
7303 btf_put(btf); in btf_get_fd_by_id()
7308 u32 btf_obj_id(const struct btf *btf) in btf_obj_id() argument
7310 return btf->id; in btf_obj_id()
7313 bool btf_is_kernel(const struct btf *btf) in btf_is_kernel() argument
7315 return btf->kernel_btf; in btf_is_kernel()
7318 bool btf_is_module(const struct btf *btf) in btf_is_module() argument
7320 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; in btf_is_module()
7331 struct btf *btf; member
7344 const struct btf *btf = bin_attr->private; in btf_module_read() local
7346 memcpy(buf, btf->data + off, len); in btf_module_read()
7350 static void purge_cand_cache(struct btf *btf);
7357 struct btf *btf; in btf_module_notify() local
7372 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size); in btf_module_notify()
7373 if (IS_ERR(btf)) { in btf_module_notify()
7377 mod->name, PTR_ERR(btf)); in btf_module_notify()
7378 err = PTR_ERR(btf); in btf_module_notify()
7384 err = btf_alloc_id(btf); in btf_module_notify()
7386 btf_free(btf); in btf_module_notify()
7394 btf_mod->btf = btf; in btf_module_notify()
7406 attr->attr.name = btf->name; in btf_module_notify()
7408 attr->size = btf->data_size; in btf_module_notify()
7409 attr->private = btf; in btf_module_notify()
7445 purge_cand_cache(btf_mod->btf); in btf_module_notify()
7446 btf_put(btf_mod->btf); in btf_module_notify()
7471 struct module *btf_try_get_module(const struct btf *btf) in btf_try_get_module() argument
7479 if (btf_mod->btf != btf) in btf_try_get_module()
7501 static struct btf *btf_get_module_btf(const struct module *module) in btf_get_module_btf()
7506 struct btf *btf = NULL; in btf_get_module_btf() local
7509 btf = bpf_get_btf_vmlinux(); in btf_get_module_btf()
7510 if (!IS_ERR_OR_NULL(btf)) in btf_get_module_btf()
7511 btf_get(btf); in btf_get_module_btf()
7512 return btf; in btf_get_module_btf()
7521 btf_get(btf_mod->btf); in btf_get_module_btf()
7522 btf = btf_mod->btf; in btf_get_module_btf()
7528 return btf; in btf_get_module_btf()
7533 struct btf *btf = NULL; in BPF_CALL_4() local
7543 ret = bpf_find_btf_id(name, kind, &btf); in BPF_CALL_4()
7544 if (ret > 0 && btf_is_module(btf)) { in BPF_CALL_4()
7545 btf_obj_fd = __btf_new_fd(btf); in BPF_CALL_4()
7547 btf_put(btf); in BPF_CALL_4()
7553 btf_put(btf); in BPF_CALL_4()
7572 static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, in BTF_ID_LIST_GLOBAL()
7592 t = btf_type_skip_modifiers(btf, arg->type, NULL); in BTF_ID_LIST_GLOBAL()
7595 t = btf_type_skip_modifiers(btf, t->type, NULL); in BTF_ID_LIST_GLOBAL()
7599 name = btf_name_by_offset(btf, t->name_off); in BTF_ID_LIST_GLOBAL()
7630 t = btf_type_skip_modifiers(btf, func->type, NULL); in BTF_ID_LIST_GLOBAL()
7637 t = btf_type_by_id(btf, func->type); in BTF_ID_LIST_GLOBAL()
7645 static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) in btf_check_kfunc_protos() argument
7652 func = btf_type_by_id(btf, func_id); in btf_check_kfunc_protos()
7657 func_name = btf_name_by_offset(btf, func->name_off); in btf_check_kfunc_protos()
7661 func = btf_type_by_id(btf, func->type); in btf_check_kfunc_protos()
7666 err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); in btf_check_kfunc_protos()
7676 static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, in btf_populate_kfunc_set() argument
7681 bool vmlinux_set = !btf_is_module(btf); in btf_populate_kfunc_set()
7696 tab = btf->kfunc_set_tab; in btf_populate_kfunc_set()
7719 btf->kfunc_set_tab = tab; in btf_populate_kfunc_set()
7786 btf_free_kfunc_set_tab(btf); in btf_populate_kfunc_set()
7790 static u32 *__btf_kfunc_id_set_contains(const struct btf *btf, in __btf_kfunc_id_set_contains() argument
7801 if (!btf->kfunc_set_tab) in __btf_kfunc_id_set_contains()
7803 hook_filter = &btf->kfunc_set_tab->hook_filters[hook]; in __btf_kfunc_id_set_contains()
7808 set = btf->kfunc_set_tab->sets[hook]; in __btf_kfunc_id_set_contains()
7861 u32 *btf_kfunc_id_set_contains(const struct btf *btf, in btf_kfunc_id_set_contains() argument
7869 kfunc_flags = __btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id, prog); in btf_kfunc_id_set_contains()
7874 return __btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id, prog); in btf_kfunc_id_set_contains()
7877 u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id, in btf_kfunc_is_modify_return() argument
7880 return __btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id, prog); in btf_kfunc_is_modify_return()
7886 struct btf *btf; in __register_btf_kfunc_id_set() local
7889 btf = btf_get_module_btf(kset->owner); in __register_btf_kfunc_id_set()
7890 if (!btf) { in __register_btf_kfunc_id_set()
7899 if (IS_ERR(btf)) in __register_btf_kfunc_id_set()
7900 return PTR_ERR(btf); in __register_btf_kfunc_id_set()
7903 ret = btf_check_kfunc_protos(btf, kset->set->pairs[i].id, in __register_btf_kfunc_id_set()
7909 ret = btf_populate_kfunc_set(btf, hook, kset); in __register_btf_kfunc_id_set()
7912 btf_put(btf); in __register_btf_kfunc_id_set()
7934 s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id) in btf_find_dtor_kfunc() argument
7936 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_find_dtor_kfunc()
7951 static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt) in btf_check_dtor_kfuncs() argument
7961 dtor_func = btf_type_by_id(btf, dtor_btf_id); in btf_check_dtor_kfuncs()
7965 dtor_func_proto = btf_type_by_id(btf, dtor_func->type); in btf_check_dtor_kfuncs()
7970 t = btf_type_by_id(btf, dtor_func_proto->type); in btf_check_dtor_kfuncs()
7978 t = btf_type_by_id(btf, args[0].type); in btf_check_dtor_kfuncs()
7993 struct btf *btf; in register_btf_id_dtor_kfuncs() local
7997 btf = btf_get_module_btf(owner); in register_btf_id_dtor_kfuncs()
7998 if (!btf) { in register_btf_id_dtor_kfuncs()
8009 if (IS_ERR(btf)) in register_btf_id_dtor_kfuncs()
8010 return PTR_ERR(btf); in register_btf_id_dtor_kfuncs()
8019 ret = btf_check_dtor_kfuncs(btf, dtors, add_cnt); in register_btf_id_dtor_kfuncs()
8023 tab = btf->dtor_kfunc_tab; in register_btf_id_dtor_kfuncs()
8025 if (WARN_ON_ONCE(tab && btf_is_module(btf))) { in register_btf_id_dtor_kfuncs()
8041 tab = krealloc(btf->dtor_kfunc_tab, in register_btf_id_dtor_kfuncs()
8049 if (!btf->dtor_kfunc_tab) in register_btf_id_dtor_kfuncs()
8051 btf->dtor_kfunc_tab = tab; in register_btf_id_dtor_kfuncs()
8060 btf_free_dtor_kfunc_tab(btf); in register_btf_id_dtor_kfuncs()
8061 btf_put(btf); in register_btf_id_dtor_kfuncs()
8087 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, in bpf_core_types_are_compat()
8088 const struct btf *targ_btf, __u32 targ_id) in bpf_core_types_are_compat()
8096 int bpf_core_types_match(const struct btf *local_btf, u32 local_id, in bpf_core_types_match()
8097 const struct btf *targ_btf, u32 targ_id) in bpf_core_types_match()
8129 const struct btf *btf; member
8239 static void __purge_cand_cache(struct btf *btf, struct bpf_cand_cache **cache, in __purge_cand_cache() argument
8249 if (!btf) { in __purge_cand_cache()
8262 if (cc->cands[j].btf == btf) { in __purge_cand_cache()
8271 static void purge_cand_cache(struct btf *btf) in purge_cand_cache() argument
8274 __purge_cand_cache(btf, module_cand_cache, MODULE_CAND_CACHE_SIZE); in purge_cand_cache()
8280 bpf_core_add_cands(struct bpf_cand_cache *cands, const struct btf *targ_btf, in bpf_core_add_cands()
8321 cands->cands[cands->cnt].btf = targ_btf; in bpf_core_add_cands()
8332 const struct btf *local_btf = ctx->btf; in bpf_core_find_cands()
8334 const struct btf *main_btf; in bpf_core_find_cands()
8336 struct btf *mod_btf; in bpf_core_find_cands()
8455 cands.cands[i].btf = cc->cands[i].btf; in bpf_core_apply()
8466 err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs, in bpf_core_apply()
8489 struct btf *btf = reg->btf; in btf_nested_type_is_trusted() local
8497 walk_type = btf_type_by_id(btf, reg->btf_id); in btf_nested_type_is_trusted()
8501 tname = btf_name_by_offset(btf, walk_type->name_off); in btf_nested_type_is_trusted()
8507 safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info)); in btf_nested_type_is_trusted()
8511 safe_type = btf_type_by_id(btf, safe_id); in btf_nested_type_is_trusted()
8516 const char *m_name = __btf_name_by_offset(btf, member->name_off); in btf_nested_type_is_trusted()
8517 const struct btf_type *mtype = btf_type_by_id(btf, member->type); in btf_nested_type_is_trusted()
8523 btf_type_skip_modifiers(btf, mtype->type, &id); in btf_nested_type_is_trusted()
8533 const struct btf *reg_btf, u32 reg_id, in btf_type_ids_nocast_alias()
8534 const struct btf *arg_btf, u32 arg_id) in btf_type_ids_nocast_alias()