Lines Matching +full:input +full:- +full:justification
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
34 /* raw BTF data in non-native endianness */
46 * +--------------------------------+
48 * +--------------------------------+
52 * types_data-+ |
53 * strs_data------------+
64 * +----------+ +---------+ +-----------+
66 * +----------+ +---------+ +-----------+
70 * types_data----+ |
71 * strset__data(strs_set)-----+
73 * +----------+---------+-----------+
75 * raw_data----->+----------+---------+-----------+
80 size_t types_data_cap; /* used size stored in hdr->type_len */
83 * type_offs[0] corresponds to the first non-VOID type:
84 * - for base BTF it's type [1];
85 * - for split BTF it's the first non-base BTF type.
90 * - doesn't include special [0] void type;
91 * - for split BTF counts number of types added on top of base BTF.
99 * - for base BTF it's equal to 1;
100 * - for split BTF it's equal to biggest type ID of base BTF plus 1.
104 * - for base BTF it's equal to 0;
105 * - for split BTF it's equal to total size of base BTF's string section size.
109 /* only one of strs_data or strs_set can be non-NULL, depending on
168 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); in libbpf_add_mem()
185 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); in libbpf_ensure_mem()
187 return -ENOMEM; in libbpf_ensure_mem()
194 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), in btf_add_type_offs_mem()
195 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); in btf_add_type_offs_mem()
204 return -ENOMEM; in btf_add_type_idx_entry()
212 h->magic = bswap_16(h->magic); in btf_bswap_hdr()
213 h->hdr_len = bswap_32(h->hdr_len); in btf_bswap_hdr()
214 h->type_off = bswap_32(h->type_off); in btf_bswap_hdr()
215 h->type_len = bswap_32(h->type_len); in btf_bswap_hdr()
216 h->str_off = bswap_32(h->str_off); in btf_bswap_hdr()
217 h->str_len = bswap_32(h->str_len); in btf_bswap_hdr()
222 struct btf_header *hdr = btf->hdr; in btf_parse_hdr()
225 if (btf->raw_size < sizeof(struct btf_header)) { in btf_parse_hdr()
227 return -EINVAL; in btf_parse_hdr()
230 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_parse_hdr()
231 btf->swapped_endian = true; in btf_parse_hdr()
232 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { in btf_parse_hdr()
233 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", in btf_parse_hdr()
234 bswap_32(hdr->hdr_len)); in btf_parse_hdr()
235 return -ENOTSUP; in btf_parse_hdr()
238 } else if (hdr->magic != BTF_MAGIC) { in btf_parse_hdr()
239 pr_debug("Invalid BTF magic: %x\n", hdr->magic); in btf_parse_hdr()
240 return -EINVAL; in btf_parse_hdr()
243 if (btf->raw_size < hdr->hdr_len) { in btf_parse_hdr()
245 hdr->hdr_len, btf->raw_size); in btf_parse_hdr()
246 return -EINVAL; in btf_parse_hdr()
249 meta_left = btf->raw_size - hdr->hdr_len; in btf_parse_hdr()
250 if (meta_left < (long long)hdr->str_off + hdr->str_len) { in btf_parse_hdr()
251 pr_debug("Invalid BTF total size: %u\n", btf->raw_size); in btf_parse_hdr()
252 return -EINVAL; in btf_parse_hdr()
255 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { in btf_parse_hdr()
257 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); in btf_parse_hdr()
258 return -EINVAL; in btf_parse_hdr()
261 if (hdr->type_off % 4) { in btf_parse_hdr()
263 return -EINVAL; in btf_parse_hdr()
271 const struct btf_header *hdr = btf->hdr; in btf_parse_str_sec()
272 const char *start = btf->strs_data; in btf_parse_str_sec()
273 const char *end = start + btf->hdr->str_len; in btf_parse_str_sec()
275 if (btf->base_btf && hdr->str_len == 0) in btf_parse_str_sec()
277 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { in btf_parse_str_sec()
279 return -EINVAL; in btf_parse_str_sec()
281 if (!btf->base_btf && start[0]) { in btf_parse_str_sec()
283 return -EINVAL; in btf_parse_str_sec()
325 return -EINVAL; in btf_type_size()
331 t->name_off = bswap_32(t->name_off); in btf_bswap_type_base()
332 t->info = bswap_32(t->info); in btf_bswap_type_base()
333 t->type = bswap_32(t->type); in btf_bswap_type_base()
363 e->name_off = bswap_32(e->name_off); in btf_bswap_type_rest()
364 e->val = bswap_32(e->val); in btf_bswap_type_rest()
369 e64->name_off = bswap_32(e64->name_off); in btf_bswap_type_rest()
370 e64->val_lo32 = bswap_32(e64->val_lo32); in btf_bswap_type_rest()
371 e64->val_hi32 = bswap_32(e64->val_hi32); in btf_bswap_type_rest()
376 a->type = bswap_32(a->type); in btf_bswap_type_rest()
377 a->index_type = bswap_32(a->index_type); in btf_bswap_type_rest()
378 a->nelems = bswap_32(a->nelems); in btf_bswap_type_rest()
383 m->name_off = bswap_32(m->name_off); in btf_bswap_type_rest()
384 m->type = bswap_32(m->type); in btf_bswap_type_rest()
385 m->offset = bswap_32(m->offset); in btf_bswap_type_rest()
390 p->name_off = bswap_32(p->name_off); in btf_bswap_type_rest()
391 p->type = bswap_32(p->type); in btf_bswap_type_rest()
395 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); in btf_bswap_type_rest()
399 v->type = bswap_32(v->type); in btf_bswap_type_rest()
400 v->offset = bswap_32(v->offset); in btf_bswap_type_rest()
401 v->size = bswap_32(v->size); in btf_bswap_type_rest()
405 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); in btf_bswap_type_rest()
409 return -EINVAL; in btf_bswap_type_rest()
415 struct btf_header *hdr = btf->hdr; in btf_parse_type_sec()
416 void *next_type = btf->types_data; in btf_parse_type_sec()
417 void *end_type = next_type + hdr->type_len; in btf_parse_type_sec()
421 if (btf->swapped_endian) in btf_parse_type_sec()
428 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); in btf_parse_type_sec()
429 return -EINVAL; in btf_parse_type_sec()
432 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) in btf_parse_type_sec()
433 return -EINVAL; in btf_parse_type_sec()
435 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); in btf_parse_type_sec()
440 btf->nr_types++; in btf_parse_type_sec()
445 return -EINVAL; in btf_parse_type_sec()
453 return btf->start_id + btf->nr_types; in btf__type_cnt()
458 return btf->base_btf; in btf__base_btf()
461 /* internal helper returning non-const pointer to a type */
466 if (type_id < btf->start_id) in btf_type_by_id()
467 return btf_type_by_id(btf->base_btf, type_id); in btf_type_by_id()
468 return btf->types_data + btf->type_offs[type_id - btf->start_id]; in btf_type_by_id()
473 if (type_id >= btf->start_id + btf->nr_types) in btf__type_by_id()
497 if (btf->base_btf && btf->base_btf->ptr_sz > 0) in determine_ptr_size()
498 return btf->base_btf->ptr_sz; in determine_ptr_size()
506 if (t->size != 4 && t->size != 8) in determine_ptr_size()
509 name = btf__name_by_offset(btf, t->name_off); in determine_ptr_size()
515 return t->size; in determine_ptr_size()
519 return -1; in determine_ptr_size()
524 if (!btf->ptr_sz) in btf_ptr_sz()
525 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf_ptr_sz()
526 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; in btf_ptr_sz()
539 if (!btf->ptr_sz) in btf__pointer_size()
540 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf__pointer_size()
542 if (btf->ptr_sz < 0) in btf__pointer_size()
546 return btf->ptr_sz; in btf__pointer_size()
555 return libbpf_err(-EINVAL); in btf__set_pointer_size()
556 btf->ptr_sz = ptr_sz; in btf__set_pointer_size()
574 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; in btf__endianness()
576 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; in btf__endianness()
582 return libbpf_err(-EINVAL); in btf__set_endianness()
584 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); in btf__set_endianness()
585 if (!btf->swapped_endian) { in btf__set_endianness()
586 free(btf->raw_data_swapped); in btf__set_endianness()
587 btf->raw_data_swapped = NULL; in btf__set_endianness()
609 __s64 size = -1; in btf__resolve_size()
622 size = t->size; in btf__resolve_size()
634 type_id = t->type; in btf__resolve_size()
638 if (nelems && array->nelems > UINT32_MAX / nelems) in btf__resolve_size()
639 return libbpf_err(-E2BIG); in btf__resolve_size()
640 nelems *= array->nelems; in btf__resolve_size()
641 type_id = array->type; in btf__resolve_size()
644 return libbpf_err(-EINVAL); in btf__resolve_size()
652 return libbpf_err(-EINVAL); in btf__resolve_size()
654 return libbpf_err(-E2BIG); in btf__resolve_size()
669 return min(btf_ptr_sz(btf), (size_t)t->size); in btf__align_of()
677 return btf__align_of(btf, t->type); in btf__align_of()
679 return btf__align_of(btf, btf_array(t)->type); in btf__align_of()
687 align = btf__align_of(btf, m->type); in btf__align_of()
710 type_id = t->type; in btf__resolve_type()
716 return libbpf_err(-EINVAL); in btf__resolve_type()
730 const char *name = btf__name_by_offset(btf, t->name_off); in btf__find_by_name()
736 return libbpf_err(-ENOENT); in btf__find_by_name()
753 name = btf__name_by_offset(btf, t->name_off); in btf_find_by_name_kind()
758 return libbpf_err(-ENOENT); in btf_find_by_name_kind()
764 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); in btf__find_by_name_kind_own()
775 return (void *)btf->hdr != btf->raw_data; in btf_is_modifiable()
783 if (btf->fd >= 0) in btf__free()
784 close(btf->fd); in btf__free()
788 * in-memory representation for header, types, and strings in btf__free()
793 free(btf->hdr); in btf__free()
794 free(btf->types_data); in btf__free()
795 strset__free(btf->strs_set); in btf__free()
797 free(btf->raw_data); in btf__free()
798 free(btf->raw_data_swapped); in btf__free()
799 free(btf->type_offs); in btf__free()
809 return ERR_PTR(-ENOMEM); in btf_new_empty()
811 btf->nr_types = 0; in btf_new_empty()
812 btf->start_id = 1; in btf_new_empty()
813 btf->start_str_off = 0; in btf_new_empty()
814 btf->fd = -1; in btf_new_empty()
815 btf->ptr_sz = sizeof(void *); in btf_new_empty()
816 btf->swapped_endian = false; in btf_new_empty()
819 btf->base_btf = base_btf; in btf_new_empty()
820 btf->start_id = btf__type_cnt(base_btf); in btf_new_empty()
821 btf->start_str_off = base_btf->hdr->str_len; in btf_new_empty()
825 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); in btf_new_empty()
826 btf->raw_data = calloc(1, btf->raw_size); in btf_new_empty()
827 if (!btf->raw_data) { in btf_new_empty()
829 return ERR_PTR(-ENOMEM); in btf_new_empty()
832 btf->hdr = btf->raw_data; in btf_new_empty()
833 btf->hdr->hdr_len = sizeof(struct btf_header); in btf_new_empty()
834 btf->hdr->magic = BTF_MAGIC; in btf_new_empty()
835 btf->hdr->version = BTF_VERSION; in btf_new_empty()
837 btf->types_data = btf->raw_data + btf->hdr->hdr_len; in btf_new_empty()
838 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; in btf_new_empty()
839 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ in btf_new_empty()
861 return ERR_PTR(-ENOMEM); in btf_new()
863 btf->nr_types = 0; in btf_new()
864 btf->start_id = 1; in btf_new()
865 btf->start_str_off = 0; in btf_new()
866 btf->fd = -1; in btf_new()
869 btf->base_btf = base_btf; in btf_new()
870 btf->start_id = btf__type_cnt(base_btf); in btf_new()
871 btf->start_str_off = base_btf->hdr->str_len; in btf_new()
874 btf->raw_data = malloc(size); in btf_new()
875 if (!btf->raw_data) { in btf_new()
876 err = -ENOMEM; in btf_new()
879 memcpy(btf->raw_data, data, size); in btf_new()
880 btf->raw_size = size; in btf_new()
882 btf->hdr = btf->raw_data; in btf_new()
887 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; in btf_new()
888 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; in btf_new()
913 int err = 0, fd = -1, idx = 0; in btf_parse_elf()
922 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); in btf_parse_elf()
927 err = -errno; in btf_parse_elf()
932 err = -LIBBPF_ERRNO__FORMAT; in btf_parse_elf()
993 err = -ENOENT; in btf_parse_elf()
996 btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf); in btf_parse_elf()
1014 *btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); in btf_parse_elf()
1057 err = -errno; in btf_parse_raw()
1063 err = -EIO; in btf_parse_raw()
1068 err = -EPROTO; in btf_parse_raw()
1074 err = -errno; in btf_parse_raw()
1079 err = -errno; in btf_parse_raw()
1084 err = -errno; in btf_parse_raw()
1088 /* pre-alloc memory and read all of BTF data */ in btf_parse_raw()
1091 err = -ENOMEM; in btf_parse_raw()
1095 err = -EIO; in btf_parse_raw()
1131 if (err != -EPROTO) in btf_parse()
1156 if (btf->fd >= 0) in btf_load_into_kernel()
1157 return libbpf_err(-EEXIST); in btf_load_into_kernel()
1159 return libbpf_err(-EINVAL); in btf_load_into_kernel()
1164 err = -ENOMEM; in btf_load_into_kernel()
1167 btf->raw_size = raw_size; in btf_load_into_kernel()
1168 btf->raw_data = raw_data; in btf_load_into_kernel()
1173 * retry, using either auto-allocated or custom log_buf. This way in btf_load_into_kernel()
1174 * non-NULL custom log_buf provides a buffer just in case, but hopes in btf_load_into_kernel()
1186 err = -ENOMEM; in btf_load_into_kernel()
1198 btf->fd = bpf_btf_load(raw_data, raw_size, &opts); in btf_load_into_kernel()
1199 if (btf->fd < 0) { in btf_load_into_kernel()
1211 err = -errno; in btf_load_into_kernel()
1215 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); in btf_load_into_kernel()
1230 return btf->fd; in btf__fd()
1235 btf->fd = fd; in btf__set_fd()
1240 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); in btf_strs_data()
1245 struct btf_header *hdr = btf->hdr; in btf_get_raw_data()
1251 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; in btf_get_raw_data()
1253 *size = btf->raw_size; in btf_get_raw_data()
1257 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; in btf_get_raw_data()
1263 memcpy(p, hdr, hdr->hdr_len); in btf_get_raw_data()
1266 p += hdr->hdr_len; in btf_get_raw_data()
1268 memcpy(p, btf->types_data, hdr->type_len); in btf_get_raw_data()
1270 for (i = 0; i < btf->nr_types; i++) { in btf_get_raw_data()
1271 t = p + btf->type_offs[i]; in btf_get_raw_data()
1272 /* btf_bswap_type_rest() relies on native t->info, so in btf_get_raw_data()
1281 p += hdr->type_len; in btf_get_raw_data()
1283 memcpy(p, btf_strs_data(btf), hdr->str_len); in btf_get_raw_data()
1284 p += hdr->str_len; in btf_get_raw_data()
1299 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); in btf__raw_data()
1303 btf->raw_size = data_sz; in btf__raw_data()
1304 if (btf->swapped_endian) in btf__raw_data()
1305 btf->raw_data_swapped = data; in btf__raw_data()
1307 btf->raw_data = data; in btf__raw_data()
1317 if (offset < btf->start_str_off) in btf__str_by_offset()
1318 return btf__str_by_offset(btf->base_btf, offset); in btf__str_by_offset()
1319 else if (offset - btf->start_str_off < btf->hdr->str_len) in btf__str_by_offset()
1320 return btf_strs_data(btf) + (offset - btf->start_str_off); in btf__str_by_offset()
1340 * let's start with a sane default - 4KiB here - and resize it only if in btf_get_from_fd()
1346 return ERR_PTR(-ENOMEM); in btf_get_from_fd()
1359 btf = ERR_PTR(-ENOMEM); in btf_get_from_fd()
1373 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); in btf_get_from_fd()
1391 return libbpf_err_ptr(-errno); in btf__load_from_kernel_by_id_split()
1406 if (btf->raw_data) { in btf_invalidate_raw_data()
1407 free(btf->raw_data); in btf_invalidate_raw_data()
1408 btf->raw_data = NULL; in btf_invalidate_raw_data()
1410 if (btf->raw_data_swapped) { in btf_invalidate_raw_data()
1411 free(btf->raw_data_swapped); in btf_invalidate_raw_data()
1412 btf->raw_data_swapped = NULL; in btf_invalidate_raw_data()
1424 int err = -ENOMEM; in btf_ensure_modifiable()
1433 hdr = malloc(btf->hdr->hdr_len); in btf_ensure_modifiable()
1434 types = malloc(btf->hdr->type_len); in btf_ensure_modifiable()
1438 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); in btf_ensure_modifiable()
1439 memcpy(types, btf->types_data, btf->hdr->type_len); in btf_ensure_modifiable()
1442 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); in btf_ensure_modifiable()
1449 btf->hdr = hdr; in btf_ensure_modifiable()
1450 btf->types_data = types; in btf_ensure_modifiable()
1451 btf->types_data_cap = btf->hdr->type_len; in btf_ensure_modifiable()
1452 btf->strs_data = NULL; in btf_ensure_modifiable()
1453 btf->strs_set = set; in btf_ensure_modifiable()
1457 if (btf->hdr->str_len == 0) in btf_ensure_modifiable()
1458 btf->strs_deduped = true; in btf_ensure_modifiable()
1459 if (!btf->base_btf && btf->hdr->str_len == 1) in btf_ensure_modifiable()
1460 btf->strs_deduped = true; in btf_ensure_modifiable()
1476 * - >0 offset into string section, if string is found;
1477 * - -ENOENT, if string is not in the string section;
1478 * - <0, on any other error.
1484 if (btf->base_btf) { in btf__find_str()
1485 off = btf__find_str(btf->base_btf, s); in btf__find_str()
1486 if (off != -ENOENT) in btf__find_str()
1492 return libbpf_err(-ENOMEM); in btf__find_str()
1494 off = strset__find_str(btf->strs_set, s); in btf__find_str()
1498 return btf->start_str_off + off; in btf__find_str()
1503 * - > 0 offset into string section, on success;
1504 * - < 0, on error.
1510 if (btf->base_btf) { in btf__add_str()
1511 off = btf__find_str(btf->base_btf, s); in btf__add_str()
1512 if (off != -ENOENT) in btf__add_str()
1517 return libbpf_err(-ENOMEM); in btf__add_str()
1519 off = strset__add_str(btf->strs_set, s); in btf__add_str()
1523 btf->hdr->str_len = strset__data_size(btf->strs_set); in btf__add_str()
1525 return btf->start_str_off + off; in btf__add_str()
1530 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, in btf_add_type_mem()
1531 btf->hdr->type_len, UINT_MAX, add_sz); in btf_add_type_mem()
1536 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); in btf_type_inc_vlen()
1543 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf_commit_type()
1547 btf->hdr->type_len += data_sz; in btf_commit_type()
1548 btf->hdr->str_off += data_sz; in btf_commit_type()
1549 btf->nr_types++; in btf_commit_type()
1550 return btf->start_id + btf->nr_types - 1; in btf_commit_type()
1568 if (p->str_off_map && in btf_rewrite_str()
1569 hashmap__find(p->str_off_map, (void *)(long)*str_off, &mapped_off)) { in btf_rewrite_str()
1574 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); in btf_rewrite_str()
1581 if (p->str_off_map) { in btf_rewrite_str()
1582 err = hashmap__append(p->str_off_map, (void *)(long)*str_off, (void *)(long)off); in btf_rewrite_str()
1603 return libbpf_err(-ENOMEM); in btf__add_type()
1607 return libbpf_err(-ENOMEM); in btf__add_type()
1626 * btf->start_id + btf->nr_types - 1 is the type ID offset we should in btf_rewrite_type_ids()
1629 *type_id += btf->start_id + btf->nr_types - 1; in btf_rewrite_type_ids()
1644 if (src_btf->base_btf) in btf__add_btf()
1645 return libbpf_err(-ENOTSUP); in btf__add_btf()
1649 return libbpf_err(-ENOMEM); in btf__add_btf()
1654 old_strs_len = btf->hdr->str_len; in btf__add_btf()
1656 data_sz = src_btf->hdr->type_len; in btf__add_btf()
1657 cnt = btf__type_cnt(src_btf) - 1; in btf__add_btf()
1659 /* pre-allocate enough memory for new types */ in btf__add_btf()
1662 return libbpf_err(-ENOMEM); in btf__add_btf()
1664 /* pre-allocate enough memory for type offset index for new types */ in btf__add_btf()
1667 return libbpf_err(-ENOMEM); in btf__add_btf()
1672 return libbpf_err(-ENOMEM); in btf__add_btf()
1675 memcpy(t, src_btf->types_data, data_sz); in btf__add_btf()
1686 *off = t - btf->types_data; in btf__add_btf()
1706 * pre-allocated, but it would not be available for querying. But now in btf__add_btf()
1711 btf->hdr->type_len += data_sz; in btf__add_btf()
1712 btf->hdr->str_off += data_sz; in btf__add_btf()
1713 btf->nr_types += cnt; in btf__add_btf()
1718 return btf->start_id + btf->nr_types - cnt; in btf__add_btf()
1723 memset(btf->types_data + btf->hdr->type_len, 0, data_sz); in btf__add_btf()
1724 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); in btf__add_btf()
1728 btf->hdr->str_len = old_strs_len; in btf__add_btf()
1737 * - *name* - non-empty, non-NULL type name;
1738 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1739 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1741 * - >0, type ID of newly added BTF type;
1742 * - <0, on error.
1749 /* non-empty name */ in btf__add_int()
1751 return libbpf_err(-EINVAL); in btf__add_int()
1753 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) in btf__add_int()
1754 return libbpf_err(-EINVAL); in btf__add_int()
1756 return libbpf_err(-EINVAL); in btf__add_int()
1760 return libbpf_err(-ENOMEM); in btf__add_int()
1765 return libbpf_err(-ENOMEM); in btf__add_int()
1775 t->name_off = name_off; in btf__add_int()
1776 t->info = btf_type_info(BTF_KIND_INT, 0, 0); in btf__add_int()
1777 t->size = byte_sz; in btf__add_int()
1786 * - *name* - non-empty, non-NULL type name;
1787 * - *sz* - size of the type, in bytes;
1789 * - >0, type ID of newly added BTF type;
1790 * - <0, on error.
1797 /* non-empty name */ in btf__add_float()
1799 return libbpf_err(-EINVAL); in btf__add_float()
1804 return libbpf_err(-EINVAL); in btf__add_float()
1807 return libbpf_err(-ENOMEM); in btf__add_float()
1812 return libbpf_err(-ENOMEM); in btf__add_float()
1818 t->name_off = name_off; in btf__add_float()
1819 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); in btf__add_float()
1820 t->size = byte_sz; in btf__add_float()
1832 return -EINVAL; in validate_type_id()
1843 return libbpf_err(-EINVAL); in btf_add_ref_kind()
1846 return libbpf_err(-ENOMEM); in btf_add_ref_kind()
1851 return libbpf_err(-ENOMEM); in btf_add_ref_kind()
1859 t->name_off = name_off; in btf_add_ref_kind()
1860 t->info = btf_type_info(kind, 0, 0); in btf_add_ref_kind()
1861 t->type = ref_type_id; in btf_add_ref_kind()
1868 * - *ref_type_id* - referenced type ID, it might not exist yet;
1870 * - >0, type ID of newly added BTF type;
1871 * - <0, on error.
1880 * - *index_type_id* - type ID of the type describing array index;
1881 * - *elem_type_id* - type ID of the type describing array element;
1882 * - *nr_elems* - the size of the array;
1884 * - >0, type ID of newly added BTF type;
1885 * - <0, on error.
1894 return libbpf_err(-EINVAL); in btf__add_array()
1897 return libbpf_err(-ENOMEM); in btf__add_array()
1902 return libbpf_err(-ENOMEM); in btf__add_array()
1904 t->name_off = 0; in btf__add_array()
1905 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); in btf__add_array()
1906 t->size = 0; in btf__add_array()
1909 a->type = elem_type_id; in btf__add_array()
1910 a->index_type = index_type_id; in btf__add_array()
1911 a->nelems = nr_elems; in btf__add_array()
1923 return libbpf_err(-ENOMEM); in btf_add_composite()
1928 return libbpf_err(-ENOMEM); in btf_add_composite()
1939 t->name_off = name_off; in btf_add_composite()
1940 t->info = btf_type_info(kind, 0, 0); in btf_add_composite()
1941 t->size = bytes_sz; in btf_add_composite()
1948 * - *name* - name of the struct, can be NULL or empty for anonymous structs;
1949 * - *byte_sz* - size of the struct, in bytes;
1955 * - >0, type ID of newly added BTF type;
1956 * - <0, on error.
1965 * - *name* - name of the union, can be NULL or empty for anonymous union;
1966 * - *byte_sz* - size of the union, in bytes;
1973 * - >0, type ID of newly added BTF type;
1974 * - <0, on error.
1983 return btf_type_by_id(btf, btf__type_cnt(btf) - 1); in btf_last_type()
1988 * - *name* - name of the field, can be NULL or empty for anonymous field;
1989 * - *type_id* - type ID for the type describing field type;
1990 * - *bit_offset* - bit offset of the start of the field within struct/union;
1991 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
1993 * - 0, on success;
1994 * - <0, on error.
2005 if (btf->nr_types == 0) in btf__add_field()
2006 return libbpf_err(-EINVAL); in btf__add_field()
2009 return libbpf_err(-EINVAL); in btf__add_field()
2012 return libbpf_err(-EINVAL); in btf__add_field()
2013 /* best-effort bit field offset/size enforcement */ in btf__add_field()
2016 return libbpf_err(-EINVAL); in btf__add_field()
2020 return libbpf_err(-EINVAL); in btf__add_field()
2024 return libbpf_err(-ENOMEM); in btf__add_field()
2029 return libbpf_err(-ENOMEM); in btf__add_field()
2037 m->name_off = name_off; in btf__add_field()
2038 m->type = type_id; in btf__add_field()
2039 m->offset = bit_offset | (bit_size << 24); in btf__add_field()
2044 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); in btf__add_field()
2046 btf->hdr->type_len += sz; in btf__add_field()
2047 btf->hdr->str_off += sz; in btf__add_field()
2058 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) in btf_add_enum_common()
2059 return libbpf_err(-EINVAL); in btf_add_enum_common()
2062 return libbpf_err(-ENOMEM); in btf_add_enum_common()
2067 return libbpf_err(-ENOMEM); in btf_add_enum_common()
2076 t->name_off = name_off; in btf_add_enum_common()
2077 t->info = btf_type_info(kind, 0, is_signed); in btf_add_enum_common()
2078 t->size = byte_sz; in btf_add_enum_common()
2085 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2086 * - *byte_sz* - size of the enum, in bytes.
2093 * - >0, type ID of newly added BTF type;
2094 * - <0, on error.
2107 * - *name* - name of the enumerator value, can't be NULL or empty;
2108 * - *value* - integer value corresponding to enum value *name*;
2110 * - 0, on success;
2111 * - <0, on error.
2120 if (btf->nr_types == 0) in btf__add_enum_value()
2121 return libbpf_err(-EINVAL); in btf__add_enum_value()
2124 return libbpf_err(-EINVAL); in btf__add_enum_value()
2126 /* non-empty name */ in btf__add_enum_value()
2128 return libbpf_err(-EINVAL); in btf__add_enum_value()
2130 return libbpf_err(-E2BIG); in btf__add_enum_value()
2134 return libbpf_err(-ENOMEM); in btf__add_enum_value()
2139 return libbpf_err(-ENOMEM); in btf__add_enum_value()
2145 v->name_off = name_off; in btf__add_enum_value()
2146 v->val = value; in btf__add_enum_value()
2154 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); in btf__add_enum_value()
2156 btf->hdr->type_len += sz; in btf__add_enum_value()
2157 btf->hdr->str_off += sz; in btf__add_enum_value()
2163 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2164 * - *byte_sz* - size of the enum, in bytes.
2165 * - *is_signed* - whether the enum values are signed or not;
2172 * - >0, type ID of newly added BTF type;
2173 * - <0, on error.
2184 * - *name* - name of the enumerator value, can't be NULL or empty;
2185 * - *value* - integer value corresponding to enum value *name*;
2187 * - 0, on success;
2188 * - <0, on error.
2197 if (btf->nr_types == 0) in btf__add_enum64_value()
2198 return libbpf_err(-EINVAL); in btf__add_enum64_value()
2201 return libbpf_err(-EINVAL); in btf__add_enum64_value()
2203 /* non-empty name */ in btf__add_enum64_value()
2205 return libbpf_err(-EINVAL); in btf__add_enum64_value()
2209 return libbpf_err(-ENOMEM); in btf__add_enum64_value()
2214 return libbpf_err(-ENOMEM); in btf__add_enum64_value()
2220 v->name_off = name_off; in btf__add_enum64_value()
2221 v->val_lo32 = (__u32)value; in btf__add_enum64_value()
2222 v->val_hi32 = value >> 32; in btf__add_enum64_value()
2228 btf->hdr->type_len += sz; in btf__add_enum64_value()
2229 btf->hdr->str_off += sz; in btf__add_enum64_value()
2235 * - *name*, non-empty/non-NULL name;
2236 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
2239 * - >0, type ID of newly added BTF type;
2240 * - <0, on error.
2245 return libbpf_err(-EINVAL); in btf__add_fwd()
2257 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); in btf__add_fwd()
2262 * values; we also assume a standard 4-byte size for it in btf__add_fwd()
2266 return libbpf_err(-EINVAL); in btf__add_fwd()
2272 * - *name*, non-empty/non-NULL name;
2273 * - *ref_type_id* - referenced type ID, it might not exist yet;
2275 * - >0, type ID of newly added BTF type;
2276 * - <0, on error.
2281 return libbpf_err(-EINVAL); in btf__add_typedef()
2288 * - *ref_type_id* - referenced type ID, it might not exist yet;
2290 * - >0, type ID of newly added BTF type;
2291 * - <0, on error.
2300 * - *ref_type_id* - referenced type ID, it might not exist yet;
2302 * - >0, type ID of newly added BTF type;
2303 * - <0, on error.
2312 * - *ref_type_id* - referenced type ID, it might not exist yet;
2314 * - >0, type ID of newly added BTF type;
2315 * - <0, on error.
2324 * - *value*, non-empty/non-NULL tag value;
2325 * - *ref_type_id* - referenced type ID, it might not exist yet;
2327 * - >0, type ID of newly added BTF type;
2328 * - <0, on error.
2333 return libbpf_err(-EINVAL); in btf__add_type_tag()
2340 * - *name*, non-empty/non-NULL name;
2341 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2343 * - >0, type ID of newly added BTF type;
2344 * - <0, on error.
2352 return libbpf_err(-EINVAL); in btf__add_func()
2355 return libbpf_err(-EINVAL); in btf__add_func()
2361 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); in btf__add_func()
2368 * - *ret_type_id* - type ID for return result of a function.
2375 * - >0, type ID of newly added BTF type;
2376 * - <0, on error.
2384 return libbpf_err(-EINVAL); in btf__add_func_proto()
2387 return libbpf_err(-ENOMEM); in btf__add_func_proto()
2392 return libbpf_err(-ENOMEM); in btf__add_func_proto()
2397 t->name_off = 0; in btf__add_func_proto()
2398 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); in btf__add_func_proto()
2399 t->type = ret_type_id; in btf__add_func_proto()
2406 * - *name* - parameter name, can be NULL or empty;
2407 * - *type_id* - type ID describing the type of the parameter.
2409 * - 0, on success;
2410 * - <0, on error.
2419 return libbpf_err(-EINVAL); in btf__add_func_param()
2422 if (btf->nr_types == 0) in btf__add_func_param()
2423 return libbpf_err(-EINVAL); in btf__add_func_param()
2426 return libbpf_err(-EINVAL); in btf__add_func_param()
2430 return libbpf_err(-ENOMEM); in btf__add_func_param()
2435 return libbpf_err(-ENOMEM); in btf__add_func_param()
2443 p->name_off = name_off; in btf__add_func_param()
2444 p->type = type_id; in btf__add_func_param()
2450 btf->hdr->type_len += sz; in btf__add_func_param()
2451 btf->hdr->str_off += sz; in btf__add_func_param()
2457 * - *name* - non-empty/non-NULL name;
2458 * - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2460 * - *type_id* - type ID of the type describing the type of the variable.
2462 * - >0, type ID of newly added BTF type;
2463 * - <0, on error.
2471 /* non-empty name */ in btf__add_var()
2473 return libbpf_err(-EINVAL); in btf__add_var()
2476 return libbpf_err(-EINVAL); in btf__add_var()
2478 return libbpf_err(-EINVAL); in btf__add_var()
2482 return libbpf_err(-ENOMEM); in btf__add_var()
2487 return libbpf_err(-ENOMEM); in btf__add_var()
2493 t->name_off = name_off; in btf__add_var()
2494 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); in btf__add_var()
2495 t->type = type_id; in btf__add_var()
2498 v->linkage = linkage; in btf__add_var()
2505 * - *name* - non-empty/non-NULL name;
2506 * - *byte_sz* - data section size, in bytes.
2512 * - >0, type ID of newly added BTF type;
2513 * - <0, on error.
2520 /* non-empty name */ in btf__add_datasec()
2522 return libbpf_err(-EINVAL); in btf__add_datasec()
2525 return libbpf_err(-ENOMEM); in btf__add_datasec()
2530 return libbpf_err(-ENOMEM); in btf__add_datasec()
2537 t->name_off = name_off; in btf__add_datasec()
2538 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); in btf__add_datasec()
2539 t->size = byte_sz; in btf__add_datasec()
2546 * - *var_type_id* - type ID, describing type of the variable;
2547 * - *offset* - variable offset within data section, in bytes;
2548 * - *byte_sz* - variable size, in bytes.
2551 * - 0, on success;
2552 * - <0, on error.
2561 if (btf->nr_types == 0) in btf__add_datasec_var_info()
2562 return libbpf_err(-EINVAL); in btf__add_datasec_var_info()
2565 return libbpf_err(-EINVAL); in btf__add_datasec_var_info()
2568 return libbpf_err(-EINVAL); in btf__add_datasec_var_info()
2572 return libbpf_err(-ENOMEM); in btf__add_datasec_var_info()
2577 return libbpf_err(-ENOMEM); in btf__add_datasec_var_info()
2579 v->type = var_type_id; in btf__add_datasec_var_info()
2580 v->offset = offset; in btf__add_datasec_var_info()
2581 v->size = byte_sz; in btf__add_datasec_var_info()
2587 btf->hdr->type_len += sz; in btf__add_datasec_var_info()
2588 btf->hdr->str_off += sz; in btf__add_datasec_var_info()
2594 * - *value* - non-empty/non-NULL string;
2595 * - *ref_type_id* - referenced type ID, it might not exist yet;
2596 * - *component_idx* - -1 for tagging reference type, otherwise struct/union
2599 * - >0, type ID of newly added BTF type;
2600 * - <0, on error.
2608 if (!value || !value[0] || component_idx < -1) in btf__add_decl_tag()
2609 return libbpf_err(-EINVAL); in btf__add_decl_tag()
2612 return libbpf_err(-EINVAL); in btf__add_decl_tag()
2615 return libbpf_err(-ENOMEM); in btf__add_decl_tag()
2620 return libbpf_err(-ENOMEM); in btf__add_decl_tag()
2626 t->name_off = value_off; in btf__add_decl_tag()
2627 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false); in btf__add_decl_tag()
2628 t->type = ref_type_id; in btf__add_decl_tag()
2629 btf_decl_tag(t)->component_idx = component_idx; in btf__add_decl_tag()
2652 if (ext_sec->len == 0) in btf_ext_setup_info()
2655 if (ext_sec->off & 0x03) { in btf_ext_setup_info()
2657 ext_sec->desc); in btf_ext_setup_info()
2658 return -EINVAL; in btf_ext_setup_info()
2661 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; in btf_ext_setup_info()
2662 info_left = ext_sec->len; in btf_ext_setup_info()
2664 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { in btf_ext_setup_info()
2666 ext_sec->desc, ext_sec->off, ext_sec->len); in btf_ext_setup_info()
2667 return -EINVAL; in btf_ext_setup_info()
2672 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); in btf_ext_setup_info()
2673 return -EINVAL; in btf_ext_setup_info()
2678 if (record_size < ext_sec->min_rec_size || in btf_ext_setup_info()
2681 ext_sec->desc, record_size); in btf_ext_setup_info()
2682 return -EINVAL; in btf_ext_setup_info()
2686 info_left -= sizeof(__u32); in btf_ext_setup_info()
2690 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc); in btf_ext_setup_info()
2691 return -EINVAL; in btf_ext_setup_info()
2701 ext_sec->desc); in btf_ext_setup_info()
2702 return -EINVAL; in btf_ext_setup_info()
2705 num_records = sinfo->num_info; in btf_ext_setup_info()
2708 ext_sec->desc); in btf_ext_setup_info()
2709 return -EINVAL; in btf_ext_setup_info()
2715 ext_sec->desc); in btf_ext_setup_info()
2716 return -EINVAL; in btf_ext_setup_info()
2719 info_left -= total_record_size; in btf_ext_setup_info()
2724 ext_info = ext_sec->ext_info; in btf_ext_setup_info()
2725 ext_info->len = ext_sec->len - sizeof(__u32); in btf_ext_setup_info()
2726 ext_info->rec_size = record_size; in btf_ext_setup_info()
2727 ext_info->info = info + sizeof(__u32); in btf_ext_setup_info()
2728 ext_info->sec_cnt = sec_cnt; in btf_ext_setup_info()
2736 .off = btf_ext->hdr->func_info_off, in btf_ext_setup_func_info()
2737 .len = btf_ext->hdr->func_info_len, in btf_ext_setup_func_info()
2739 .ext_info = &btf_ext->func_info, in btf_ext_setup_func_info()
2749 .off = btf_ext->hdr->line_info_off, in btf_ext_setup_line_info()
2750 .len = btf_ext->hdr->line_info_len, in btf_ext_setup_line_info()
2752 .ext_info = &btf_ext->line_info, in btf_ext_setup_line_info()
2762 .off = btf_ext->hdr->core_relo_off, in btf_ext_setup_core_relos()
2763 .len = btf_ext->hdr->core_relo_len, in btf_ext_setup_core_relos()
2765 .ext_info = &btf_ext->core_relo_info, in btf_ext_setup_core_relos()
2777 data_size < hdr->hdr_len) { in btf_ext_parse_hdr()
2779 return -EINVAL; in btf_ext_parse_hdr()
2782 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_ext_parse_hdr()
2783 pr_warn("BTF.ext in non-native endianness is not supported\n"); in btf_ext_parse_hdr()
2784 return -ENOTSUP; in btf_ext_parse_hdr()
2785 } else if (hdr->magic != BTF_MAGIC) { in btf_ext_parse_hdr()
2786 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); in btf_ext_parse_hdr()
2787 return -EINVAL; in btf_ext_parse_hdr()
2790 if (hdr->version != BTF_VERSION) { in btf_ext_parse_hdr()
2791 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); in btf_ext_parse_hdr()
2792 return -ENOTSUP; in btf_ext_parse_hdr()
2795 if (hdr->flags) { in btf_ext_parse_hdr()
2796 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); in btf_ext_parse_hdr()
2797 return -ENOTSUP; in btf_ext_parse_hdr()
2800 if (data_size == hdr->hdr_len) { in btf_ext_parse_hdr()
2802 return -EINVAL; in btf_ext_parse_hdr()
2812 free(btf_ext->func_info.sec_idxs); in btf_ext__free()
2813 free(btf_ext->line_info.sec_idxs); in btf_ext__free()
2814 free(btf_ext->core_relo_info.sec_idxs); in btf_ext__free()
2815 free(btf_ext->data); in btf_ext__free()
2826 return libbpf_err_ptr(-ENOMEM); in btf_ext__new()
2828 btf_ext->data_size = size; in btf_ext__new()
2829 btf_ext->data = malloc(size); in btf_ext__new()
2830 if (!btf_ext->data) { in btf_ext__new()
2831 err = -ENOMEM; in btf_ext__new()
2834 memcpy(btf_ext->data, data, size); in btf_ext__new()
2836 err = btf_ext_parse_hdr(btf_ext->data, size); in btf_ext__new()
2840 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { in btf_ext__new()
2841 err = -EINVAL; in btf_ext__new()
2853 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) in btf_ext__new()
2871 *size = btf_ext->data_size; in btf_ext__get_raw_data()
2872 return btf_ext->data; in btf_ext__get_raw_data()
2890 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF
2892 * memory in-place with deduplicated types and strings without any loss of
2902 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
2904 * Problem description and justification
3006 * other CUs. Each input BTF type eventually gets either mapped to itself, if
3009 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
3013 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
3030 return libbpf_err(-EINVAL); in btf__dedup()
3035 return libbpf_err(-EINVAL); in btf__dedup()
3039 err = -ENOMEM; in btf__dedup()
3084 #define BTF_UNPROCESSED_ID ((__u32)-1)
3085 #define BTF_IN_PROGRESS_ID ((__u32)-2)
3088 /* .BTF section to be deduped in-place */
3129 hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
3133 return hashmap__append(d->dedup_table, in btf_dedup_table_add()
3140 if (d->hypot_cnt == d->hypot_cap) { in btf_dedup_hypot_map_add()
3143 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); in btf_dedup_hypot_map_add()
3144 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); in btf_dedup_hypot_map_add()
3146 return -ENOMEM; in btf_dedup_hypot_map_add()
3147 d->hypot_list = new_list; in btf_dedup_hypot_map_add()
3149 d->hypot_list[d->hypot_cnt++] = from_id; in btf_dedup_hypot_map_add()
3150 d->hypot_map[from_id] = to_id; in btf_dedup_hypot_map_add()
3158 for (i = 0; i < d->hypot_cnt; i++) in btf_dedup_clear_hypot_map()
3159 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; in btf_dedup_clear_hypot_map()
3160 d->hypot_cnt = 0; in btf_dedup_clear_hypot_map()
3161 d->hypot_adjust_canon = false; in btf_dedup_clear_hypot_map()
3166 hashmap__free(d->dedup_table); in btf_dedup_free()
3167 d->dedup_table = NULL; in btf_dedup_free()
3169 free(d->map); in btf_dedup_free()
3170 d->map = NULL; in btf_dedup_free()
3172 free(d->hypot_map); in btf_dedup_free()
3173 d->hypot_map = NULL; in btf_dedup_free()
3175 free(d->hypot_list); in btf_dedup_free()
3176 d->hypot_list = NULL; in btf_dedup_free()
3203 return ERR_PTR(-ENOMEM); in btf_dedup_new()
3208 d->btf = btf; in btf_dedup_new()
3209 d->btf_ext = OPTS_GET(opts, btf_ext, NULL); in btf_dedup_new()
3211 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); in btf_dedup_new()
3212 if (IS_ERR(d->dedup_table)) { in btf_dedup_new()
3213 err = PTR_ERR(d->dedup_table); in btf_dedup_new()
3214 d->dedup_table = NULL; in btf_dedup_new()
3219 d->map = malloc(sizeof(__u32) * type_cnt); in btf_dedup_new()
3220 if (!d->map) { in btf_dedup_new()
3221 err = -ENOMEM; in btf_dedup_new()
3225 d->map[0] = 0; in btf_dedup_new()
3227 struct btf_type *t = btf_type_by_id(d->btf, i); in btf_dedup_new()
3229 /* VAR and DATASEC are never deduped and are self-canonical */ in btf_dedup_new()
3231 d->map[i] = i; in btf_dedup_new()
3233 d->map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3236 d->hypot_map = malloc(sizeof(__u32) * type_cnt); in btf_dedup_new()
3237 if (!d->hypot_map) { in btf_dedup_new()
3238 err = -ENOMEM; in btf_dedup_new()
3242 d->hypot_map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3261 for (i = 0; i < d->btf->nr_types; i++) { in btf_for_each_str_off()
3262 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); in btf_for_each_str_off()
3269 if (!d->btf_ext) in btf_for_each_str_off()
3272 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); in btf_for_each_str_off()
3287 if (str_off == 0 || str_off < d->btf->start_str_off) in strs_dedup_remap_str_off()
3290 s = btf__str_by_offset(d->btf, str_off); in strs_dedup_remap_str_off()
3291 if (d->btf->base_btf) { in strs_dedup_remap_str_off()
3292 err = btf__find_str(d->btf->base_btf, s); in strs_dedup_remap_str_off()
3297 if (err != -ENOENT) in strs_dedup_remap_str_off()
3301 off = strset__add_str(d->strs_set, s); in strs_dedup_remap_str_off()
3305 *str_off_ptr = d->btf->start_str_off + off; in strs_dedup_remap_str_off()
3324 if (d->btf->strs_deduped) in btf_dedup_strings()
3327 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); in btf_dedup_strings()
3328 if (IS_ERR(d->strs_set)) { in btf_dedup_strings()
3329 err = PTR_ERR(d->strs_set); in btf_dedup_strings()
3333 if (!d->btf->base_btf) { in btf_dedup_strings()
3337 err = strset__add_str(d->strs_set, ""); in btf_dedup_strings()
3348 strset__free(d->btf->strs_set); in btf_dedup_strings()
3349 d->btf->hdr->str_len = strset__data_size(d->strs_set); in btf_dedup_strings()
3350 d->btf->strs_set = d->strs_set; in btf_dedup_strings()
3351 d->strs_set = NULL; in btf_dedup_strings()
3352 d->btf->strs_deduped = true; in btf_dedup_strings()
3356 strset__free(d->strs_set); in btf_dedup_strings()
3357 d->strs_set = NULL; in btf_dedup_strings()
3366 h = hash_combine(0, t->name_off); in btf_hash_common()
3367 h = hash_combine(h, t->info); in btf_hash_common()
3368 h = hash_combine(h, t->size); in btf_hash_common()
3374 return t1->name_off == t2->name_off && in btf_equal_common()
3375 t1->info == t2->info && in btf_equal_common()
3376 t1->size == t2->size; in btf_equal_common()
3408 h = hash_combine(0, t->name_off); in btf_hash_enum()
3409 h = hash_combine(h, t->info & ~0xffff); in btf_hash_enum()
3410 h = hash_combine(h, t->size); in btf_hash_enum()
3428 if (m1->name_off != m2->name_off || m1->val != m2->val) in btf_equal_enum()
3449 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || in btf_equal_enum64()
3450 m1->val_hi32 != m2->val_hi32) in btf_equal_enum64()
3468 return t1->name_off == t2->name_off && in btf_compat_enum()
3469 (t1->info & ~0xffff) == (t2->info & ~0xffff) && in btf_compat_enum()
3470 t1->size == t2->size; in btf_compat_enum()
3479 return t1->name_off == t2->name_off && in btf_compat_enum64()
3480 (t1->info & ~0xffff) == (t2->info & ~0xffff) && in btf_compat_enum64()
3481 t1->size == t2->size; in btf_compat_enum64()
3497 h = hash_combine(h, member->name_off); in btf_hash_struct()
3498 h = hash_combine(h, member->offset); in btf_hash_struct()
3523 if (m1->name_off != m2->name_off || m1->offset != m2->offset) in btf_shallow_equal_struct()
3541 h = hash_combine(h, info->type); in btf_hash_array()
3542 h = hash_combine(h, info->index_type); in btf_hash_array()
3543 h = hash_combine(h, info->nelems); in btf_hash_array()
3563 return info1->type == info2->type && in btf_equal_array()
3564 info1->index_type == info2->index_type && in btf_equal_array()
3565 info1->nelems == info2->nelems; in btf_equal_array()
3578 return btf_array(t1)->nelems == btf_array(t2)->nelems; in btf_compat_array()
3594 h = hash_combine(h, member->name_off); in btf_hash_fnproto()
3595 h = hash_combine(h, member->type); in btf_hash_fnproto()
3621 if (m1->name_off != m2->name_off || m1->type != m2->type) in btf_equal_fnproto()
3641 if (t1->name_off != t2->name_off || t1->info != t2->info) in btf_compat_fnproto()
3648 if (m1->name_off != m2->name_off) in btf_compat_fnproto()
3666 if (!d->btf->base_btf) in btf_dedup_prep()
3669 for (type_id = 1; type_id < d->btf->start_id; type_id++) { in btf_dedup_prep()
3670 t = btf_type_by_id(d->btf, type_id); in btf_dedup_prep()
3672 /* all base BTF types are self-canonical by definition */ in btf_dedup_prep()
3673 d->map[type_id] = type_id; in btf_dedup_prep()
3711 return -EINVAL; in btf_dedup_prep()
3714 return -ENOMEM; in btf_dedup_prep()
3724 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
3728 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_prim_type()
3756 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3757 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3768 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3769 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3781 d->map[cand_id] = type_id; in btf_dedup_prim_type()
3789 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3790 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3802 d->map[cand_id] = type_id; in btf_dedup_prim_type()
3811 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3812 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3821 return -EINVAL; in btf_dedup_prim_type()
3824 d->map[type_id] = new_id; in btf_dedup_prim_type()
3826 return -ENOMEM; in btf_dedup_prim_type()
3835 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_prim_types()
3836 err = btf_dedup_prim_type(d, d->btf->start_id + i); in btf_dedup_prim_types()
3848 return d->map[type_id] <= BTF_MAX_NR_TYPES; in is_type_mapped()
3858 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_type_id()
3859 type_id = d->map[type_id]; in resolve_type_id()
3871 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3874 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_fwd_id()
3875 type_id = d->map[type_id]; in resolve_fwd_id()
3877 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3894 t1 = btf_type_by_id(d->btf, id1); in btf_dedup_identical_arrays()
3895 t2 = btf_type_by_id(d->btf, id2); in btf_dedup_identical_arrays()
3909 t1 = btf_type_by_id(d->btf, id1); in btf_dedup_identical_structs()
3910 t2 = btf_type_by_id(d->btf, id2); in btf_dedup_identical_structs()
3921 if (m1->type != m2->type) in btf_dedup_identical_structs()
3935 * - >0, if type graphs are equivalent;
3936 * - 0, if not equivalent;
3937 * - <0, on error.
3939 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
3944 * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
3951 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
3954 * - `canon_id` is mapped to different type. This is contradiction in our
3968 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
3993 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
3997 * - Both types in canonical and candidate graphs are FWDs. If they are
4001 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
4003 * - Type in canonical graph is FWD, while type in candidate is concrete
4012 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
4017 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
4036 hypot_type_id = d->hypot_map[canon_id]; in btf_dedup_is_equiv()
4044 * types sub-graph has a consistent and deduped-by-compiler in btf_dedup_is_equiv()
4055 * complicated and requires a many-to-many equivalence mapping. in btf_dedup_is_equiv()
4063 return -ENOMEM; in btf_dedup_is_equiv()
4065 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_is_equiv()
4066 canon_type = btf_type_by_id(d->btf, canon_id); in btf_dedup_is_equiv()
4070 if (cand_type->name_off != canon_type->name_off) in btf_dedup_is_equiv()
4073 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ in btf_dedup_is_equiv()
4086 if (fwd_kind == real_kind && canon_id < d->btf->start_id) in btf_dedup_is_equiv()
4087 d->hypot_adjust_canon = true; in btf_dedup_is_equiv()
4116 if (cand_type->info != canon_type->info) in btf_dedup_is_equiv()
4118 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
4127 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); in btf_dedup_is_equiv()
4130 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); in btf_dedup_is_equiv()
4144 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); in btf_dedup_is_equiv()
4160 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
4167 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); in btf_dedup_is_equiv()
4177 return -EINVAL; in btf_dedup_is_equiv()
4187 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
4215 for (i = 0; i < d->hypot_cnt; i++) { in btf_dedup_merge_hypot_map()
4216 canon_type_id = d->hypot_list[i]; in btf_dedup_merge_hypot_map()
4217 targ_type_id = d->hypot_map[canon_type_id]; in btf_dedup_merge_hypot_map()
4220 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); in btf_dedup_merge_hypot_map()
4221 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); in btf_dedup_merge_hypot_map()
4226 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because in btf_dedup_merge_hypot_map()
4243 d->map[c_id] = t_id; in btf_dedup_merge_hypot_map()
4250 if (d->hypot_adjust_canon) in btf_dedup_merge_hypot_map()
4254 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4266 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4283 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4284 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4290 * as canonical for itself and is added into btf_dedup->dedup_table hash map
4303 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_struct_type()
4306 t = btf_type_by_id(d->btf, type_id); in btf_dedup_struct_type()
4314 __u32 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_struct_type()
4324 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because in btf_dedup_struct_type()
4327 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_struct_type()
4338 if (d->hypot_adjust_canon) /* not really equivalent */ in btf_dedup_struct_type()
4344 d->map[type_id] = new_id; in btf_dedup_struct_type()
4346 return -ENOMEM; in btf_dedup_struct_type()
4355 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_struct_types()
4356 err = btf_dedup_struct_type(d, d->btf->start_id + i); in btf_dedup_struct_types()
4385 * btf_dedup->dedup_table as another possible canonical representative.
4396 if (d->map[type_id] == BTF_IN_PROGRESS_ID) in btf_dedup_ref_type()
4397 return -ELOOP; in btf_dedup_ref_type()
4398 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_ref_type()
4401 t = btf_type_by_id(d->btf, type_id); in btf_dedup_ref_type()
4402 d->map[type_id] = BTF_IN_PROGRESS_ID; in btf_dedup_ref_type()
4412 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4415 t->type = ref_type_id; in btf_dedup_ref_type()
4419 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4420 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4429 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4432 t->type = ref_type_id; in btf_dedup_ref_type()
4436 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4437 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4448 ref_type_id = btf_dedup_ref_type(d, info->type); in btf_dedup_ref_type()
4451 info->type = ref_type_id; in btf_dedup_ref_type()
4453 ref_type_id = btf_dedup_ref_type(d, info->index_type); in btf_dedup_ref_type()
4456 info->index_type = ref_type_id; in btf_dedup_ref_type()
4460 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4461 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4475 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4478 t->type = ref_type_id; in btf_dedup_ref_type()
4483 ref_type_id = btf_dedup_ref_type(d, param->type); in btf_dedup_ref_type()
4486 param->type = ref_type_id; in btf_dedup_ref_type()
4492 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4493 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4503 return -EINVAL; in btf_dedup_ref_type()
4506 d->map[type_id] = new_id; in btf_dedup_ref_type()
4508 return -ENOMEM; in btf_dedup_ref_type()
4517 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_ref_types()
4518 err = btf_dedup_ref_type(d, d->btf->start_id + i); in btf_dedup_ref_types()
4522 /* we won't need d->dedup_table anymore */ in btf_dedup_ref_types()
4523 hashmap__free(d->dedup_table); in btf_dedup_ref_types()
4524 d->dedup_table = NULL; in btf_dedup_ref_types()
4534 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
4542 __u32 next_type_id = d->btf->start_id; in btf_dedup_compact_types()
4548 d->hypot_map[0] = 0; in btf_dedup_compact_types()
4550 for (id = 1; id < d->btf->start_id; id++) in btf_dedup_compact_types()
4551 d->hypot_map[id] = id; in btf_dedup_compact_types()
4552 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) in btf_dedup_compact_types()
4553 d->hypot_map[id] = BTF_UNPROCESSED_ID; in btf_dedup_compact_types()
4555 p = d->btf->types_data; in btf_dedup_compact_types()
4557 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { in btf_dedup_compact_types()
4558 if (d->map[id] != id) in btf_dedup_compact_types()
4561 t = btf__type_by_id(d->btf, id); in btf_dedup_compact_types()
4567 d->hypot_map[id] = next_type_id; in btf_dedup_compact_types()
4568 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; in btf_dedup_compact_types()
4574 d->btf->nr_types = next_type_id - d->btf->start_id; in btf_dedup_compact_types()
4575 d->btf->type_offs_cap = d->btf->nr_types; in btf_dedup_compact_types()
4576 d->btf->hdr->type_len = p - d->btf->types_data; in btf_dedup_compact_types()
4577 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, in btf_dedup_compact_types()
4579 if (d->btf->type_offs_cap && !new_offs) in btf_dedup_compact_types()
4580 return -ENOMEM; in btf_dedup_compact_types()
4581 d->btf->type_offs = new_offs; in btf_dedup_compact_types()
4582 d->btf->hdr->str_off = d->btf->hdr->type_len; in btf_dedup_compact_types()
4583 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; in btf_dedup_compact_types()
4590 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
4599 new_type_id = d->hypot_map[resolved_type_id]; in btf_dedup_remap_type_id()
4601 return -EINVAL; in btf_dedup_remap_type_id()
4612 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
4621 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_remap_types()
4622 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); in btf_dedup_remap_types()
4629 if (!d->btf_ext) in btf_dedup_remap_types()
4632 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); in btf_dedup_remap_types()
4640 * Probe few well-known locations for vmlinux kernel image and try to load BTF
4649 "/boot/vmlinux-%1$s", in btf__load_vmlinux_btf()
4650 "/lib/modules/%1$s/vmlinux-%1$s", in btf__load_vmlinux_btf()
4653 "/usr/lib/debug/boot/vmlinux-%1$s", in btf__load_vmlinux_btf()
4654 "/usr/lib/debug/boot/vmlinux-%1$s.debug", in btf__load_vmlinux_btf()
4680 return libbpf_err_ptr(-ESRCH); in btf__load_vmlinux_btf()
4714 return visit(&t->type, ctx); in btf_type_visit_type_ids()
4719 err = visit(&a->type, ctx); in btf_type_visit_type_ids()
4720 err = err ?: visit(&a->index_type, ctx); in btf_type_visit_type_ids()
4729 err = visit(&m->type, ctx); in btf_type_visit_type_ids()
4739 err = visit(&t->type, ctx); in btf_type_visit_type_ids()
4743 err = visit(&m->type, ctx); in btf_type_visit_type_ids()
4754 err = visit(&m->type, ctx); in btf_type_visit_type_ids()
4762 return -EINVAL; in btf_type_visit_type_ids()
4770 err = visit(&t->name_off, ctx); in btf_type_visit_str_offs()
4780 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4790 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4800 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4810 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4829 seg = &btf_ext->func_info; in btf_ext_visit_type_ids()
4834 err = visit(&rec->type_id, ctx); in btf_ext_visit_type_ids()
4840 seg = &btf_ext->core_relo_info; in btf_ext_visit_type_ids()
4845 err = visit(&rec->type_id, ctx); in btf_ext_visit_type_ids()
4860 seg = &btf_ext->func_info; in btf_ext_visit_str_offs()
4862 err = visit(&sec->sec_name_off, ctx); in btf_ext_visit_str_offs()
4867 seg = &btf_ext->line_info; in btf_ext_visit_str_offs()
4871 err = visit(&sec->sec_name_off, ctx); in btf_ext_visit_str_offs()
4876 err = visit(&rec->file_name_off, ctx); in btf_ext_visit_str_offs()
4879 err = visit(&rec->line_off, ctx); in btf_ext_visit_str_offs()
4885 seg = &btf_ext->core_relo_info; in btf_ext_visit_str_offs()
4889 err = visit(&sec->sec_name_off, ctx); in btf_ext_visit_str_offs()
4894 err = visit(&rec->access_str_off, ctx); in btf_ext_visit_str_offs()