1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
34 	/* raw BTF data in non-native endianness */
46 	 * +--------------------------------+
48 	 * +--------------------------------+
52 	 * types_data-+         |
53 	 * strs_data------------+
64 	 * +----------+  +---------+  +-----------+
66 	 * +----------+  +---------+  +-----------+
70 	 * types_data----+            |
71 	 * strset__data(strs_set)-----+
73 	 *               +----------+---------+-----------+
75 	 * raw_data----->+----------+---------+-----------+
80 	size_t types_data_cap; /* used size stored in hdr->type_len */
83 	 * type_offs[0] corresponds to the first non-VOID type:
84 	 *   - for base BTF it's type [1];
85 	 *   - for split BTF it's the first non-base BTF type.
90 	 *   - doesn't include special [0] void type;
91 	 *   - for split BTF counts number of types added on top of base BTF.
99 	 *   - for base BTF it's equal to 1;
100 	 *   - for split BTF it's equal to biggest type ID of base BTF plus 1.
104 	 *   - for base BTF it's equal to 0;
105 	 *   - for split BTF it's equal to total size of base BTF's string section size.
109 	/* only one of strs_data or strs_set can be non-NULL, depending on
168 	memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz);  in libbpf_add_mem()
185 	p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt);  in libbpf_ensure_mem()
187 		return -ENOMEM;  in libbpf_ensure_mem()
196 	p = libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),  in btf_add_type_idx_entry()
197 			   btf->nr_types, BTF_MAX_NR_TYPES, 1);  in btf_add_type_idx_entry()
199 		return -ENOMEM;  in btf_add_type_idx_entry()
207 	h->magic = bswap_16(h->magic);  in btf_bswap_hdr()
208 	h->hdr_len = bswap_32(h->hdr_len);  in btf_bswap_hdr()
209 	h->type_off = bswap_32(h->type_off);  in btf_bswap_hdr()
210 	h->type_len = bswap_32(h->type_len);  in btf_bswap_hdr()
211 	h->str_off = bswap_32(h->str_off);  in btf_bswap_hdr()
212 	h->str_len = bswap_32(h->str_len);  in btf_bswap_hdr()
217 	struct btf_header *hdr = btf->hdr;  in btf_parse_hdr()
220 	if (btf->raw_size < sizeof(struct btf_header)) {  in btf_parse_hdr()
222 		return -EINVAL;  in btf_parse_hdr()
225 	if (hdr->magic == bswap_16(BTF_MAGIC)) {  in btf_parse_hdr()
226 		btf->swapped_endian = true;  in btf_parse_hdr()
227 		if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {  in btf_parse_hdr()
228 			pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",  in btf_parse_hdr()
229 				bswap_32(hdr->hdr_len));  in btf_parse_hdr()
230 			return -ENOTSUP;  in btf_parse_hdr()
233 	} else if (hdr->magic != BTF_MAGIC) {  in btf_parse_hdr()
234 		pr_debug("Invalid BTF magic:%x\n", hdr->magic);  in btf_parse_hdr()
235 		return -EINVAL;  in btf_parse_hdr()
238 	meta_left = btf->raw_size - sizeof(*hdr);  in btf_parse_hdr()
239 	if (meta_left < hdr->str_off + hdr->str_len) {  in btf_parse_hdr()
240 		pr_debug("Invalid BTF total size:%u\n", btf->raw_size);  in btf_parse_hdr()
241 		return -EINVAL;  in btf_parse_hdr()
244 	if (hdr->type_off + hdr->type_len > hdr->str_off) {  in btf_parse_hdr()
246 			 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);  in btf_parse_hdr()
247 		return -EINVAL;  in btf_parse_hdr()
250 	if (hdr->type_off % 4) {  in btf_parse_hdr()
252 		return -EINVAL;  in btf_parse_hdr()
260 	const struct btf_header *hdr = btf->hdr;  in btf_parse_str_sec()
261 	const char *start = btf->strs_data;  in btf_parse_str_sec()
262 	const char *end = start + btf->hdr->str_len;  in btf_parse_str_sec()
264 	if (btf->base_btf && hdr->str_len == 0)  in btf_parse_str_sec()
266 	if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {  in btf_parse_str_sec()
268 		return -EINVAL;  in btf_parse_str_sec()
270 	if (!btf->base_btf && start[0]) {  in btf_parse_str_sec()
272 		return -EINVAL;  in btf_parse_str_sec()
309 		return -EINVAL;  in btf_type_size()
315 	t->name_off = bswap_32(t->name_off);  in btf_bswap_type_base()
316 	t->info = bswap_32(t->info);  in btf_bswap_type_base()
317 	t->type = bswap_32(t->type);  in btf_bswap_type_base()
345 			e->name_off = bswap_32(e->name_off);  in btf_bswap_type_rest()
346 			e->val = bswap_32(e->val);  in btf_bswap_type_rest()
351 		a->type = bswap_32(a->type);  in btf_bswap_type_rest()
352 		a->index_type = bswap_32(a->index_type);  in btf_bswap_type_rest()
353 		a->nelems = bswap_32(a->nelems);  in btf_bswap_type_rest()
358 			m->name_off = bswap_32(m->name_off);  in btf_bswap_type_rest()
359 			m->type = bswap_32(m->type);  in btf_bswap_type_rest()
360 			m->offset = bswap_32(m->offset);  in btf_bswap_type_rest()
365 			p->name_off = bswap_32(p->name_off);  in btf_bswap_type_rest()
366 			p->type = bswap_32(p->type);  in btf_bswap_type_rest()
370 		btf_var(t)->linkage = bswap_32(btf_var(t)->linkage);  in btf_bswap_type_rest()
374 			v->type = bswap_32(v->type);  in btf_bswap_type_rest()
375 			v->offset = bswap_32(v->offset);  in btf_bswap_type_rest()
376 			v->size = bswap_32(v->size);  in btf_bswap_type_rest()
381 		return -EINVAL;  in btf_bswap_type_rest()
387 	struct btf_header *hdr = btf->hdr;  in btf_parse_type_sec()
388 	void *next_type = btf->types_data;  in btf_parse_type_sec()
389 	void *end_type = next_type + hdr->type_len;  in btf_parse_type_sec()
393 		if (btf->swapped_endian)  in btf_parse_type_sec()
400 			pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);  in btf_parse_type_sec()
401 			return -EINVAL;  in btf_parse_type_sec()
404 		if (btf->swapped_endian && btf_bswap_type_rest(next_type))  in btf_parse_type_sec()
405 			return -EINVAL;  in btf_parse_type_sec()
407 		err = btf_add_type_idx_entry(btf, next_type - btf->types_data);  in btf_parse_type_sec()
412 		btf->nr_types++;  in btf_parse_type_sec()
417 		return -EINVAL;  in btf_parse_type_sec()
425 	return btf->start_id + btf->nr_types - 1;  in btf__get_nr_types()
430 	return btf->base_btf;  in btf__base_btf()
433 /* internal helper returning non-const pointer to a type */
438 	if (type_id < btf->start_id)  in btf_type_by_id()
439 		return btf_type_by_id(btf->base_btf, type_id);  in btf_type_by_id()
440 	return btf->types_data + btf->type_offs[type_id - btf->start_id];  in btf_type_by_id()
445 	if (type_id >= btf->start_id + btf->nr_types)  in btf__type_by_id()
456 	if (btf->base_btf && btf->base_btf->ptr_sz > 0)  in determine_ptr_size()
457 		return btf->base_btf->ptr_sz;  in determine_ptr_size()
465 		name = btf__name_by_offset(btf, t->name_off);  in determine_ptr_size()
471 			if (t->size != 4 && t->size != 8)  in determine_ptr_size()
473 			return t->size;  in determine_ptr_size()
477 	return -1;  in determine_ptr_size()
482 	if (!btf->ptr_sz)  in btf_ptr_sz()
483 		((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);  in btf_ptr_sz()
484 	return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz;  in btf_ptr_sz()
497 	if (!btf->ptr_sz)  in btf__pointer_size()
498 		((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);  in btf__pointer_size()
500 	if (btf->ptr_sz < 0)  in btf__pointer_size()
504 	return btf->ptr_sz;  in btf__pointer_size()
513 		return libbpf_err(-EINVAL);  in btf__set_pointer_size()
514 	btf->ptr_sz = ptr_sz;  in btf__set_pointer_size()
532 		return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;  in btf__endianness()
534 		return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;  in btf__endianness()
540 		return libbpf_err(-EINVAL);  in btf__set_endianness()
542 	btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);  in btf__set_endianness()
543 	if (!btf->swapped_endian) {  in btf__set_endianness()
544 		free(btf->raw_data_swapped);  in btf__set_endianness()
545 		btf->raw_data_swapped = NULL;  in btf__set_endianness()
567 	__s64 size = -1;  in btf__resolve_size()
579 			size = t->size;  in btf__resolve_size()
589 			type_id = t->type;  in btf__resolve_size()
593 			if (nelems && array->nelems > UINT32_MAX / nelems)  in btf__resolve_size()
594 				return libbpf_err(-E2BIG);  in btf__resolve_size()
595 			nelems *= array->nelems;  in btf__resolve_size()
596 			type_id = array->type;  in btf__resolve_size()
599 			return libbpf_err(-EINVAL);  in btf__resolve_size()
607 		return libbpf_err(-EINVAL);  in btf__resolve_size()
609 		return libbpf_err(-E2BIG);  in btf__resolve_size()
623 		return min(btf_ptr_sz(btf), (size_t)t->size);  in btf__align_of()
630 		return btf__align_of(btf, t->type);  in btf__align_of()
632 		return btf__align_of(btf, btf_array(t)->type);  in btf__align_of()
640 			align = btf__align_of(btf, m->type);  in btf__align_of()
663 		type_id = t->type;  in btf__resolve_type()
669 		return libbpf_err(-EINVAL);  in btf__resolve_type()
683 		const char *name = btf__name_by_offset(btf, t->name_off);  in btf__find_by_name()
689 	return libbpf_err(-ENOENT);  in btf__find_by_name()
706 		name = btf__name_by_offset(btf, t->name_off);  in btf__find_by_name_kind()
711 	return libbpf_err(-ENOENT);  in btf__find_by_name_kind()
716 	return (void *)btf->hdr != btf->raw_data;  in btf_is_modifiable()
724 	if (btf->fd >= 0)  in btf__free()
725 		close(btf->fd);  in btf__free()
729 		 * in-memory representation for header, types, and strings  in btf__free()
734 		free(btf->hdr);  in btf__free()
735 		free(btf->types_data);  in btf__free()
736 		strset__free(btf->strs_set);  in btf__free()
738 	free(btf->raw_data);  in btf__free()
739 	free(btf->raw_data_swapped);  in btf__free()
740 	free(btf->type_offs);  in btf__free()
750 		return ERR_PTR(-ENOMEM);  in btf_new_empty()
752 	btf->nr_types = 0;  in btf_new_empty()
753 	btf->start_id = 1;  in btf_new_empty()
754 	btf->start_str_off = 0;  in btf_new_empty()
755 	btf->fd = -1;  in btf_new_empty()
756 	btf->ptr_sz = sizeof(void *);  in btf_new_empty()
757 	btf->swapped_endian = false;  in btf_new_empty()
760 		btf->base_btf = base_btf;  in btf_new_empty()
761 		btf->start_id = btf__get_nr_types(base_btf) + 1;  in btf_new_empty()
762 		btf->start_str_off = base_btf->hdr->str_len;  in btf_new_empty()
766 	btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);  in btf_new_empty()
767 	btf->raw_data = calloc(1, btf->raw_size);  in btf_new_empty()
768 	if (!btf->raw_data) {  in btf_new_empty()
770 		return ERR_PTR(-ENOMEM);  in btf_new_empty()
773 	btf->hdr = btf->raw_data;  in btf_new_empty()
774 	btf->hdr->hdr_len = sizeof(struct btf_header);  in btf_new_empty()
775 	btf->hdr->magic = BTF_MAGIC;  in btf_new_empty()
776 	btf->hdr->version = BTF_VERSION;  in btf_new_empty()
778 	btf->types_data = btf->raw_data + btf->hdr->hdr_len;  in btf_new_empty()
779 	btf->strs_data = btf->raw_data + btf->hdr->hdr_len;  in btf_new_empty()
780 	btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */  in btf_new_empty()
802 		return ERR_PTR(-ENOMEM);  in btf_new()
804 	btf->nr_types = 0;  in btf_new()
805 	btf->start_id = 1;  in btf_new()
806 	btf->start_str_off = 0;  in btf_new()
807 	btf->fd = -1;  in btf_new()
810 		btf->base_btf = base_btf;  in btf_new()
811 		btf->start_id = btf__get_nr_types(base_btf) + 1;  in btf_new()
812 		btf->start_str_off = base_btf->hdr->str_len;  in btf_new()
815 	btf->raw_data = malloc(size);  in btf_new()
816 	if (!btf->raw_data) {  in btf_new()
817 		err = -ENOMEM;  in btf_new()
820 	memcpy(btf->raw_data, data, size);  in btf_new()
821 	btf->raw_size = size;  in btf_new()
823 	btf->hdr = btf->raw_data;  in btf_new()
828 	btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off;  in btf_new()
829 	btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;  in btf_new()
854 	int err = 0, fd = -1, idx = 0;  in btf_parse_elf()
863 		return ERR_PTR(-LIBBPF_ERRNO__LIBELF);  in btf_parse_elf()
868 		err = -errno;  in btf_parse_elf()
873 	err = -LIBBPF_ERRNO__FORMAT;  in btf_parse_elf()
934 		err = -ENOENT;  in btf_parse_elf()
937 	btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf);  in btf_parse_elf()
955 		*btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);  in btf_parse_elf()
998 		err = -errno;  in btf_parse_raw()
1004 		err = -EIO;  in btf_parse_raw()
1009 		err = -EPROTO;  in btf_parse_raw()
1015 		err = -errno;  in btf_parse_raw()
1020 		err = -errno;  in btf_parse_raw()
1025 		err = -errno;  in btf_parse_raw()
1029 	/* pre-alloc memory and read all of BTF data */  in btf_parse_raw()
1032 		err = -ENOMEM;  in btf_parse_raw()
1036 		err = -EIO;  in btf_parse_raw()
1072 	if (err != -EPROTO)  in btf_parse()
1092 	return a->offset - b->offset;  in compare_vsi_off()
1099 	const char *name = btf__name_by_offset(btf, t->name_off);  in btf_fixup_datasec()
1107 		return -ENOENT;  in btf_fixup_datasec()
1113 	if (t->size)  in btf_fixup_datasec()
1117 	if (ret || !size || (t->size && t->size != size)) {  in btf_fixup_datasec()
1119 		return -ENOENT;  in btf_fixup_datasec()
1122 	t->size = size;  in btf_fixup_datasec()
1125 		t_var = btf__type_by_id(btf, vsi->type);  in btf_fixup_datasec()
1129 			pr_debug("Non-VAR type seen in section %s\n", name);  in btf_fixup_datasec()
1130 			return -EINVAL;  in btf_fixup_datasec()
1133 		if (var->linkage == BTF_VAR_STATIC)  in btf_fixup_datasec()
1136 		name = btf__name_by_offset(btf, t_var->name_off);  in btf_fixup_datasec()
1139 			return -ENOENT;  in btf_fixup_datasec()
1146 			return -ENOENT;  in btf_fixup_datasec()
1149 		vsi->offset = off;  in btf_fixup_datasec()
1162 	for (i = 1; i <= btf->nr_types; i++) {  in btf__finalize_data()
1189 	if (btf->fd >= 0)  in btf__load_into_kernel()
1190 		return libbpf_err(-EEXIST);  in btf__load_into_kernel()
1196 			return libbpf_err(-ENOMEM);  in btf__load_into_kernel()
1203 		err = -ENOMEM;  in btf__load_into_kernel()
1207 	btf->raw_size = raw_size;  in btf__load_into_kernel()
1208 	btf->raw_data = raw_data;  in btf__load_into_kernel()
1210 	btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false);  in btf__load_into_kernel()
1211 	if (btf->fd < 0) {  in btf__load_into_kernel()
1219 		err = -errno;  in btf__load_into_kernel()
1234 	return btf->fd;  in btf__fd()
1239 	btf->fd = fd;  in btf__set_fd()
1244 	return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set);  in btf_strs_data()
1249 	struct btf_header *hdr = btf->hdr;  in btf_get_raw_data()
1255 	data = swap_endian ? btf->raw_data_swapped : btf->raw_data;  in btf_get_raw_data()
1257 		*size = btf->raw_size;  in btf_get_raw_data()
1261 	data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;  in btf_get_raw_data()
1267 	memcpy(p, hdr, hdr->hdr_len);  in btf_get_raw_data()
1270 	p += hdr->hdr_len;  in btf_get_raw_data()
1272 	memcpy(p, btf->types_data, hdr->type_len);  in btf_get_raw_data()
1274 		for (i = 0; i < btf->nr_types; i++) {  in btf_get_raw_data()
1275 			t = p + btf->type_offs[i];  in btf_get_raw_data()
1276 			/* btf_bswap_type_rest() relies on native t->info, so  in btf_get_raw_data()
1285 	p += hdr->type_len;  in btf_get_raw_data()
1287 	memcpy(p, btf_strs_data(btf), hdr->str_len);  in btf_get_raw_data()
1288 	p += hdr->str_len;  in btf_get_raw_data()
1303 	data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian);  in btf__get_raw_data()
1305 		return errno = -ENOMEM, NULL;  in btf__get_raw_data()
1307 	btf->raw_size = data_sz;  in btf__get_raw_data()
1308 	if (btf->swapped_endian)  in btf__get_raw_data()
1309 		btf->raw_data_swapped = data;  in btf__get_raw_data()
1311 		btf->raw_data = data;  in btf__get_raw_data()
1318 	if (offset < btf->start_str_off)  in btf__str_by_offset()
1319 		return btf__str_by_offset(btf->base_btf, offset);  in btf__str_by_offset()
1320 	else if (offset - btf->start_str_off < btf->hdr->str_len)  in btf__str_by_offset()
1321 		return btf_strs_data(btf) + (offset - btf->start_str_off);  in btf__str_by_offset()
1341 	 * let's start with a sane default - 4KiB here - and resize it only if  in btf_get_from_fd()
1347 		return ERR_PTR(-ENOMEM);  in btf_get_from_fd()
1360 			btf = ERR_PTR(-ENOMEM);  in btf_get_from_fd()
1374 		btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);  in btf_get_from_fd()
1392 		return libbpf_err_ptr(-errno);  in btf__load_from_kernel_by_id_split()
1435 		return libbpf_err(-EINVAL);  in btf__get_map_kv_tids()
1449 		return libbpf_err(-EINVAL);  in btf__get_map_kv_tids()
1455 		return libbpf_err(-EINVAL);  in btf__get_map_kv_tids()
1461 	key_size = btf__resolve_size(btf, key->type);  in btf__get_map_kv_tids()
1470 		return libbpf_err(-EINVAL);  in btf__get_map_kv_tids()
1473 	value_size = btf__resolve_size(btf, value->type);  in btf__get_map_kv_tids()
1482 		return libbpf_err(-EINVAL);  in btf__get_map_kv_tids()
1485 	*key_type_id = key->type;  in btf__get_map_kv_tids()
1486 	*value_type_id = value->type;  in btf__get_map_kv_tids()
1493 	if (btf->raw_data) {  in btf_invalidate_raw_data()
1494 		free(btf->raw_data);  in btf_invalidate_raw_data()
1495 		btf->raw_data = NULL;  in btf_invalidate_raw_data()
1497 	if (btf->raw_data_swapped) {  in btf_invalidate_raw_data()
1498 		free(btf->raw_data_swapped);  in btf_invalidate_raw_data()
1499 		btf->raw_data_swapped = NULL;  in btf_invalidate_raw_data()
1511 	int err = -ENOMEM;  in btf_ensure_modifiable()
1520 	hdr = malloc(btf->hdr->hdr_len);  in btf_ensure_modifiable()
1521 	types = malloc(btf->hdr->type_len);  in btf_ensure_modifiable()
1525 	memcpy(hdr, btf->hdr, btf->hdr->hdr_len);  in btf_ensure_modifiable()
1526 	memcpy(types, btf->types_data, btf->hdr->type_len);  in btf_ensure_modifiable()
1529 	set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len);  in btf_ensure_modifiable()
1536 	btf->hdr = hdr;  in btf_ensure_modifiable()
1537 	btf->types_data = types;  in btf_ensure_modifiable()
1538 	btf->types_data_cap = btf->hdr->type_len;  in btf_ensure_modifiable()
1539 	btf->strs_data = NULL;  in btf_ensure_modifiable()
1540 	btf->strs_set = set;  in btf_ensure_modifiable()
1544 	if (btf->hdr->str_len == 0)  in btf_ensure_modifiable()
1545 		btf->strs_deduped = true;  in btf_ensure_modifiable()
1546 	if (!btf->base_btf && btf->hdr->str_len == 1)  in btf_ensure_modifiable()
1547 		btf->strs_deduped = true;  in btf_ensure_modifiable()
1563  *   - >0 offset into string section, if string is found;
1564  *   - -ENOENT, if string is not in the string section;
1565  *   - <0, on any other error.
1571 	if (btf->base_btf) {  in btf__find_str()
1572 		off = btf__find_str(btf->base_btf, s);  in btf__find_str()
1573 		if (off != -ENOENT)  in btf__find_str()
1579 		return libbpf_err(-ENOMEM);  in btf__find_str()
1581 	off = strset__find_str(btf->strs_set, s);  in btf__find_str()
1585 	return btf->start_str_off + off;  in btf__find_str()
1590  *   - > 0 offset into string section, on success;
1591  *   - < 0, on error.
1597 	if (btf->base_btf) {  in btf__add_str()
1598 		off = btf__find_str(btf->base_btf, s);  in btf__add_str()
1599 		if (off != -ENOENT)  in btf__add_str()
1604 		return libbpf_err(-ENOMEM);  in btf__add_str()
1606 	off = strset__add_str(btf->strs_set, s);  in btf__add_str()
1610 	btf->hdr->str_len = strset__data_size(btf->strs_set);  in btf__add_str()
1612 	return btf->start_str_off + off;  in btf__add_str()
1617 	return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1,  in btf_add_type_mem()
1618 			      btf->hdr->type_len, UINT_MAX, add_sz);  in btf_add_type_mem()
1623 	t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));  in btf_type_inc_vlen()
1630 	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);  in btf_commit_type()
1634 	btf->hdr->type_len += data_sz;  in btf_commit_type()
1635 	btf->hdr->str_off += data_sz;  in btf_commit_type()
1636 	btf->nr_types++;  in btf_commit_type()
1637 	return btf->start_id + btf->nr_types - 1;  in btf_commit_type()
1653 	off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off));  in btf_rewrite_str()
1673 		return libbpf_err(-ENOMEM);  in btf__add_type()
1677 		return libbpf_err(-ENOMEM);  in btf__add_type()
1690  *   - *name* - non-empty, non-NULL type name;
1691  *   - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1692  *   - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1694  *   - >0, type ID of newly added BTF type;
1695  *   - <0, on error.
1702 	/* non-empty name */  in btf__add_int()
1704 		return libbpf_err(-EINVAL);  in btf__add_int()
1706 	if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16)  in btf__add_int()
1707 		return libbpf_err(-EINVAL);  in btf__add_int()
1709 		return libbpf_err(-EINVAL);  in btf__add_int()
1713 		return libbpf_err(-ENOMEM);  in btf__add_int()
1718 		return libbpf_err(-ENOMEM);  in btf__add_int()
1728 	t->name_off = name_off;  in btf__add_int()
1729 	t->info = btf_type_info(BTF_KIND_INT, 0, 0);  in btf__add_int()
1730 	t->size = byte_sz;  in btf__add_int()
1739  *   - *name* - non-empty, non-NULL type name;
1740  *   - *sz* - size of the type, in bytes;
1742  *   - >0, type ID of newly added BTF type;
1743  *   - <0, on error.
1750 	/* non-empty name */  in btf__add_float()
1752 		return libbpf_err(-EINVAL);  in btf__add_float()
1757 		return libbpf_err(-EINVAL);  in btf__add_float()
1760 		return libbpf_err(-ENOMEM);  in btf__add_float()
1765 		return libbpf_err(-ENOMEM);  in btf__add_float()
1771 	t->name_off = name_off;  in btf__add_float()
1772 	t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0);  in btf__add_float()
1773 	t->size = byte_sz;  in btf__add_float()
1785 		return -EINVAL;  in validate_type_id()
1796 		return libbpf_err(-EINVAL);  in btf_add_ref_kind()
1799 		return libbpf_err(-ENOMEM);  in btf_add_ref_kind()
1804 		return libbpf_err(-ENOMEM);  in btf_add_ref_kind()
1812 	t->name_off = name_off;  in btf_add_ref_kind()
1813 	t->info = btf_type_info(kind, 0, 0);  in btf_add_ref_kind()
1814 	t->type = ref_type_id;  in btf_add_ref_kind()
1821  *   - *ref_type_id* - referenced type ID, it might not exist yet;
1823  *   - >0, type ID of newly added BTF type;
1824  *   - <0, on error.
1833  *   - *index_type_id* - type ID of the type describing array index;
1834  *   - *elem_type_id* - type ID of the type describing array element;
1835  *   - *nr_elems* - the size of the array;
1837  *   - >0, type ID of newly added BTF type;
1838  *   - <0, on error.
1847 		return libbpf_err(-EINVAL);  in btf__add_array()
1850 		return libbpf_err(-ENOMEM);  in btf__add_array()
1855 		return libbpf_err(-ENOMEM);  in btf__add_array()
1857 	t->name_off = 0;  in btf__add_array()
1858 	t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0);  in btf__add_array()
1859 	t->size = 0;  in btf__add_array()
1862 	a->type = elem_type_id;  in btf__add_array()
1863 	a->index_type = index_type_id;  in btf__add_array()
1864 	a->nelems = nr_elems;  in btf__add_array()
1876 		return libbpf_err(-ENOMEM);  in btf_add_composite()
1881 		return libbpf_err(-ENOMEM);  in btf_add_composite()
1892 	t->name_off = name_off;  in btf_add_composite()
1893 	t->info = btf_type_info(kind, 0, 0);  in btf_add_composite()
1894 	t->size = bytes_sz;  in btf_add_composite()
1901  *   - *name* - name of the struct, can be NULL or empty for anonymous structs;
1902  *   - *byte_sz* - size of the struct, in bytes;
1908  *   - >0, type ID of newly added BTF type;
1909  *   - <0, on error.
1918  *   - *name* - name of the union, can be NULL or empty for anonymous union;
1919  *   - *byte_sz* - size of the union, in bytes;
1926  *   - >0, type ID of newly added BTF type;
1927  *   - <0, on error.
1941  *   - *name* - name of the field, can be NULL or empty for anonymous field;
1942  *   - *type_id* - type ID for the type describing field type;
1943  *   - *bit_offset* - bit offset of the start of the field within struct/union;
1944  *   - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
1946  *   -  0, on success;
1947  *   - <0, on error.
1958 	if (btf->nr_types == 0)  in btf__add_field()
1959 		return libbpf_err(-EINVAL);  in btf__add_field()
1962 		return libbpf_err(-EINVAL);  in btf__add_field()
1965 		return libbpf_err(-EINVAL);  in btf__add_field()
1966 	/* best-effort bit field offset/size enforcement */  in btf__add_field()
1969 		return libbpf_err(-EINVAL);  in btf__add_field()
1973 		return libbpf_err(-EINVAL);  in btf__add_field()
1977 		return libbpf_err(-ENOMEM);  in btf__add_field()
1982 		return libbpf_err(-ENOMEM);  in btf__add_field()
1990 	m->name_off = name_off;  in btf__add_field()
1991 	m->type = type_id;  in btf__add_field()
1992 	m->offset = bit_offset | (bit_size << 24);  in btf__add_field()
1997 	t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));  in btf__add_field()
1999 	btf->hdr->type_len += sz;  in btf__add_field()
2000 	btf->hdr->str_off += sz;  in btf__add_field()
2006  *   - *name* - name of the enum, can be NULL or empty for anonymous enums;
2007  *   - *byte_sz* - size of the enum, in bytes.
2014  *   - >0, type ID of newly added BTF type;
2015  *   - <0, on error.
2023 	if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)  in btf__add_enum()
2024 		return libbpf_err(-EINVAL);  in btf__add_enum()
2027 		return libbpf_err(-ENOMEM);  in btf__add_enum()
2032 		return libbpf_err(-ENOMEM);  in btf__add_enum()
2041 	t->name_off = name_off;  in btf__add_enum()
2042 	t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);  in btf__add_enum()
2043 	t->size = byte_sz;  in btf__add_enum()
2050  *   - *name* - name of the enumerator value, can't be NULL or empty;
2051  *   - *value* - integer value corresponding to enum value *name*;
2053  *   -  0, on success;
2054  *   - <0, on error.
2063 	if (btf->nr_types == 0)  in btf__add_enum_value()
2064 		return libbpf_err(-EINVAL);  in btf__add_enum_value()
2067 		return libbpf_err(-EINVAL);  in btf__add_enum_value()
2069 	/* non-empty name */  in btf__add_enum_value()
2071 		return libbpf_err(-EINVAL);  in btf__add_enum_value()
2073 		return libbpf_err(-E2BIG);  in btf__add_enum_value()
2077 		return libbpf_err(-ENOMEM);  in btf__add_enum_value()
2082 		return libbpf_err(-ENOMEM);  in btf__add_enum_value()
2088 	v->name_off = name_off;  in btf__add_enum_value()
2089 	v->val = value;  in btf__add_enum_value()
2095 	btf->hdr->type_len += sz;  in btf__add_enum_value()
2096 	btf->hdr->str_off += sz;  in btf__add_enum_value()
2102  *   - *name*, non-empty/non-NULL name;
2103  *   - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
2106  *   - >0, type ID of newly added BTF type;
2107  *   - <0, on error.
2112 		return libbpf_err(-EINVAL);  in btf__add_fwd()
2124 		t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION);  in btf__add_fwd()
2129 		 * values; we also assume a standard 4-byte size for it  in btf__add_fwd()
2133 		return libbpf_err(-EINVAL);  in btf__add_fwd()
2139  *   - *name*, non-empty/non-NULL name;
2140  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2142  *   - >0, type ID of newly added BTF type;
2143  *   - <0, on error.
2148 		return libbpf_err(-EINVAL);  in btf__add_typedef()
2155  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2157  *   - >0, type ID of newly added BTF type;
2158  *   - <0, on error.
2167  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2169  *   - >0, type ID of newly added BTF type;
2170  *   - <0, on error.
2179  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2181  *   - >0, type ID of newly added BTF type;
2182  *   - <0, on error.
2191  *   - *name*, non-empty/non-NULL name;
2192  *   - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2194  *   - >0, type ID of newly added BTF type;
2195  *   - <0, on error.
2203 		return libbpf_err(-EINVAL);  in btf__add_func()
2206 		return libbpf_err(-EINVAL);  in btf__add_func()
2212 		t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0);  in btf__add_func()
2219  *   - *ret_type_id* - type ID for return result of a function.
2226  *   - >0, type ID of newly added BTF type;
2227  *   - <0, on error.
2235 		return libbpf_err(-EINVAL);  in btf__add_func_proto()
2238 		return libbpf_err(-ENOMEM);  in btf__add_func_proto()
2243 		return libbpf_err(-ENOMEM);  in btf__add_func_proto()
2248 	t->name_off = 0;  in btf__add_func_proto()
2249 	t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);  in btf__add_func_proto()
2250 	t->type = ret_type_id;  in btf__add_func_proto()
2257  *   - *name* - parameter name, can be NULL or empty;
2258  *   - *type_id* - type ID describing the type of the parameter.
2260  *   -  0, on success;
2261  *   - <0, on error.
2270 		return libbpf_err(-EINVAL);  in btf__add_func_param()
2273 	if (btf->nr_types == 0)  in btf__add_func_param()
2274 		return libbpf_err(-EINVAL);  in btf__add_func_param()
2277 		return libbpf_err(-EINVAL);  in btf__add_func_param()
2281 		return libbpf_err(-ENOMEM);  in btf__add_func_param()
2286 		return libbpf_err(-ENOMEM);  in btf__add_func_param()
2294 	p->name_off = name_off;  in btf__add_func_param()
2295 	p->type = type_id;  in btf__add_func_param()
2301 	btf->hdr->type_len += sz;  in btf__add_func_param()
2302 	btf->hdr->str_off += sz;  in btf__add_func_param()
2308  *   - *name* - non-empty/non-NULL name;
2309  *   - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2311  *   - *type_id* - type ID of the type describing the type of the variable.
2313  *   - >0, type ID of newly added BTF type;
2314  *   - <0, on error.
2322 	/* non-empty name */  in btf__add_var()
2324 		return libbpf_err(-EINVAL);  in btf__add_var()
2327 		return libbpf_err(-EINVAL);  in btf__add_var()
2329 		return libbpf_err(-EINVAL);  in btf__add_var()
2333 		return libbpf_err(-ENOMEM);  in btf__add_var()
2338 		return libbpf_err(-ENOMEM);  in btf__add_var()
2344 	t->name_off = name_off;  in btf__add_var()
2345 	t->info = btf_type_info(BTF_KIND_VAR, 0, 0);  in btf__add_var()
2346 	t->type = type_id;  in btf__add_var()
2349 	v->linkage = linkage;  in btf__add_var()
2356  *   - *name* - non-empty/non-NULL name;
2357  *   - *byte_sz* - data section size, in bytes.
2363  *   - >0, type ID of newly added BTF type;
2364  *   - <0, on error.
2371 	/* non-empty name */  in btf__add_datasec()
2373 		return libbpf_err(-EINVAL);  in btf__add_datasec()
2376 		return libbpf_err(-ENOMEM);  in btf__add_datasec()
2381 		return libbpf_err(-ENOMEM);  in btf__add_datasec()
2388 	t->name_off = name_off;  in btf__add_datasec()
2389 	t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);  in btf__add_datasec()
2390 	t->size = byte_sz;  in btf__add_datasec()
2397  *   - *var_type_id* - type ID, describing type of the variable;
2398  *   - *offset* - variable offset within data section, in bytes;
2399  *   - *byte_sz* - variable size, in bytes.
2402  *   -  0, on success;
2403  *   - <0, on error.
2412 	if (btf->nr_types == 0)  in btf__add_datasec_var_info()
2413 		return libbpf_err(-EINVAL);  in btf__add_datasec_var_info()
2416 		return libbpf_err(-EINVAL);  in btf__add_datasec_var_info()
2419 		return libbpf_err(-EINVAL);  in btf__add_datasec_var_info()
2423 		return libbpf_err(-ENOMEM);  in btf__add_datasec_var_info()
2428 		return libbpf_err(-ENOMEM);  in btf__add_datasec_var_info()
2430 	v->type = var_type_id;  in btf__add_datasec_var_info()
2431 	v->offset = offset;  in btf__add_datasec_var_info()
2432 	v->size = byte_sz;  in btf__add_datasec_var_info()
2438 	btf->hdr->type_len += sz;  in btf__add_datasec_var_info()
2439 	btf->hdr->str_off += sz;  in btf__add_datasec_var_info()
2460 	if (ext_sec->len == 0)  in btf_ext_setup_info()
2463 	if (ext_sec->off & 0x03) {  in btf_ext_setup_info()
2465 		     ext_sec->desc);  in btf_ext_setup_info()
2466 		return -EINVAL;  in btf_ext_setup_info()
2469 	info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;  in btf_ext_setup_info()
2470 	info_left = ext_sec->len;  in btf_ext_setup_info()
2472 	if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) {  in btf_ext_setup_info()
2474 			 ext_sec->desc, ext_sec->off, ext_sec->len);  in btf_ext_setup_info()
2475 		return -EINVAL;  in btf_ext_setup_info()
2480 		pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc);  in btf_ext_setup_info()
2481 		return -EINVAL;  in btf_ext_setup_info()
2486 	if (record_size < ext_sec->min_rec_size ||  in btf_ext_setup_info()
2489 			 ext_sec->desc, record_size);  in btf_ext_setup_info()
2490 		return -EINVAL;  in btf_ext_setup_info()
2494 	info_left -= sizeof(__u32);  in btf_ext_setup_info()
2498 		pr_debug("%s section in .BTF.ext has no records", ext_sec->desc);  in btf_ext_setup_info()
2499 		return -EINVAL;  in btf_ext_setup_info()
2509 			     ext_sec->desc);  in btf_ext_setup_info()
2510 			return -EINVAL;  in btf_ext_setup_info()
2513 		num_records = sinfo->num_info;  in btf_ext_setup_info()
2516 			     ext_sec->desc);  in btf_ext_setup_info()
2517 			return -EINVAL;  in btf_ext_setup_info()
2524 			     ext_sec->desc);  in btf_ext_setup_info()
2525 			return -EINVAL;  in btf_ext_setup_info()
2528 		info_left -= total_record_size;  in btf_ext_setup_info()
2532 	ext_info = ext_sec->ext_info;  in btf_ext_setup_info()
2533 	ext_info->len = ext_sec->len - sizeof(__u32);  in btf_ext_setup_info()
2534 	ext_info->rec_size = record_size;  in btf_ext_setup_info()
2535 	ext_info->info = info + sizeof(__u32);  in btf_ext_setup_info()
2543 		.off = btf_ext->hdr->func_info_off,  in btf_ext_setup_func_info()
2544 		.len = btf_ext->hdr->func_info_len,  in btf_ext_setup_func_info()
2546 		.ext_info = &btf_ext->func_info,  in btf_ext_setup_func_info()
2556 		.off = btf_ext->hdr->line_info_off,  in btf_ext_setup_line_info()
2557 		.len = btf_ext->hdr->line_info_len,  in btf_ext_setup_line_info()
2559 		.ext_info = &btf_ext->line_info,  in btf_ext_setup_line_info()
2569 		.off = btf_ext->hdr->core_relo_off,  in btf_ext_setup_core_relos()
2570 		.len = btf_ext->hdr->core_relo_len,  in btf_ext_setup_core_relos()
2572 		.ext_info = &btf_ext->core_relo_info,  in btf_ext_setup_core_relos()
2584 	    data_size < hdr->hdr_len) {  in btf_ext_parse_hdr()
2586 		return -EINVAL;  in btf_ext_parse_hdr()
2589 	if (hdr->magic == bswap_16(BTF_MAGIC)) {  in btf_ext_parse_hdr()
2590 		pr_warn("BTF.ext in non-native endianness is not supported\n");  in btf_ext_parse_hdr()
2591 		return -ENOTSUP;  in btf_ext_parse_hdr()
2592 	} else if (hdr->magic != BTF_MAGIC) {  in btf_ext_parse_hdr()
2593 		pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);  in btf_ext_parse_hdr()
2594 		return -EINVAL;  in btf_ext_parse_hdr()
2597 	if (hdr->version != BTF_VERSION) {  in btf_ext_parse_hdr()
2598 		pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);  in btf_ext_parse_hdr()
2599 		return -ENOTSUP;  in btf_ext_parse_hdr()
2602 	if (hdr->flags) {  in btf_ext_parse_hdr()
2603 		pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags);  in btf_ext_parse_hdr()
2604 		return -ENOTSUP;  in btf_ext_parse_hdr()
2607 	if (data_size == hdr->hdr_len) {  in btf_ext_parse_hdr()
2609 		return -EINVAL;  in btf_ext_parse_hdr()
2619 	free(btf_ext->data);  in btf_ext__free()
2634 		return libbpf_err_ptr(-ENOMEM);  in btf_ext__new()
2636 	btf_ext->data_size = size;  in btf_ext__new()
2637 	btf_ext->data = malloc(size);  in btf_ext__new()
2638 	if (!btf_ext->data) {  in btf_ext__new()
2639 		err = -ENOMEM;  in btf_ext__new()
2642 	memcpy(btf_ext->data, data, size);  in btf_ext__new()
2644 	if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {  in btf_ext__new()
2645 		err = -EINVAL;  in btf_ext__new()
2657 	if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) {  in btf_ext__new()
2658 		err = -EINVAL;  in btf_ext__new()
2677 	*size = btf_ext->data_size;  in btf_ext__get_raw_data()
2678 	return btf_ext->data;  in btf_ext__get_raw_data()
2693 	record_size = ext_info->rec_size;  in btf_ext_reloc_info()
2694 	sinfo = ext_info->info;  in btf_ext_reloc_info()
2695 	remain_len = ext_info->len;  in btf_ext_reloc_info()
2697 		records_len = sinfo->num_info * record_size;  in btf_ext_reloc_info()
2698 		info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);  in btf_ext_reloc_info()
2700 			remain_len -= sec_hdrlen + records_len;  in btf_ext_reloc_info()
2708 			return libbpf_err(-ENOMEM);  in btf_ext_reloc_info()
2710 		memcpy(data + existing_len, sinfo->data, records_len);  in btf_ext_reloc_info()
2714 		for (i = 0; i < sinfo->num_info; i++) {  in btf_ext_reloc_info()
2721 		*cnt += sinfo->num_info;  in btf_ext_reloc_info()
2725 	return libbpf_err(-ENOENT);  in btf_ext_reloc_info()
2733 	return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,  in btf_ext__reloc_func_info()
2742 	return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,  in btf_ext__reloc_line_info()
2748 	return btf_ext->func_info.rec_size;  in btf_ext__func_info_rec_size()
2753 	return btf_ext->line_info.rec_size;  in btf_ext__line_info_rec_size()
2774  * memory in-place with deduplicated types and strings without any loss of
2784  * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
2812  *	struct A* self;
2826  *	struct B* self;
2841  * single CU with complete type information describing structs `S`, `A`, and
2857  *	struct A* self;
2862  *	struct B* self;
2891  * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
2895  * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
2914 		return libbpf_err(-EINVAL);  in btf__dedup()
2918 		return libbpf_err(-ENOMEM);  in btf__dedup()
2961 #define BTF_UNPROCESSED_ID ((__u32)-1)
2962 #define BTF_IN_PROGRESS_ID ((__u32)-2)
2965 	/* .BTF section to be deduped in-place */
3006 	hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
3010 	return hashmap__append(d->dedup_table,  in btf_dedup_table_add()
3017 	if (d->hypot_cnt == d->hypot_cap) {  in btf_dedup_hypot_map_add()
3020 		d->hypot_cap += max((size_t)16, d->hypot_cap / 2);  in btf_dedup_hypot_map_add()
3021 		new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32));  in btf_dedup_hypot_map_add()
3023 			return -ENOMEM;  in btf_dedup_hypot_map_add()
3024 		d->hypot_list = new_list;  in btf_dedup_hypot_map_add()
3026 	d->hypot_list[d->hypot_cnt++] = from_id;  in btf_dedup_hypot_map_add()
3027 	d->hypot_map[from_id] = to_id;  in btf_dedup_hypot_map_add()
3035 	for (i = 0; i < d->hypot_cnt; i++)  in btf_dedup_clear_hypot_map()
3036 		d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;  in btf_dedup_clear_hypot_map()
3037 	d->hypot_cnt = 0;  in btf_dedup_clear_hypot_map()
3038 	d->hypot_adjust_canon = false;  in btf_dedup_clear_hypot_map()
3043 	hashmap__free(d->dedup_table);  in btf_dedup_free()
3044 	d->dedup_table = NULL;  in btf_dedup_free()
3046 	free(d->map);  in btf_dedup_free()
3047 	d->map = NULL;  in btf_dedup_free()
3049 	free(d->hypot_map);  in btf_dedup_free()
3050 	d->hypot_map = NULL;  in btf_dedup_free()
3052 	free(d->hypot_list);  in btf_dedup_free()
3053 	d->hypot_list = NULL;  in btf_dedup_free()
3081 		return ERR_PTR(-ENOMEM);  in btf_dedup_new()
3083 	d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds;  in btf_dedup_new()
3085 	if (opts && opts->dedup_table_size == 1)  in btf_dedup_new()
3088 	d->btf = btf;  in btf_dedup_new()
3089 	d->btf_ext = btf_ext;  in btf_dedup_new()
3091 	d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL);  in btf_dedup_new()
3092 	if (IS_ERR(d->dedup_table)) {  in btf_dedup_new()
3093 		err = PTR_ERR(d->dedup_table);  in btf_dedup_new()
3094 		d->dedup_table = NULL;  in btf_dedup_new()
3099 	d->map = malloc(sizeof(__u32) * type_cnt);  in btf_dedup_new()
3100 	if (!d->map) {  in btf_dedup_new()
3101 		err = -ENOMEM;  in btf_dedup_new()
3105 	d->map[0] = 0;  in btf_dedup_new()
3107 		struct btf_type *t = btf_type_by_id(d->btf, i);  in btf_dedup_new()
3109 		/* VAR and DATASEC are never deduped and are self-canonical */  in btf_dedup_new()
3111 			d->map[i] = i;  in btf_dedup_new()
3113 			d->map[i] = BTF_UNPROCESSED_ID;  in btf_dedup_new()
3116 	d->hypot_map = malloc(sizeof(__u32) * type_cnt);  in btf_dedup_new()
3117 	if (!d->hypot_map) {  in btf_dedup_new()
3118 		err = -ENOMEM;  in btf_dedup_new()
3122 		d->hypot_map[i] = BTF_UNPROCESSED_ID;  in btf_dedup_new()
3141 	for (i = 0; i < d->btf->nr_types; i++) {  in btf_for_each_str_off()
3142 		struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);  in btf_for_each_str_off()
3149 	if (!d->btf_ext)  in btf_for_each_str_off()
3152 	r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx);  in btf_for_each_str_off()
3167 	if (str_off == 0 || str_off < d->btf->start_str_off)  in strs_dedup_remap_str_off()
3170 	s = btf__str_by_offset(d->btf, str_off);  in strs_dedup_remap_str_off()
3171 	if (d->btf->base_btf) {  in strs_dedup_remap_str_off()
3172 		err = btf__find_str(d->btf->base_btf, s);  in strs_dedup_remap_str_off()
3177 		if (err != -ENOENT)  in strs_dedup_remap_str_off()
3181 	off = strset__add_str(d->strs_set, s);  in strs_dedup_remap_str_off()
3185 	*str_off_ptr = d->btf->start_str_off + off;  in strs_dedup_remap_str_off()
3204 	if (d->btf->strs_deduped)  in btf_dedup_strings()
3207 	d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0);  in btf_dedup_strings()
3208 	if (IS_ERR(d->strs_set)) {  in btf_dedup_strings()
3209 		err = PTR_ERR(d->strs_set);  in btf_dedup_strings()
3213 	if (!d->btf->base_btf) {  in btf_dedup_strings()
3217 		err = strset__add_str(d->strs_set, "");  in btf_dedup_strings()
3228 	strset__free(d->btf->strs_set);  in btf_dedup_strings()
3229 	d->btf->hdr->str_len = strset__data_size(d->strs_set);  in btf_dedup_strings()
3230 	d->btf->strs_set = d->strs_set;  in btf_dedup_strings()
3231 	d->strs_set = NULL;  in btf_dedup_strings()
3232 	d->btf->strs_deduped = true;  in btf_dedup_strings()
3236 	strset__free(d->strs_set);  in btf_dedup_strings()
3237 	d->strs_set = NULL;  in btf_dedup_strings()
3246 	h = hash_combine(0, t->name_off);  in btf_hash_common()
3247 	h = hash_combine(h, t->info);  in btf_hash_common()
3248 	h = hash_combine(h, t->size);  in btf_hash_common()
3254 	return t1->name_off == t2->name_off &&  in btf_equal_common()
3255 	       t1->info == t2->info &&  in btf_equal_common()
3256 	       t1->size == t2->size;  in btf_equal_common()
3288 	h = hash_combine(0, t->name_off);  in btf_hash_enum()
3289 	h = hash_combine(h, t->info & ~0xffff);  in btf_hash_enum()
3290 	h = hash_combine(h, t->size);  in btf_hash_enum()
3308 		if (m1->name_off != m2->name_off || m1->val != m2->val)  in btf_equal_enum()
3326 	return t1->name_off == t2->name_off &&  in btf_compat_enum()
3327 	       (t1->info & ~0xffff) == (t2->info & ~0xffff) &&  in btf_compat_enum()
3328 	       t1->size == t2->size;  in btf_compat_enum()
3344 		h = hash_combine(h, member->name_off);  in btf_hash_struct()
3345 		h = hash_combine(h, member->offset);  in btf_hash_struct()
3370 		if (m1->name_off != m2->name_off || m1->offset != m2->offset)  in btf_shallow_equal_struct()
3388 	h = hash_combine(h, info->type);  in btf_hash_array()
3389 	h = hash_combine(h, info->index_type);  in btf_hash_array()
3390 	h = hash_combine(h, info->nelems);  in btf_hash_array()
3410 	return info1->type == info2->type &&  in btf_equal_array()
3411 	       info1->index_type == info2->index_type &&  in btf_equal_array()
3412 	       info1->nelems == info2->nelems;  in btf_equal_array()
3425 	return btf_array(t1)->nelems == btf_array(t2)->nelems;  in btf_compat_array()
3441 		h = hash_combine(h, member->name_off);  in btf_hash_fnproto()
3442 		h = hash_combine(h, member->type);  in btf_hash_fnproto()
3468 		if (m1->name_off != m2->name_off || m1->type != m2->type)  in btf_equal_fnproto()
3488 	if (t1->name_off != t2->name_off || t1->info != t2->info)  in btf_compat_fnproto()
3495 		if (m1->name_off != m2->name_off)  in btf_compat_fnproto()
3513 	if (!d->btf->base_btf)  in btf_dedup_prep()
3516 	for (type_id = 1; type_id < d->btf->start_id; type_id++) {  in btf_dedup_prep()
3517 		t = btf_type_by_id(d->btf, type_id);  in btf_dedup_prep()
3519 		/* all base BTF types are self-canonical by definition */  in btf_dedup_prep()
3520 		d->map[type_id] = type_id;  in btf_dedup_prep()
3555 			return -EINVAL;  in btf_dedup_prep()
3558 			return -ENOMEM;  in btf_dedup_prep()
3568  * canonical and is added into `btf_dedup->dedup_table` as another candidate.
3572 	struct btf_type *t = btf_type_by_id(d->btf, type_id);  in btf_dedup_prim_type()
3598 			cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_prim_type()
3599 			cand = btf_type_by_id(d->btf, cand_id);  in btf_dedup_prim_type()
3610 			cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_prim_type()
3611 			cand = btf_type_by_id(d->btf, cand_id);  in btf_dedup_prim_type()
3616 			if (d->opts.dont_resolve_fwds)  in btf_dedup_prim_type()
3625 				d->map[cand_id] = type_id;  in btf_dedup_prim_type()
3634 			cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_prim_type()
3635 			cand = btf_type_by_id(d->btf, cand_id);  in btf_dedup_prim_type()
3644 		return -EINVAL;  in btf_dedup_prim_type()
3647 	d->map[type_id] = new_id;  in btf_dedup_prim_type()
3649 		return -ENOMEM;  in btf_dedup_prim_type()
3658 	for (i = 0; i < d->btf->nr_types; i++) {  in btf_dedup_prim_types()
3659 		err = btf_dedup_prim_type(d, d->btf->start_id + i);  in btf_dedup_prim_types()
3671 	return d->map[type_id] <= BTF_MAX_NR_TYPES;  in is_type_mapped()
3681 	while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)  in resolve_type_id()
3682 		type_id = d->map[type_id];  in resolve_type_id()
3694 	if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))  in resolve_fwd_id()
3697 	while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)  in resolve_fwd_id()
3698 		type_id = d->map[type_id];  in resolve_fwd_id()
3700 	if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))  in resolve_fwd_id()
3717 	t1 = btf_type_by_id(d->btf, id1);  in btf_dedup_identical_arrays()
3718 	t2 = btf_type_by_id(d->btf, id2);  in btf_dedup_identical_arrays()
3733  *  - >0, if type graphs are equivalent;
3734  *  -  0, if not equivalent;
3735  *  - <0, on error.
3737  * Algorithm performs side-by-side DFS traversal of both type graphs and checks
3742  * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
3749  *   - `canon_id` is mapped to exactly the same type as `cand_id`. This will
3752  *   - `canon_id` is mapped to different type. This is contradiction in our
3766  * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
3791  * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
3795  *   - Both types in canonical and candidate graphs are FWDs. If they are
3799  *   - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
3801  *   - Type in canonical graph is FWD, while type in candidate is concrete
3810  *   - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
3815  *     turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
3834 	hypot_type_id = d->hypot_map[canon_id];  in btf_dedup_is_equiv()
3840 		 * types sub-graph has a consistent and deduped-by-compiler  in btf_dedup_is_equiv()
3849 		return -ENOMEM;  in btf_dedup_is_equiv()
3851 	cand_type = btf_type_by_id(d->btf, cand_id);  in btf_dedup_is_equiv()
3852 	canon_type = btf_type_by_id(d->btf, canon_id);  in btf_dedup_is_equiv()
3856 	if (cand_type->name_off != canon_type->name_off)  in btf_dedup_is_equiv()
3859 	/* FWD <--> STRUCT/UNION equivalence check, if enabled */  in btf_dedup_is_equiv()
3860 	if (!d->opts.dont_resolve_fwds  in btf_dedup_is_equiv()
3873 			if (fwd_kind == real_kind && canon_id < d->btf->start_id)  in btf_dedup_is_equiv()
3874 				d->hypot_adjust_canon = true;  in btf_dedup_is_equiv()
3887 		if (d->opts.dont_resolve_fwds)  in btf_dedup_is_equiv()
3902 		if (cand_type->info != canon_type->info)  in btf_dedup_is_equiv()
3904 		return btf_dedup_is_equiv(d, cand_type->type, canon_type->type);  in btf_dedup_is_equiv()
3913 		eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);  in btf_dedup_is_equiv()
3916 		return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);  in btf_dedup_is_equiv()
3930 			eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);  in btf_dedup_is_equiv()
3946 		eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type);  in btf_dedup_is_equiv()
3953 			eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type);  in btf_dedup_is_equiv()
3963 		return -EINVAL;  in btf_dedup_is_equiv()
3973  * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
4001 	for (i = 0; i < d->hypot_cnt; i++) {  in btf_dedup_merge_hypot_map()
4002 		canon_type_id = d->hypot_list[i];  in btf_dedup_merge_hypot_map()
4003 		targ_type_id = d->hypot_map[canon_type_id];  in btf_dedup_merge_hypot_map()
4006 		t_kind = btf_kind(btf__type_by_id(d->btf, t_id));  in btf_dedup_merge_hypot_map()
4007 		c_kind = btf_kind(btf__type_by_id(d->btf, c_id));  in btf_dedup_merge_hypot_map()
4012 		 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because  in btf_dedup_merge_hypot_map()
4029 			d->map[c_id] = t_id;  in btf_dedup_merge_hypot_map()
4036 		if (d->hypot_adjust_canon)  in btf_dedup_merge_hypot_map()
4040 			d->map[t_id] = c_id;  in btf_dedup_merge_hypot_map()
4052 			d->map[t_id] = c_id;  in btf_dedup_merge_hypot_map()
4069  * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4070  * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4076  * as canonical for itself and is added into btf_dedup->dedup_table hash map
4089 	if (d->map[type_id] <= BTF_MAX_NR_TYPES)  in btf_dedup_struct_type()
4092 	t = btf_type_by_id(d->btf, type_id);  in btf_dedup_struct_type()
4100 		__u32 cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_struct_type()
4110 		 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because  in btf_dedup_struct_type()
4113 		cand_type = btf_type_by_id(d->btf, cand_id);  in btf_dedup_struct_type()
4124 		if (d->hypot_adjust_canon) /* not really equivalent */  in btf_dedup_struct_type()
4130 	d->map[type_id] = new_id;  in btf_dedup_struct_type()
4132 		return -ENOMEM;  in btf_dedup_struct_type()
4141 	for (i = 0; i < d->btf->nr_types; i++) {  in btf_dedup_struct_types()
4142 		err = btf_dedup_struct_type(d, d->btf->start_id + i);  in btf_dedup_struct_types()
4171  * btf_dedup->dedup_table as another possible canonical representative.
4182 	if (d->map[type_id] == BTF_IN_PROGRESS_ID)  in btf_dedup_ref_type()
4183 		return -ELOOP;  in btf_dedup_ref_type()
4184 	if (d->map[type_id] <= BTF_MAX_NR_TYPES)  in btf_dedup_ref_type()
4187 	t = btf_type_by_id(d->btf, type_id);  in btf_dedup_ref_type()
4188 	d->map[type_id] = BTF_IN_PROGRESS_ID;  in btf_dedup_ref_type()
4197 		ref_type_id = btf_dedup_ref_type(d, t->type);  in btf_dedup_ref_type()
4200 		t->type = ref_type_id;  in btf_dedup_ref_type()
4204 			cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_ref_type()
4205 			cand = btf_type_by_id(d->btf, cand_id);  in btf_dedup_ref_type()
4216 		ref_type_id = btf_dedup_ref_type(d, info->type);  in btf_dedup_ref_type()
4219 		info->type = ref_type_id;  in btf_dedup_ref_type()
4221 		ref_type_id = btf_dedup_ref_type(d, info->index_type);  in btf_dedup_ref_type()
4224 		info->index_type = ref_type_id;  in btf_dedup_ref_type()
4228 			cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_ref_type()
4229 			cand = btf_type_by_id(d->btf, cand_id);  in btf_dedup_ref_type()
4243 		ref_type_id = btf_dedup_ref_type(d, t->type);  in btf_dedup_ref_type()
4246 		t->type = ref_type_id;  in btf_dedup_ref_type()
4251 			ref_type_id = btf_dedup_ref_type(d, param->type);  in btf_dedup_ref_type()
4254 			param->type = ref_type_id;  in btf_dedup_ref_type()
4260 			cand_id = (__u32)(long)hash_entry->value;  in btf_dedup_ref_type()
4261 			cand = btf_type_by_id(d->btf, cand_id);  in btf_dedup_ref_type()
4271 		return -EINVAL;  in btf_dedup_ref_type()
4274 	d->map[type_id] = new_id;  in btf_dedup_ref_type()
4276 		return -ENOMEM;  in btf_dedup_ref_type()
4285 	for (i = 0; i < d->btf->nr_types; i++) {  in btf_dedup_ref_types()
4286 		err = btf_dedup_ref_type(d, d->btf->start_id + i);  in btf_dedup_ref_types()
4290 	/* we won't need d->dedup_table anymore */  in btf_dedup_ref_types()
4291 	hashmap__free(d->dedup_table);  in btf_dedup_ref_types()
4292 	d->dedup_table = NULL;  in btf_dedup_ref_types()
4302  * duplicates. During compaction btf_dedup->hypot_map array is reused to store
4310 	__u32 next_type_id = d->btf->start_id;  in btf_dedup_compact_types()
4316 	d->hypot_map[0] = 0;  in btf_dedup_compact_types()
4318 	for (id = 1; id < d->btf->start_id; id++)  in btf_dedup_compact_types()
4319 		d->hypot_map[id] = id;  in btf_dedup_compact_types()
4320 	for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)  in btf_dedup_compact_types()
4321 		d->hypot_map[id] = BTF_UNPROCESSED_ID;  in btf_dedup_compact_types()
4323 	p = d->btf->types_data;  in btf_dedup_compact_types()
4325 	for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {  in btf_dedup_compact_types()
4326 		if (d->map[id] != id)  in btf_dedup_compact_types()
4329 		t = btf__type_by_id(d->btf, id);  in btf_dedup_compact_types()
4335 		d->hypot_map[id] = next_type_id;  in btf_dedup_compact_types()
4336 		d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;  in btf_dedup_compact_types()
4342 	d->btf->nr_types = next_type_id - d->btf->start_id;  in btf_dedup_compact_types()
4343 	d->btf->type_offs_cap = d->btf->nr_types;  in btf_dedup_compact_types()
4344 	d->btf->hdr->type_len = p - d->btf->types_data;  in btf_dedup_compact_types()
4345 	new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,  in btf_dedup_compact_types()
4347 	if (d->btf->type_offs_cap && !new_offs)  in btf_dedup_compact_types()
4348 		return -ENOMEM;  in btf_dedup_compact_types()
4349 	d->btf->type_offs = new_offs;  in btf_dedup_compact_types()
4350 	d->btf->hdr->str_off = d->btf->hdr->type_len;  in btf_dedup_compact_types()
4351 	d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len;  in btf_dedup_compact_types()
4358  * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
4367 	new_type_id = d->hypot_map[resolved_type_id];  in btf_dedup_remap_type_id()
4369 		return -EINVAL;  in btf_dedup_remap_type_id()
4380  * deduped type ID is stored in btf_dedup->hypot_map and is populated during
4389 	for (i = 0; i < d->btf->nr_types; i++) {  in btf_dedup_remap_types()
4390 		struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);  in btf_dedup_remap_types()
4397 	if (!d->btf_ext)  in btf_dedup_remap_types()
4400 	r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d);  in btf_dedup_remap_types()
4408  * Probe few well-known locations for vmlinux kernel image and try to load BTF
4420 		{ "/boot/vmlinux-%1$s" },  in btf__load_vmlinux_btf()
4421 		{ "/lib/modules/%1$s/vmlinux-%1$s" },  in btf__load_vmlinux_btf()
4424 		{ "/usr/lib/debug/boot/vmlinux-%1$s" },  in btf__load_vmlinux_btf()
4425 		{ "/usr/lib/debug/boot/vmlinux-%1$s.debug" },  in btf__load_vmlinux_btf()
4454 	return libbpf_err_ptr(-ESRCH);  in btf__load_vmlinux_btf()
4485 		return visit(&t->type, ctx);  in btf_type_visit_type_ids()
4490 		err = visit(&a->type, ctx);  in btf_type_visit_type_ids()
4491 		err = err ?: visit(&a->index_type, ctx);  in btf_type_visit_type_ids()
4500 			err = visit(&m->type, ctx);  in btf_type_visit_type_ids()
4510 		err = visit(&t->type, ctx);  in btf_type_visit_type_ids()
4514 			err = visit(&m->type, ctx);  in btf_type_visit_type_ids()
4525 			err = visit(&m->type, ctx);  in btf_type_visit_type_ids()
4533 		return -EINVAL;  in btf_type_visit_type_ids()
4541 	err = visit(&t->name_off, ctx);  in btf_type_visit_str_offs()
4551 			err = visit(&m->name_off, ctx);  in btf_type_visit_str_offs()
4561 			err = visit(&m->name_off, ctx);  in btf_type_visit_str_offs()
4571 			err = visit(&m->name_off, ctx);  in btf_type_visit_str_offs()
4590 	seg = &btf_ext->func_info;  in btf_ext_visit_type_ids()
4595 			err = visit(&rec->type_id, ctx);  in btf_ext_visit_type_ids()
4601 	seg = &btf_ext->core_relo_info;  in btf_ext_visit_type_ids()
4606 			err = visit(&rec->type_id, ctx);  in btf_ext_visit_type_ids()
4621 	seg = &btf_ext->func_info;  in btf_ext_visit_str_offs()
4623 		err = visit(&sec->sec_name_off, ctx);  in btf_ext_visit_str_offs()
4628 	seg = &btf_ext->line_info;  in btf_ext_visit_str_offs()
4632 		err = visit(&sec->sec_name_off, ctx);  in btf_ext_visit_str_offs()
4637 			err = visit(&rec->file_name_off, ctx);  in btf_ext_visit_str_offs()
4640 			err = visit(&rec->line_off, ctx);  in btf_ext_visit_str_offs()
4646 	seg = &btf_ext->core_relo_info;  in btf_ext_visit_str_offs()
4650 		err = visit(&sec->sec_name_off, ctx);  in btf_ext_visit_str_offs()
4655 			err = visit(&rec->access_str_off, ctx);  in btf_ext_visit_str_offs()