201 			     u32 func_id, u16 offset, unsigned long *addr);
276 	int func_id;  member
288 	u32 func_id;  member
508 static bool is_acquire_function(enum bpf_func_id func_id,  in is_acquire_function()  argument
513 	if (func_id == BPF_FUNC_sk_lookup_tcp ||  in is_acquire_function()
514 	    func_id == BPF_FUNC_sk_lookup_udp ||  in is_acquire_function()
515 	    func_id == BPF_FUNC_skc_lookup_tcp ||  in is_acquire_function()
516 	    func_id == BPF_FUNC_ringbuf_reserve ||  in is_acquire_function()
517 	    func_id == BPF_FUNC_kptr_xchg)  in is_acquire_function()
520 	if (func_id == BPF_FUNC_map_lookup_elem &&  in is_acquire_function()
528 static bool is_ptr_cast_function(enum bpf_func_id func_id)  in is_ptr_cast_function()  argument
530 	return func_id == BPF_FUNC_tcp_sock ||  in is_ptr_cast_function()
531 		func_id == BPF_FUNC_sk_fullsock ||  in is_ptr_cast_function()
532 		func_id == BPF_FUNC_skc_to_tcp_sock ||  in is_ptr_cast_function()
533 		func_id == BPF_FUNC_skc_to_tcp6_sock ||  in is_ptr_cast_function()
534 		func_id == BPF_FUNC_skc_to_udp6_sock ||  in is_ptr_cast_function()
535 		func_id == BPF_FUNC_skc_to_mptcp_sock ||  in is_ptr_cast_function()
536 		func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||  in is_ptr_cast_function()
537 		func_id == BPF_FUNC_skc_to_tcp_request_sock;  in is_ptr_cast_function()
540 static bool is_dynptr_ref_function(enum bpf_func_id func_id)  in is_dynptr_ref_function()  argument
542 	return func_id == BPF_FUNC_dynptr_data;  in is_dynptr_ref_function()
547 static bool is_callback_calling_function(enum bpf_func_id func_id)  in is_callback_calling_function()  argument
549 	return func_id == BPF_FUNC_for_each_map_elem ||  in is_callback_calling_function()
550 	       func_id == BPF_FUNC_timer_set_callback ||  in is_callback_calling_function()
551 	       func_id == BPF_FUNC_find_vma ||  in is_callback_calling_function()
552 	       func_id == BPF_FUNC_loop ||  in is_callback_calling_function()
553 	       func_id == BPF_FUNC_user_ringbuf_drain;  in is_callback_calling_function()
556 static bool is_async_callback_calling_function(enum bpf_func_id func_id)  in is_async_callback_calling_function()  argument
558 	return func_id == BPF_FUNC_timer_set_callback;  in is_async_callback_calling_function()
561 static bool is_storage_get_function(enum bpf_func_id func_id)  in is_storage_get_function()  argument
563 	return func_id == BPF_FUNC_sk_storage_get ||  in is_storage_get_function()
564 	       func_id == BPF_FUNC_inode_storage_get ||  in is_storage_get_function()
565 	       func_id == BPF_FUNC_task_storage_get ||  in is_storage_get_function()
566 	       func_id == BPF_FUNC_cgrp_storage_get;  in is_storage_get_function()
569 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,  in helper_multiple_ref_obj_use()  argument
574 	if (is_ptr_cast_function(func_id))  in helper_multiple_ref_obj_use()
576 	if (is_acquire_function(func_id, map))  in helper_multiple_ref_obj_use()
578 	if (is_dynptr_ref_function(func_id))  in helper_multiple_ref_obj_use()
2462 	u32 func_id;  member
2495 	return d0->func_id - d1->func_id ?: d0->offset - d1->offset;  in kfunc_desc_cmp_by_id_off()
2507 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)  in find_kfunc_desc()  argument
2510 		.func_id = func_id,  in find_kfunc_desc()
2520 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,  in bpf_get_kfunc_addr()  argument
2525 	desc = find_kfunc_desc(prog, func_id, btf_fd_idx);  in bpf_get_kfunc_addr()
2619 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)  in add_kfunc_call()  argument
2668 	if (!func_id && !offset)  in add_kfunc_call()
2684 	if (find_kfunc_desc(env->prog, func_id, offset))  in add_kfunc_call()
2692 	func = btf_type_by_id(desc_btf, func_id);  in add_kfunc_call()
2695 			func_id);  in add_kfunc_call()
2701 			func_id);  in add_kfunc_call()
2712 	specialize_kfunc(env, func_id, offset, &addr);  in add_kfunc_call()
2715 		call_imm = func_id;  in add_kfunc_call()
2726 	if (bpf_dev_bound_kfunc_id(func_id)) {  in add_kfunc_call()
2733 	desc->func_id = func_id;  in add_kfunc_call()
7671 		if (meta->func_id == BPF_FUNC_map_peek_elem)  in resolve_map_arg_type()
7832 	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type))  in check_reg_type()
7858 				func_id_name(meta->func_id),  in check_reg_type()
7877 					 meta->func_id != BPF_FUNC_sk_release;  in check_reg_type()
7893 		if (meta->func_id == BPF_FUNC_kptr_xchg) {  in check_reg_type()
7916 		if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&  in check_reg_type()
7917 		    meta->func_id != BPF_FUNC_kptr_xchg) {  in check_reg_type()
7921 		if (meta->func_id == BPF_FUNC_kptr_xchg) {  in check_reg_type()
8267 		if (meta->func_id == BPF_FUNC_spin_lock) {  in check_func_arg()
8271 		} else if (meta->func_id == BPF_FUNC_spin_unlock) {  in check_func_arg()
8384 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)  in may_update_sockmap()  argument
8389 	if (func_id != BPF_FUNC_map_update_elem)  in may_update_sockmap()
8423 					struct bpf_map *map, int func_id)  in check_map_func_compatibility()  argument
8431 		if (func_id != BPF_FUNC_tail_call)  in check_map_func_compatibility()
8435 		if (func_id != BPF_FUNC_perf_event_read &&  in check_map_func_compatibility()
8436 		    func_id != BPF_FUNC_perf_event_output &&  in check_map_func_compatibility()
8437 		    func_id != BPF_FUNC_skb_output &&  in check_map_func_compatibility()
8438 		    func_id != BPF_FUNC_perf_event_read_value &&  in check_map_func_compatibility()
8439 		    func_id != BPF_FUNC_xdp_output)  in check_map_func_compatibility()
8443 		if (func_id != BPF_FUNC_ringbuf_output &&  in check_map_func_compatibility()
8444 		    func_id != BPF_FUNC_ringbuf_reserve &&  in check_map_func_compatibility()
8445 		    func_id != BPF_FUNC_ringbuf_query &&  in check_map_func_compatibility()
8446 		    func_id != BPF_FUNC_ringbuf_reserve_dynptr &&  in check_map_func_compatibility()
8447 		    func_id != BPF_FUNC_ringbuf_submit_dynptr &&  in check_map_func_compatibility()
8448 		    func_id != BPF_FUNC_ringbuf_discard_dynptr)  in check_map_func_compatibility()
8452 		if (func_id != BPF_FUNC_user_ringbuf_drain)  in check_map_func_compatibility()
8456 		if (func_id != BPF_FUNC_get_stackid)  in check_map_func_compatibility()
8460 		if (func_id != BPF_FUNC_skb_under_cgroup &&  in check_map_func_compatibility()
8461 		    func_id != BPF_FUNC_current_task_under_cgroup)  in check_map_func_compatibility()
8466 		if (func_id != BPF_FUNC_get_local_storage)  in check_map_func_compatibility()
8471 		if (func_id != BPF_FUNC_redirect_map &&  in check_map_func_compatibility()
8472 		    func_id != BPF_FUNC_map_lookup_elem)  in check_map_func_compatibility()
8479 		if (func_id != BPF_FUNC_redirect_map)  in check_map_func_compatibility()
8483 		if (func_id != BPF_FUNC_redirect_map &&  in check_map_func_compatibility()
8484 		    func_id != BPF_FUNC_map_lookup_elem)  in check_map_func_compatibility()
8489 		if (func_id != BPF_FUNC_map_lookup_elem)  in check_map_func_compatibility()
8493 		if (func_id != BPF_FUNC_sk_redirect_map &&  in check_map_func_compatibility()
8494 		    func_id != BPF_FUNC_sock_map_update &&  in check_map_func_compatibility()
8495 		    func_id != BPF_FUNC_map_delete_elem &&  in check_map_func_compatibility()
8496 		    func_id != BPF_FUNC_msg_redirect_map &&  in check_map_func_compatibility()
8497 		    func_id != BPF_FUNC_sk_select_reuseport &&  in check_map_func_compatibility()
8498 		    func_id != BPF_FUNC_map_lookup_elem &&  in check_map_func_compatibility()
8499 		    !may_update_sockmap(env, func_id))  in check_map_func_compatibility()
8503 		if (func_id != BPF_FUNC_sk_redirect_hash &&  in check_map_func_compatibility()
8504 		    func_id != BPF_FUNC_sock_hash_update &&  in check_map_func_compatibility()
8505 		    func_id != BPF_FUNC_map_delete_elem &&  in check_map_func_compatibility()
8506 		    func_id != BPF_FUNC_msg_redirect_hash &&  in check_map_func_compatibility()
8507 		    func_id != BPF_FUNC_sk_select_reuseport &&  in check_map_func_compatibility()
8508 		    func_id != BPF_FUNC_map_lookup_elem &&  in check_map_func_compatibility()
8509 		    !may_update_sockmap(env, func_id))  in check_map_func_compatibility()
8513 		if (func_id != BPF_FUNC_sk_select_reuseport)  in check_map_func_compatibility()
8518 		if (func_id != BPF_FUNC_map_peek_elem &&  in check_map_func_compatibility()
8519 		    func_id != BPF_FUNC_map_pop_elem &&  in check_map_func_compatibility()
8520 		    func_id != BPF_FUNC_map_push_elem)  in check_map_func_compatibility()
8524 		if (func_id != BPF_FUNC_sk_storage_get &&  in check_map_func_compatibility()
8525 		    func_id != BPF_FUNC_sk_storage_delete &&  in check_map_func_compatibility()
8526 		    func_id != BPF_FUNC_kptr_xchg)  in check_map_func_compatibility()
8530 		if (func_id != BPF_FUNC_inode_storage_get &&  in check_map_func_compatibility()
8531 		    func_id != BPF_FUNC_inode_storage_delete &&  in check_map_func_compatibility()
8532 		    func_id != BPF_FUNC_kptr_xchg)  in check_map_func_compatibility()
8536 		if (func_id != BPF_FUNC_task_storage_get &&  in check_map_func_compatibility()
8537 		    func_id != BPF_FUNC_task_storage_delete &&  in check_map_func_compatibility()
8538 		    func_id != BPF_FUNC_kptr_xchg)  in check_map_func_compatibility()
8542 		if (func_id != BPF_FUNC_cgrp_storage_get &&  in check_map_func_compatibility()
8543 		    func_id != BPF_FUNC_cgrp_storage_delete &&  in check_map_func_compatibility()
8544 		    func_id != BPF_FUNC_kptr_xchg)  in check_map_func_compatibility()
8548 		if (func_id != BPF_FUNC_map_peek_elem &&  in check_map_func_compatibility()
8549 		    func_id != BPF_FUNC_map_push_elem)  in check_map_func_compatibility()
8557 	switch (func_id) {  in check_map_func_compatibility()
8671 		map->map_type, func_id_name(func_id), func_id);  in check_map_func_compatibility()
8749 static int check_func_proto(const struct bpf_func_proto *fn, int func_id)  in check_func_proto()  argument
9299 				   int func_id,  in do_refine_retval_range()  argument
9307 	switch (func_id) {  in do_refine_retval_range()
9335 		int func_id, int insn_idx)  in record_func_map()  argument
9340 	if (func_id != BPF_FUNC_tail_call &&  in record_func_map()
9341 	    func_id != BPF_FUNC_map_lookup_elem &&  in record_func_map()
9342 	    func_id != BPF_FUNC_map_update_elem &&  in record_func_map()
9343 	    func_id != BPF_FUNC_map_delete_elem &&  in record_func_map()
9344 	    func_id != BPF_FUNC_map_push_elem &&  in record_func_map()
9345 	    func_id != BPF_FUNC_map_pop_elem &&  in record_func_map()
9346 	    func_id != BPF_FUNC_map_peek_elem &&  in record_func_map()
9347 	    func_id != BPF_FUNC_for_each_map_elem &&  in record_func_map()
9348 	    func_id != BPF_FUNC_redirect_map &&  in record_func_map()
9349 	    func_id != BPF_FUNC_map_lookup_percpu_elem)  in record_func_map()
9362 	    (func_id == BPF_FUNC_map_delete_elem ||  in record_func_map()
9363 	     func_id == BPF_FUNC_map_update_elem ||  in record_func_map()
9364 	     func_id == BPF_FUNC_map_push_elem ||  in record_func_map()
9365 	     func_id == BPF_FUNC_map_pop_elem)) {  in record_func_map()
9381 		int func_id, int insn_idx)  in record_func_key()  argument
9389 	if (func_id != BPF_FUNC_tail_call)  in record_func_key()
9476 	int func_id = BPF_FUNC_get_func_ip;  in check_get_func_ip()  local
9481 				func_id_name(func_id), func_id);  in check_get_func_ip()
9490 		func_id_name(func_id), func_id, type);  in check_get_func_ip()
9540 	int i, err, func_id;  in check_helper_call()  local
9543 	func_id = insn->imm;  in check_helper_call()
9544 	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {  in check_helper_call()
9545 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id),  in check_helper_call()
9546 			func_id);  in check_helper_call()
9551 		fn = env->ops->get_func_proto(func_id, env->prog);  in check_helper_call()
9553 		verbose(env, "unknown func %s#%d\n", func_id_name(func_id),  in check_helper_call()
9554 			func_id);  in check_helper_call()
9578 			func_id_name(func_id), func_id);  in check_helper_call()
9585 	err = check_func_proto(fn, func_id);  in check_helper_call()
9588 			func_id_name(func_id), func_id);  in check_helper_call()
9595 				func_id_name(func_id), func_id);  in check_helper_call()
9599 		if (env->prog->aux->sleepable && is_storage_get_function(func_id))  in check_helper_call()
9603 	meta.func_id = func_id;  in check_helper_call()
9611 	err = record_func_map(env, &meta, func_id, insn_idx);  in check_helper_call()
9615 	err = record_func_key(env, &meta, func_id, insn_idx);  in check_helper_call()
9653 				func_id_name(func_id), func_id);  in check_helper_call()
9658 	switch (func_id) {  in check_helper_call()
9880 		if (func_id == BPF_FUNC_kptr_xchg) {  in check_helper_call()
9889 					func_id_name(func_id));  in check_helper_call()
9897 				base_type(ret_type), func_id_name(func_id),  in check_helper_call()
9898 				func_id);  in check_helper_call()
9907 			base_type(ret_type), func_id_name(func_id), func_id);  in check_helper_call()
9914 	if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {  in check_helper_call()
9916 			func_id_name(func_id), func_id);  in check_helper_call()
9920 	if (is_dynptr_ref_function(func_id))  in check_helper_call()
9923 	if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {  in check_helper_call()
9926 	} else if (is_acquire_function(func_id, meta.map_ptr)) {  in check_helper_call()
9937 	do_refine_retval_range(regs, fn->ret_type, func_id, &meta);  in check_helper_call()
9939 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);  in check_helper_call()
9943 	if ((func_id == BPF_FUNC_get_stack ||  in check_helper_call()
9944 	     func_id == BPF_FUNC_get_task_stack) &&  in check_helper_call()
9956 			verbose(env, err_str, func_id_name(func_id), func_id);  in check_helper_call()
9963 	if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)  in check_helper_call()
9966 	if (func_id == BPF_FUNC_get_func_ip) {  in check_helper_call()
10314 	if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&  in BTF_ID()
10324 	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];  in is_kfunc_bpf_rcu_read_lock()
10329 	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];  in is_kfunc_bpf_rcu_read_unlock()
10344 	if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx])  in get_kfunc_ptr_arg_type()
10702 	if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))  in __process_kf_arg_ptr_to_graph_root()
10770 	if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))  in __process_kf_arg_ptr_to_graph_node()
10997 			if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {  in check_kfunc_args()
11014 			    meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) {  in check_kfunc_args()
11036 			if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {  in check_kfunc_args()
11038 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {  in check_kfunc_args()
11040 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&  in check_kfunc_args()
11122 			if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) {  in check_kfunc_args()
11247 	u32 func_id, *kfunc_flags;  in fetch_kfunc_meta()  local
11261 	func_id = insn->imm;  in fetch_kfunc_meta()
11262 	func = btf_type_by_id(desc_btf, func_id);  in fetch_kfunc_meta()
11268 	kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog);  in fetch_kfunc_meta()
11275 	meta->func_id = func_id;  in fetch_kfunc_meta()
11368 				func_name, meta.func_id);  in check_kfunc_call()
11373 	if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||  in check_kfunc_call()
11374 	    meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||  in check_kfunc_call()
11375 	    meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {  in check_kfunc_call()
11382 				func_name, meta.func_id);  in check_kfunc_call()
11389 				func_name, meta.func_id);  in check_kfunc_call()
11394 	if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {  in check_kfunc_call()
11399 				func_name, meta.func_id);  in check_kfunc_call()
11413 		    (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] &&  in check_kfunc_call()
11414 		     meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) {  in check_kfunc_call()
11426 		if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {  in check_kfunc_call()
11427 			if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) {  in check_kfunc_call()
11462 			} else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {  in check_kfunc_call()
11471 			} else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] ||  in check_kfunc_call()
11472 				   meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) {  in check_kfunc_call()
11476 			} else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] ||  in check_kfunc_call()
11477 				   meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {  in check_kfunc_call()
11481 			} else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {  in check_kfunc_call()
11486 			} else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {  in check_kfunc_call()
11498 			} else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||  in check_kfunc_call()
11499 				   meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {  in check_kfunc_call()
11514 				if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {  in check_kfunc_call()
11590 		} else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {  in check_kfunc_call()
11597 		if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {  in check_kfunc_call()
11598 			if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) {  in check_kfunc_call()
18190 			     u32 func_id, u16 offset, unsigned long *addr)  in specialize_kfunc()  argument
18197 	if (bpf_dev_bound_kfunc_id(func_id)) {  in specialize_kfunc()
18198 		xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);  in specialize_kfunc()
18209 	if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {  in specialize_kfunc()
18267 	if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) {  in fixup_kfunc_call()
18277 	} else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||  in fixup_kfunc_call()
18278 		   desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {  in fixup_kfunc_call()
18282 		if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&  in fixup_kfunc_call()
18293 	} else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||  in fixup_kfunc_call()
18294 		   desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||  in fixup_kfunc_call()
18295 		   desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {  in fixup_kfunc_call()
18301 		if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {  in fixup_kfunc_call()
18314 	} else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||  in fixup_kfunc_call()
18315 		   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {  in fixup_kfunc_call()