1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __BPF_HELPERS_H 3 #define __BPF_HELPERS_H 4 5 /* helper macro to place programs, maps, license in 6 * different sections in elf_bpf file. Section names 7 * are interpreted by elf_bpf loader 8 */ 9 #define SEC(NAME) __attribute__((section(NAME), used)) 10 11 /* helper functions called from eBPF programs written in C */ 12 static void *(*bpf_map_lookup_elem)(void *map, void *key) = 13 (void *) BPF_FUNC_map_lookup_elem; 14 static int (*bpf_map_update_elem)(void *map, void *key, void *value, 15 unsigned long long flags) = 16 (void *) BPF_FUNC_map_update_elem; 17 static int (*bpf_map_delete_elem)(void *map, void *key) = 18 (void *) BPF_FUNC_map_delete_elem; 19 static int (*bpf_probe_read)(void *dst, int size, void *unsafe_ptr) = 20 (void *) BPF_FUNC_probe_read; 21 static unsigned long long (*bpf_ktime_get_ns)(void) = 22 (void *) BPF_FUNC_ktime_get_ns; 23 static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = 24 (void *) BPF_FUNC_trace_printk; 25 static void (*bpf_tail_call)(void *ctx, void *map, int index) = 26 (void *) BPF_FUNC_tail_call; 27 static unsigned long long (*bpf_get_smp_processor_id)(void) = 28 (void *) BPF_FUNC_get_smp_processor_id; 29 static unsigned long long (*bpf_get_current_pid_tgid)(void) = 30 (void *) BPF_FUNC_get_current_pid_tgid; 31 static unsigned long long (*bpf_get_current_uid_gid)(void) = 32 (void *) BPF_FUNC_get_current_uid_gid; 33 static int (*bpf_get_current_comm)(void *buf, int buf_size) = 34 (void *) BPF_FUNC_get_current_comm; 35 static unsigned long long (*bpf_perf_event_read)(void *map, 36 unsigned long long flags) = 37 (void *) BPF_FUNC_perf_event_read; 38 static int (*bpf_clone_redirect)(void *ctx, int ifindex, int flags) = 39 (void *) BPF_FUNC_clone_redirect; 40 static int (*bpf_redirect)(int ifindex, int flags) = 41 (void *) BPF_FUNC_redirect; 42 static int (*bpf_redirect_map)(void *map, int key, int flags) = 43 (void *) BPF_FUNC_redirect_map; 44 static int (*bpf_perf_event_output)(void *ctx, void *map, 45 unsigned long long flags, void *data, 46 int size) = 47 (void *) BPF_FUNC_perf_event_output; 48 static int (*bpf_get_stackid)(void *ctx, void *map, int flags) = 49 (void *) BPF_FUNC_get_stackid; 50 static int (*bpf_probe_write_user)(void *dst, void *src, int size) = 51 (void *) BPF_FUNC_probe_write_user; 52 static int (*bpf_current_task_under_cgroup)(void *map, int index) = 53 (void *) BPF_FUNC_current_task_under_cgroup; 54 static int (*bpf_skb_get_tunnel_key)(void *ctx, void *key, int size, int flags) = 55 (void *) BPF_FUNC_skb_get_tunnel_key; 56 static int (*bpf_skb_set_tunnel_key)(void *ctx, void *key, int size, int flags) = 57 (void *) BPF_FUNC_skb_set_tunnel_key; 58 static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, int size) = 59 (void *) BPF_FUNC_skb_get_tunnel_opt; 60 static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, int size) = 61 (void *) BPF_FUNC_skb_set_tunnel_opt; 62 static unsigned long long (*bpf_get_prandom_u32)(void) = 63 (void *) BPF_FUNC_get_prandom_u32; 64 static int (*bpf_xdp_adjust_head)(void *ctx, int offset) = 65 (void *) BPF_FUNC_xdp_adjust_head; 66 static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = 67 (void *) BPF_FUNC_xdp_adjust_meta; 68 static int (*bpf_get_socket_cookie)(void *ctx) = 69 (void *) BPF_FUNC_get_socket_cookie; 70 static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, 71 int optlen) = 72 (void *) BPF_FUNC_setsockopt; 73 static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, 74 int optlen) = 75 (void *) BPF_FUNC_getsockopt; 76 static int (*bpf_sock_ops_cb_flags_set)(void *ctx, int flags) = 77 (void *) BPF_FUNC_sock_ops_cb_flags_set; 78 static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = 79 (void *) BPF_FUNC_sk_redirect_map; 80 static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, int flags) = 81 (void *) BPF_FUNC_sk_redirect_hash; 82 static int (*bpf_sock_map_update)(void *map, void *key, void *value, 83 unsigned long long flags) = 84 (void *) BPF_FUNC_sock_map_update; 85 static int (*bpf_sock_hash_update)(void *map, void *key, void *value, 86 unsigned long long flags) = 87 (void *) BPF_FUNC_sock_hash_update; 88 static int (*bpf_perf_event_read_value)(void *map, unsigned long long flags, 89 void *buf, unsigned int buf_size) = 90 (void *) BPF_FUNC_perf_event_read_value; 91 static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, 92 unsigned int buf_size) = 93 (void *) BPF_FUNC_perf_prog_read_value; 94 static int (*bpf_override_return)(void *ctx, unsigned long rc) = 95 (void *) BPF_FUNC_override_return; 96 static int (*bpf_msg_redirect_map)(void *ctx, void *map, int key, int flags) = 97 (void *) BPF_FUNC_msg_redirect_map; 98 static int (*bpf_msg_redirect_hash)(void *ctx, 99 void *map, void *key, int flags) = 100 (void *) BPF_FUNC_msg_redirect_hash; 101 static int (*bpf_msg_apply_bytes)(void *ctx, int len) = 102 (void *) BPF_FUNC_msg_apply_bytes; 103 static int (*bpf_msg_cork_bytes)(void *ctx, int len) = 104 (void *) BPF_FUNC_msg_cork_bytes; 105 static int (*bpf_msg_pull_data)(void *ctx, int start, int end, int flags) = 106 (void *) BPF_FUNC_msg_pull_data; 107 static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = 108 (void *) BPF_FUNC_bind; 109 static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = 110 (void *) BPF_FUNC_xdp_adjust_tail; 111 static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state, 112 int size, int flags) = 113 (void *) BPF_FUNC_skb_get_xfrm_state; 114 static int (*bpf_sk_select_reuseport)(void *ctx, void *map, void *key, __u32 flags) = 115 (void *) BPF_FUNC_sk_select_reuseport; 116 static int (*bpf_get_stack)(void *ctx, void *buf, int size, int flags) = 117 (void *) BPF_FUNC_get_stack; 118 static int (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, 119 int plen, __u32 flags) = 120 (void *) BPF_FUNC_fib_lookup; 121 static int (*bpf_lwt_push_encap)(void *ctx, unsigned int type, void *hdr, 122 unsigned int len) = 123 (void *) BPF_FUNC_lwt_push_encap; 124 static int (*bpf_lwt_seg6_store_bytes)(void *ctx, unsigned int offset, 125 void *from, unsigned int len) = 126 (void *) BPF_FUNC_lwt_seg6_store_bytes; 127 static int (*bpf_lwt_seg6_action)(void *ctx, unsigned int action, void *param, 128 unsigned int param_len) = 129 (void *) BPF_FUNC_lwt_seg6_action; 130 static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, unsigned int offset, 131 unsigned int len) = 132 (void *) BPF_FUNC_lwt_seg6_adjust_srh; 133 static int (*bpf_rc_repeat)(void *ctx) = 134 (void *) BPF_FUNC_rc_repeat; 135 static int (*bpf_rc_keydown)(void *ctx, unsigned int protocol, 136 unsigned long long scancode, unsigned int toggle) = 137 (void *) BPF_FUNC_rc_keydown; 138 static unsigned long long (*bpf_get_current_cgroup_id)(void) = 139 (void *) BPF_FUNC_get_current_cgroup_id; 140 static void *(*bpf_get_local_storage)(void *map, unsigned long long flags) = 141 (void *) BPF_FUNC_get_local_storage; 142 static unsigned long long (*bpf_skb_cgroup_id)(void *ctx) = 143 (void *) BPF_FUNC_skb_cgroup_id; 144 static unsigned long long (*bpf_skb_ancestor_cgroup_id)(void *ctx, int level) = 145 (void *) BPF_FUNC_skb_ancestor_cgroup_id; 146 147 /* llvm builtin functions that eBPF C program may use to 148 * emit BPF_LD_ABS and BPF_LD_IND instructions 149 */ 150 struct sk_buff; 151 unsigned long long load_byte(void *skb, 152 unsigned long long off) asm("llvm.bpf.load.byte"); 153 unsigned long long load_half(void *skb, 154 unsigned long long off) asm("llvm.bpf.load.half"); 155 unsigned long long load_word(void *skb, 156 unsigned long long off) asm("llvm.bpf.load.word"); 157 158 /* a helper structure used by eBPF C program 159 * to describe map attributes to elf_bpf loader 160 */ 161 struct bpf_map_def { 162 unsigned int type; 163 unsigned int key_size; 164 unsigned int value_size; 165 unsigned int max_entries; 166 unsigned int map_flags; 167 unsigned int inner_map_idx; 168 unsigned int numa_node; 169 }; 170 171 #define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ 172 struct ____btf_map_##name { \ 173 type_key key; \ 174 type_val value; \ 175 }; \ 176 struct ____btf_map_##name \ 177 __attribute__ ((section(".maps." #name), used)) \ 178 ____btf_map_##name = { } 179 180 static int (*bpf_skb_load_bytes)(void *ctx, int off, void *to, int len) = 181 (void *) BPF_FUNC_skb_load_bytes; 182 static int (*bpf_skb_load_bytes_relative)(void *ctx, int off, void *to, int len, __u32 start_header) = 183 (void *) BPF_FUNC_skb_load_bytes_relative; 184 static int (*bpf_skb_store_bytes)(void *ctx, int off, void *from, int len, int flags) = 185 (void *) BPF_FUNC_skb_store_bytes; 186 static int (*bpf_l3_csum_replace)(void *ctx, int off, int from, int to, int flags) = 187 (void *) BPF_FUNC_l3_csum_replace; 188 static int (*bpf_l4_csum_replace)(void *ctx, int off, int from, int to, int flags) = 189 (void *) BPF_FUNC_l4_csum_replace; 190 static int (*bpf_csum_diff)(void *from, int from_size, void *to, int to_size, int seed) = 191 (void *) BPF_FUNC_csum_diff; 192 static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = 193 (void *) BPF_FUNC_skb_under_cgroup; 194 static int (*bpf_skb_change_head)(void *, int len, int flags) = 195 (void *) BPF_FUNC_skb_change_head; 196 static int (*bpf_skb_pull_data)(void *, int len) = 197 (void *) BPF_FUNC_skb_pull_data; 198 199 /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ 200 #if defined(__TARGET_ARCH_x86) 201 #define bpf_target_x86 202 #define bpf_target_defined 203 #elif defined(__TARGET_ARCH_s930x) 204 #define bpf_target_s930x 205 #define bpf_target_defined 206 #elif defined(__TARGET_ARCH_arm64) 207 #define bpf_target_arm64 208 #define bpf_target_defined 209 #elif defined(__TARGET_ARCH_mips) 210 #define bpf_target_mips 211 #define bpf_target_defined 212 #elif defined(__TARGET_ARCH_powerpc) 213 #define bpf_target_powerpc 214 #define bpf_target_defined 215 #elif defined(__TARGET_ARCH_sparc) 216 #define bpf_target_sparc 217 #define bpf_target_defined 218 #else 219 #undef bpf_target_defined 220 #endif 221 222 /* Fall back to what the compiler says */ 223 #ifndef bpf_target_defined 224 #if defined(__x86_64__) 225 #define bpf_target_x86 226 #elif defined(__s390x__) 227 #define bpf_target_s930x 228 #elif defined(__aarch64__) 229 #define bpf_target_arm64 230 #elif defined(__mips__) 231 #define bpf_target_mips 232 #elif defined(__powerpc__) 233 #define bpf_target_powerpc 234 #elif defined(__sparc__) 235 #define bpf_target_sparc 236 #endif 237 #endif 238 239 #if defined(bpf_target_x86) 240 241 #define PT_REGS_PARM1(x) ((x)->di) 242 #define PT_REGS_PARM2(x) ((x)->si) 243 #define PT_REGS_PARM3(x) ((x)->dx) 244 #define PT_REGS_PARM4(x) ((x)->cx) 245 #define PT_REGS_PARM5(x) ((x)->r8) 246 #define PT_REGS_RET(x) ((x)->sp) 247 #define PT_REGS_FP(x) ((x)->bp) 248 #define PT_REGS_RC(x) ((x)->ax) 249 #define PT_REGS_SP(x) ((x)->sp) 250 #define PT_REGS_IP(x) ((x)->ip) 251 252 #elif defined(bpf_target_s390x) 253 254 #define PT_REGS_PARM1(x) ((x)->gprs[2]) 255 #define PT_REGS_PARM2(x) ((x)->gprs[3]) 256 #define PT_REGS_PARM3(x) ((x)->gprs[4]) 257 #define PT_REGS_PARM4(x) ((x)->gprs[5]) 258 #define PT_REGS_PARM5(x) ((x)->gprs[6]) 259 #define PT_REGS_RET(x) ((x)->gprs[14]) 260 #define PT_REGS_FP(x) ((x)->gprs[11]) /* Works only with CONFIG_FRAME_POINTER */ 261 #define PT_REGS_RC(x) ((x)->gprs[2]) 262 #define PT_REGS_SP(x) ((x)->gprs[15]) 263 #define PT_REGS_IP(x) ((x)->psw.addr) 264 265 #elif defined(bpf_target_arm64) 266 267 #define PT_REGS_PARM1(x) ((x)->regs[0]) 268 #define PT_REGS_PARM2(x) ((x)->regs[1]) 269 #define PT_REGS_PARM3(x) ((x)->regs[2]) 270 #define PT_REGS_PARM4(x) ((x)->regs[3]) 271 #define PT_REGS_PARM5(x) ((x)->regs[4]) 272 #define PT_REGS_RET(x) ((x)->regs[30]) 273 #define PT_REGS_FP(x) ((x)->regs[29]) /* Works only with CONFIG_FRAME_POINTER */ 274 #define PT_REGS_RC(x) ((x)->regs[0]) 275 #define PT_REGS_SP(x) ((x)->sp) 276 #define PT_REGS_IP(x) ((x)->pc) 277 278 #elif defined(bpf_target_mips) 279 280 #define PT_REGS_PARM1(x) ((x)->regs[4]) 281 #define PT_REGS_PARM2(x) ((x)->regs[5]) 282 #define PT_REGS_PARM3(x) ((x)->regs[6]) 283 #define PT_REGS_PARM4(x) ((x)->regs[7]) 284 #define PT_REGS_PARM5(x) ((x)->regs[8]) 285 #define PT_REGS_RET(x) ((x)->regs[31]) 286 #define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */ 287 #define PT_REGS_RC(x) ((x)->regs[1]) 288 #define PT_REGS_SP(x) ((x)->regs[29]) 289 #define PT_REGS_IP(x) ((x)->cp0_epc) 290 291 #elif defined(bpf_target_powerpc) 292 293 #define PT_REGS_PARM1(x) ((x)->gpr[3]) 294 #define PT_REGS_PARM2(x) ((x)->gpr[4]) 295 #define PT_REGS_PARM3(x) ((x)->gpr[5]) 296 #define PT_REGS_PARM4(x) ((x)->gpr[6]) 297 #define PT_REGS_PARM5(x) ((x)->gpr[7]) 298 #define PT_REGS_RC(x) ((x)->gpr[3]) 299 #define PT_REGS_SP(x) ((x)->sp) 300 #define PT_REGS_IP(x) ((x)->nip) 301 302 #elif defined(bpf_target_sparc) 303 304 #define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0]) 305 #define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1]) 306 #define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2]) 307 #define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3]) 308 #define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4]) 309 #define PT_REGS_RET(x) ((x)->u_regs[UREG_I7]) 310 #define PT_REGS_RC(x) ((x)->u_regs[UREG_I0]) 311 #define PT_REGS_SP(x) ((x)->u_regs[UREG_FP]) 312 313 /* Should this also be a bpf_target check for the sparc case? */ 314 #if defined(__arch64__) 315 #define PT_REGS_IP(x) ((x)->tpc) 316 #else 317 #define PT_REGS_IP(x) ((x)->pc) 318 #endif 319 320 #endif 321 322 #ifdef bpf_target_powerpc 323 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) 324 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP 325 #elif bpf_target_sparc 326 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) 327 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP 328 #else 329 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ \ 330 bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) 331 #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ \ 332 bpf_probe_read(&(ip), sizeof(ip), \ 333 (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) 334 #endif 335 336 #endif 337
