1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3 /*
4 * AF_XDP user-space access library.
5 *
6 * Copyright(c) 2018 - 2019 Intel Corporation.
7 *
8 * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
9 */
10
11 #include <errno.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <unistd.h>
15 #include <arpa/inet.h>
16 #include <asm/barrier.h>
17 #include <linux/compiler.h>
18 #include <linux/ethtool.h>
19 #include <linux/filter.h>
20 #include <linux/if_ether.h>
21 #include <linux/if_packet.h>
22 #include <linux/if_xdp.h>
23 #include <linux/kernel.h>
24 #include <linux/list.h>
25 #include <linux/sockios.h>
26 #include <net/if.h>
27 #include <sys/ioctl.h>
28 #include <sys/mman.h>
29 #include <sys/socket.h>
30 #include <sys/types.h>
31
32 #include "bpf.h"
33 #include "libbpf.h"
34 #include "libbpf_internal.h"
35 #include "xsk.h"
36
37 #ifndef SOL_XDP
38 #define SOL_XDP 283
39 #endif
40
41 #ifndef AF_XDP
42 #define AF_XDP 44
43 #endif
44
45 #ifndef PF_XDP
46 #define PF_XDP AF_XDP
47 #endif
48
49 struct xsk_umem {
50 struct xsk_ring_prod *fill_save;
51 struct xsk_ring_cons *comp_save;
52 char *umem_area;
53 struct xsk_umem_config config;
54 int fd;
55 int refcount;
56 struct list_head ctx_list;
57 };
58
59 struct xsk_ctx {
60 struct xsk_ring_prod *fill;
61 struct xsk_ring_cons *comp;
62 __u32 queue_id;
63 struct xsk_umem *umem;
64 int refcount;
65 int ifindex;
66 struct list_head list;
67 int prog_fd;
68 int xsks_map_fd;
69 char ifname[IFNAMSIZ];
70 };
71
72 struct xsk_socket {
73 struct xsk_ring_cons *rx;
74 struct xsk_ring_prod *tx;
75 __u64 outstanding_tx;
76 struct xsk_ctx *ctx;
77 struct xsk_socket_config config;
78 int fd;
79 };
80
81 struct xsk_nl_info {
82 bool xdp_prog_attached;
83 int ifindex;
84 int fd;
85 };
86
87 /* Up until and including Linux 5.3 */
88 struct xdp_ring_offset_v1 {
89 __u64 producer;
90 __u64 consumer;
91 __u64 desc;
92 };
93
94 /* Up until and including Linux 5.3 */
95 struct xdp_mmap_offsets_v1 {
96 struct xdp_ring_offset_v1 rx;
97 struct xdp_ring_offset_v1 tx;
98 struct xdp_ring_offset_v1 fr;
99 struct xdp_ring_offset_v1 cr;
100 };
101
xsk_umem__fd(const struct xsk_umem * umem)102 int xsk_umem__fd(const struct xsk_umem *umem)
103 {
104 return umem ? umem->fd : -EINVAL;
105 }
106
xsk_socket__fd(const struct xsk_socket * xsk)107 int xsk_socket__fd(const struct xsk_socket *xsk)
108 {
109 return xsk ? xsk->fd : -EINVAL;
110 }
111
xsk_page_aligned(void * buffer)112 static bool xsk_page_aligned(void *buffer)
113 {
114 unsigned long addr = (unsigned long)buffer;
115
116 return !(addr & (getpagesize() - 1));
117 }
118
xsk_set_umem_config(struct xsk_umem_config * cfg,const struct xsk_umem_config * usr_cfg)119 static void xsk_set_umem_config(struct xsk_umem_config *cfg,
120 const struct xsk_umem_config *usr_cfg)
121 {
122 if (!usr_cfg) {
123 cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
124 cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
125 cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
126 cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
127 cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
128 return;
129 }
130
131 cfg->fill_size = usr_cfg->fill_size;
132 cfg->comp_size = usr_cfg->comp_size;
133 cfg->frame_size = usr_cfg->frame_size;
134 cfg->frame_headroom = usr_cfg->frame_headroom;
135 cfg->flags = usr_cfg->flags;
136 }
137
xsk_set_xdp_socket_config(struct xsk_socket_config * cfg,const struct xsk_socket_config * usr_cfg)138 static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
139 const struct xsk_socket_config *usr_cfg)
140 {
141 if (!usr_cfg) {
142 cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
143 cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
144 cfg->libbpf_flags = 0;
145 cfg->xdp_flags = 0;
146 cfg->bind_flags = 0;
147 return 0;
148 }
149
150 if (usr_cfg->libbpf_flags & ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)
151 return -EINVAL;
152
153 cfg->rx_size = usr_cfg->rx_size;
154 cfg->tx_size = usr_cfg->tx_size;
155 cfg->libbpf_flags = usr_cfg->libbpf_flags;
156 cfg->xdp_flags = usr_cfg->xdp_flags;
157 cfg->bind_flags = usr_cfg->bind_flags;
158
159 return 0;
160 }
161
xsk_mmap_offsets_v1(struct xdp_mmap_offsets * off)162 static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
163 {
164 struct xdp_mmap_offsets_v1 off_v1;
165
166 /* getsockopt on a kernel <= 5.3 has no flags fields.
167 * Copy over the offsets to the correct places in the >=5.4 format
168 * and put the flags where they would have been on that kernel.
169 */
170 memcpy(&off_v1, off, sizeof(off_v1));
171
172 off->rx.producer = off_v1.rx.producer;
173 off->rx.consumer = off_v1.rx.consumer;
174 off->rx.desc = off_v1.rx.desc;
175 off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
176
177 off->tx.producer = off_v1.tx.producer;
178 off->tx.consumer = off_v1.tx.consumer;
179 off->tx.desc = off_v1.tx.desc;
180 off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
181
182 off->fr.producer = off_v1.fr.producer;
183 off->fr.consumer = off_v1.fr.consumer;
184 off->fr.desc = off_v1.fr.desc;
185 off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
186
187 off->cr.producer = off_v1.cr.producer;
188 off->cr.consumer = off_v1.cr.consumer;
189 off->cr.desc = off_v1.cr.desc;
190 off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
191 }
192
xsk_get_mmap_offsets(int fd,struct xdp_mmap_offsets * off)193 static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
194 {
195 socklen_t optlen;
196 int err;
197
198 optlen = sizeof(*off);
199 err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
200 if (err)
201 return err;
202
203 if (optlen == sizeof(*off))
204 return 0;
205
206 if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
207 xsk_mmap_offsets_v1(off);
208 return 0;
209 }
210
211 return -EINVAL;
212 }
213
xsk_create_umem_rings(struct xsk_umem * umem,int fd,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)214 static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
215 struct xsk_ring_prod *fill,
216 struct xsk_ring_cons *comp)
217 {
218 struct xdp_mmap_offsets off;
219 void *map;
220 int err;
221
222 err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
223 &umem->config.fill_size,
224 sizeof(umem->config.fill_size));
225 if (err)
226 return -errno;
227
228 err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
229 &umem->config.comp_size,
230 sizeof(umem->config.comp_size));
231 if (err)
232 return -errno;
233
234 err = xsk_get_mmap_offsets(fd, &off);
235 if (err)
236 return -errno;
237
238 map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
239 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
240 XDP_UMEM_PGOFF_FILL_RING);
241 if (map == MAP_FAILED)
242 return -errno;
243
244 fill->mask = umem->config.fill_size - 1;
245 fill->size = umem->config.fill_size;
246 fill->producer = map + off.fr.producer;
247 fill->consumer = map + off.fr.consumer;
248 fill->flags = map + off.fr.flags;
249 fill->ring = map + off.fr.desc;
250 fill->cached_cons = umem->config.fill_size;
251
252 map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
253 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
254 XDP_UMEM_PGOFF_COMPLETION_RING);
255 if (map == MAP_FAILED) {
256 err = -errno;
257 goto out_mmap;
258 }
259
260 comp->mask = umem->config.comp_size - 1;
261 comp->size = umem->config.comp_size;
262 comp->producer = map + off.cr.producer;
263 comp->consumer = map + off.cr.consumer;
264 comp->flags = map + off.cr.flags;
265 comp->ring = map + off.cr.desc;
266
267 return 0;
268
269 out_mmap:
270 munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
271 return err;
272 }
273
xsk_umem__create_v0_0_4(struct xsk_umem ** umem_ptr,void * umem_area,__u64 size,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_umem_config * usr_config)274 int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
275 __u64 size, struct xsk_ring_prod *fill,
276 struct xsk_ring_cons *comp,
277 const struct xsk_umem_config *usr_config)
278 {
279 struct xdp_umem_reg mr;
280 struct xsk_umem *umem;
281 int err;
282
283 if (!umem_area || !umem_ptr || !fill || !comp)
284 return -EFAULT;
285 if (!size && !xsk_page_aligned(umem_area))
286 return -EINVAL;
287
288 umem = calloc(1, sizeof(*umem));
289 if (!umem)
290 return -ENOMEM;
291
292 umem->fd = socket(AF_XDP, SOCK_RAW, 0);
293 if (umem->fd < 0) {
294 err = -errno;
295 goto out_umem_alloc;
296 }
297
298 umem->umem_area = umem_area;
299 INIT_LIST_HEAD(&umem->ctx_list);
300 xsk_set_umem_config(&umem->config, usr_config);
301
302 memset(&mr, 0, sizeof(mr));
303 mr.addr = (uintptr_t)umem_area;
304 mr.len = size;
305 mr.chunk_size = umem->config.frame_size;
306 mr.headroom = umem->config.frame_headroom;
307 mr.flags = umem->config.flags;
308
309 err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
310 if (err) {
311 err = -errno;
312 goto out_socket;
313 }
314
315 err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
316 if (err)
317 goto out_socket;
318
319 umem->fill_save = fill;
320 umem->comp_save = comp;
321 *umem_ptr = umem;
322 return 0;
323
324 out_socket:
325 close(umem->fd);
326 out_umem_alloc:
327 free(umem);
328 return err;
329 }
330
331 struct xsk_umem_config_v1 {
332 __u32 fill_size;
333 __u32 comp_size;
334 __u32 frame_size;
335 __u32 frame_headroom;
336 };
337
xsk_umem__create_v0_0_2(struct xsk_umem ** umem_ptr,void * umem_area,__u64 size,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_umem_config * usr_config)338 int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area,
339 __u64 size, struct xsk_ring_prod *fill,
340 struct xsk_ring_cons *comp,
341 const struct xsk_umem_config *usr_config)
342 {
343 struct xsk_umem_config config;
344
345 memcpy(&config, usr_config, sizeof(struct xsk_umem_config_v1));
346 config.flags = 0;
347
348 return xsk_umem__create_v0_0_4(umem_ptr, umem_area, size, fill, comp,
349 &config);
350 }
351 COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
352 DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
353
xsk_load_xdp_prog(struct xsk_socket * xsk)354 static int xsk_load_xdp_prog(struct xsk_socket *xsk)
355 {
356 static const int log_buf_size = 16 * 1024;
357 struct xsk_ctx *ctx = xsk->ctx;
358 char log_buf[log_buf_size];
359 int err, prog_fd;
360
361 /* This is the C-program:
362 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
363 * {
364 * int ret, index = ctx->rx_queue_index;
365 *
366 * // A set entry here means that the correspnding queue_id
367 * // has an active AF_XDP socket bound to it.
368 * ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
369 * if (ret > 0)
370 * return ret;
371 *
372 * // Fallback for pre-5.3 kernels, not supporting default
373 * // action in the flags parameter.
374 * if (bpf_map_lookup_elem(&xsks_map, &index))
375 * return bpf_redirect_map(&xsks_map, index, 0);
376 * return XDP_PASS;
377 * }
378 */
379 struct bpf_insn prog[] = {
380 /* r2 = *(u32 *)(r1 + 16) */
381 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
382 /* *(u32 *)(r10 - 4) = r2 */
383 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
384 /* r1 = xskmap[] */
385 BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
386 /* r3 = XDP_PASS */
387 BPF_MOV64_IMM(BPF_REG_3, 2),
388 /* call bpf_redirect_map */
389 BPF_EMIT_CALL(BPF_FUNC_redirect_map),
390 /* if w0 != 0 goto pc+13 */
391 BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
392 /* r2 = r10 */
393 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
394 /* r2 += -4 */
395 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
396 /* r1 = xskmap[] */
397 BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
398 /* call bpf_map_lookup_elem */
399 BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
400 /* r1 = r0 */
401 BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
402 /* r0 = XDP_PASS */
403 BPF_MOV64_IMM(BPF_REG_0, 2),
404 /* if r1 == 0 goto pc+5 */
405 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
406 /* r2 = *(u32 *)(r10 - 4) */
407 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
408 /* r1 = xskmap[] */
409 BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
410 /* r3 = 0 */
411 BPF_MOV64_IMM(BPF_REG_3, 0),
412 /* call bpf_redirect_map */
413 BPF_EMIT_CALL(BPF_FUNC_redirect_map),
414 /* The jumps are to this instruction */
415 BPF_EXIT_INSN(),
416 };
417 size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn);
418
419 prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, prog, insns_cnt,
420 "LGPL-2.1 or BSD-2-Clause", 0, log_buf,
421 log_buf_size);
422 if (prog_fd < 0) {
423 pr_warn("BPF log buffer:\n%s", log_buf);
424 return prog_fd;
425 }
426
427 err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, prog_fd,
428 xsk->config.xdp_flags);
429 if (err) {
430 close(prog_fd);
431 return err;
432 }
433
434 ctx->prog_fd = prog_fd;
435 return 0;
436 }
437
xsk_get_max_queues(struct xsk_socket * xsk)438 static int xsk_get_max_queues(struct xsk_socket *xsk)
439 {
440 struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
441 struct xsk_ctx *ctx = xsk->ctx;
442 struct ifreq ifr = {};
443 int fd, err, ret;
444
445 fd = socket(AF_INET, SOCK_DGRAM, 0);
446 if (fd < 0)
447 return -errno;
448
449 ifr.ifr_data = (void *)&channels;
450 memcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ - 1);
451 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
452 err = ioctl(fd, SIOCETHTOOL, &ifr);
453 if (err && errno != EOPNOTSUPP) {
454 ret = -errno;
455 goto out;
456 }
457
458 if (err) {
459 /* If the device says it has no channels, then all traffic
460 * is sent to a single stream, so max queues = 1.
461 */
462 ret = 1;
463 } else {
464 /* Take the max of rx, tx, combined. Drivers return
465 * the number of channels in different ways.
466 */
467 ret = max(channels.max_rx, channels.max_tx);
468 ret = max(ret, (int)channels.max_combined);
469 }
470
471 out:
472 close(fd);
473 return ret;
474 }
475
xsk_create_bpf_maps(struct xsk_socket * xsk)476 static int xsk_create_bpf_maps(struct xsk_socket *xsk)
477 {
478 struct xsk_ctx *ctx = xsk->ctx;
479 int max_queues;
480 int fd;
481
482 max_queues = xsk_get_max_queues(xsk);
483 if (max_queues < 0)
484 return max_queues;
485
486 fd = bpf_create_map_name(BPF_MAP_TYPE_XSKMAP, "xsks_map",
487 sizeof(int), sizeof(int), max_queues, 0);
488 if (fd < 0)
489 return fd;
490
491 ctx->xsks_map_fd = fd;
492
493 return 0;
494 }
495
xsk_delete_bpf_maps(struct xsk_socket * xsk)496 static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
497 {
498 struct xsk_ctx *ctx = xsk->ctx;
499
500 bpf_map_delete_elem(ctx->xsks_map_fd, &ctx->queue_id);
501 close(ctx->xsks_map_fd);
502 }
503
xsk_lookup_bpf_maps(struct xsk_socket * xsk)504 static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
505 {
506 __u32 i, *map_ids, num_maps, prog_len = sizeof(struct bpf_prog_info);
507 __u32 map_len = sizeof(struct bpf_map_info);
508 struct bpf_prog_info prog_info = {};
509 struct xsk_ctx *ctx = xsk->ctx;
510 struct bpf_map_info map_info;
511 int fd, err;
512
513 err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
514 if (err)
515 return err;
516
517 num_maps = prog_info.nr_map_ids;
518
519 map_ids = calloc(prog_info.nr_map_ids, sizeof(*map_ids));
520 if (!map_ids)
521 return -ENOMEM;
522
523 memset(&prog_info, 0, prog_len);
524 prog_info.nr_map_ids = num_maps;
525 prog_info.map_ids = (__u64)(unsigned long)map_ids;
526
527 err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
528 if (err)
529 goto out_map_ids;
530
531 ctx->xsks_map_fd = -1;
532
533 for (i = 0; i < prog_info.nr_map_ids; i++) {
534 fd = bpf_map_get_fd_by_id(map_ids[i]);
535 if (fd < 0)
536 continue;
537
538 err = bpf_obj_get_info_by_fd(fd, &map_info, &map_len);
539 if (err) {
540 close(fd);
541 continue;
542 }
543
544 if (!strcmp(map_info.name, "xsks_map")) {
545 ctx->xsks_map_fd = fd;
546 continue;
547 }
548
549 close(fd);
550 }
551
552 err = 0;
553 if (ctx->xsks_map_fd == -1)
554 err = -ENOENT;
555
556 out_map_ids:
557 free(map_ids);
558 return err;
559 }
560
xsk_set_bpf_maps(struct xsk_socket * xsk)561 static int xsk_set_bpf_maps(struct xsk_socket *xsk)
562 {
563 struct xsk_ctx *ctx = xsk->ctx;
564
565 return bpf_map_update_elem(ctx->xsks_map_fd, &ctx->queue_id,
566 &xsk->fd, 0);
567 }
568
xsk_setup_xdp_prog(struct xsk_socket * xsk)569 static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
570 {
571 struct xsk_ctx *ctx = xsk->ctx;
572 __u32 prog_id = 0;
573 int err;
574
575 err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id,
576 xsk->config.xdp_flags);
577 if (err)
578 return err;
579
580 if (!prog_id) {
581 err = xsk_create_bpf_maps(xsk);
582 if (err)
583 return err;
584
585 err = xsk_load_xdp_prog(xsk);
586 if (err) {
587 xsk_delete_bpf_maps(xsk);
588 return err;
589 }
590 } else {
591 ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
592 if (ctx->prog_fd < 0)
593 return -errno;
594 err = xsk_lookup_bpf_maps(xsk);
595 if (err) {
596 close(ctx->prog_fd);
597 return err;
598 }
599 }
600
601 if (xsk->rx)
602 err = xsk_set_bpf_maps(xsk);
603 if (err) {
604 xsk_delete_bpf_maps(xsk);
605 close(ctx->prog_fd);
606 return err;
607 }
608
609 return 0;
610 }
611
xsk_get_ctx(struct xsk_umem * umem,int ifindex,__u32 queue_id)612 static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
613 __u32 queue_id)
614 {
615 struct xsk_ctx *ctx;
616
617 if (list_empty(&umem->ctx_list))
618 return NULL;
619
620 list_for_each_entry(ctx, &umem->ctx_list, list) {
621 if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
622 ctx->refcount++;
623 return ctx;
624 }
625 }
626
627 return NULL;
628 }
629
xsk_put_ctx(struct xsk_ctx * ctx)630 static void xsk_put_ctx(struct xsk_ctx *ctx)
631 {
632 struct xsk_umem *umem = ctx->umem;
633 struct xdp_mmap_offsets off;
634 int err;
635
636 if (--ctx->refcount == 0) {
637 err = xsk_get_mmap_offsets(umem->fd, &off);
638 if (!err) {
639 munmap(ctx->fill->ring - off.fr.desc,
640 off.fr.desc + umem->config.fill_size *
641 sizeof(__u64));
642 munmap(ctx->comp->ring - off.cr.desc,
643 off.cr.desc + umem->config.comp_size *
644 sizeof(__u64));
645 }
646
647 list_del(&ctx->list);
648 free(ctx);
649 }
650 }
651
xsk_create_ctx(struct xsk_socket * xsk,struct xsk_umem * umem,int ifindex,const char * ifname,__u32 queue_id,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)652 static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
653 struct xsk_umem *umem, int ifindex,
654 const char *ifname, __u32 queue_id,
655 struct xsk_ring_prod *fill,
656 struct xsk_ring_cons *comp)
657 {
658 struct xsk_ctx *ctx;
659 int err;
660
661 ctx = calloc(1, sizeof(*ctx));
662 if (!ctx)
663 return NULL;
664
665 if (!umem->fill_save) {
666 err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
667 if (err) {
668 free(ctx);
669 return NULL;
670 }
671 } else if (umem->fill_save != fill || umem->comp_save != comp) {
672 /* Copy over rings to new structs. */
673 memcpy(fill, umem->fill_save, sizeof(*fill));
674 memcpy(comp, umem->comp_save, sizeof(*comp));
675 }
676
677 ctx->ifindex = ifindex;
678 ctx->refcount = 1;
679 ctx->umem = umem;
680 ctx->queue_id = queue_id;
681 memcpy(ctx->ifname, ifname, IFNAMSIZ - 1);
682 ctx->ifname[IFNAMSIZ - 1] = '\0';
683
684 umem->fill_save = NULL;
685 umem->comp_save = NULL;
686 ctx->fill = fill;
687 ctx->comp = comp;
688 list_add(&ctx->list, &umem->ctx_list);
689 return ctx;
690 }
691
xsk_socket__create_shared(struct xsk_socket ** xsk_ptr,const char * ifname,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_socket_config * usr_config)692 int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
693 const char *ifname,
694 __u32 queue_id, struct xsk_umem *umem,
695 struct xsk_ring_cons *rx,
696 struct xsk_ring_prod *tx,
697 struct xsk_ring_prod *fill,
698 struct xsk_ring_cons *comp,
699 const struct xsk_socket_config *usr_config)
700 {
701 void *rx_map = NULL, *tx_map = NULL;
702 struct sockaddr_xdp sxdp = {};
703 struct xdp_mmap_offsets off;
704 struct xsk_socket *xsk;
705 struct xsk_ctx *ctx;
706 int err, ifindex;
707
708 if (!umem || !xsk_ptr || !(rx || tx))
709 return -EFAULT;
710
711 xsk = calloc(1, sizeof(*xsk));
712 if (!xsk)
713 return -ENOMEM;
714
715 err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
716 if (err)
717 goto out_xsk_alloc;
718
719 xsk->outstanding_tx = 0;
720 ifindex = if_nametoindex(ifname);
721 if (!ifindex) {
722 err = -errno;
723 goto out_xsk_alloc;
724 }
725
726 if (umem->refcount++ > 0) {
727 xsk->fd = socket(AF_XDP, SOCK_RAW, 0);
728 if (xsk->fd < 0) {
729 err = -errno;
730 goto out_xsk_alloc;
731 }
732 } else {
733 xsk->fd = umem->fd;
734 }
735
736 ctx = xsk_get_ctx(umem, ifindex, queue_id);
737 if (!ctx) {
738 if (!fill || !comp) {
739 err = -EFAULT;
740 goto out_socket;
741 }
742
743 ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
744 fill, comp);
745 if (!ctx) {
746 err = -ENOMEM;
747 goto out_socket;
748 }
749 }
750 xsk->ctx = ctx;
751
752 if (rx) {
753 err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
754 &xsk->config.rx_size,
755 sizeof(xsk->config.rx_size));
756 if (err) {
757 err = -errno;
758 goto out_put_ctx;
759 }
760 }
761 if (tx) {
762 err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
763 &xsk->config.tx_size,
764 sizeof(xsk->config.tx_size));
765 if (err) {
766 err = -errno;
767 goto out_put_ctx;
768 }
769 }
770
771 err = xsk_get_mmap_offsets(xsk->fd, &off);
772 if (err) {
773 err = -errno;
774 goto out_put_ctx;
775 }
776
777 if (rx) {
778 rx_map = mmap(NULL, off.rx.desc +
779 xsk->config.rx_size * sizeof(struct xdp_desc),
780 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
781 xsk->fd, XDP_PGOFF_RX_RING);
782 if (rx_map == MAP_FAILED) {
783 err = -errno;
784 goto out_put_ctx;
785 }
786
787 rx->mask = xsk->config.rx_size - 1;
788 rx->size = xsk->config.rx_size;
789 rx->producer = rx_map + off.rx.producer;
790 rx->consumer = rx_map + off.rx.consumer;
791 rx->flags = rx_map + off.rx.flags;
792 rx->ring = rx_map + off.rx.desc;
793 rx->cached_prod = *rx->producer;
794 rx->cached_cons = *rx->consumer;
795 }
796 xsk->rx = rx;
797
798 if (tx) {
799 tx_map = mmap(NULL, off.tx.desc +
800 xsk->config.tx_size * sizeof(struct xdp_desc),
801 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
802 xsk->fd, XDP_PGOFF_TX_RING);
803 if (tx_map == MAP_FAILED) {
804 err = -errno;
805 goto out_mmap_rx;
806 }
807
808 tx->mask = xsk->config.tx_size - 1;
809 tx->size = xsk->config.tx_size;
810 tx->producer = tx_map + off.tx.producer;
811 tx->consumer = tx_map + off.tx.consumer;
812 tx->flags = tx_map + off.tx.flags;
813 tx->ring = tx_map + off.tx.desc;
814 tx->cached_prod = *tx->producer;
815 /* cached_cons is r->size bigger than the real consumer pointer
816 * See xsk_prod_nb_free
817 */
818 tx->cached_cons = *tx->consumer + xsk->config.tx_size;
819 }
820 xsk->tx = tx;
821
822 sxdp.sxdp_family = PF_XDP;
823 sxdp.sxdp_ifindex = ctx->ifindex;
824 sxdp.sxdp_queue_id = ctx->queue_id;
825 if (umem->refcount > 1) {
826 sxdp.sxdp_flags |= XDP_SHARED_UMEM;
827 sxdp.sxdp_shared_umem_fd = umem->fd;
828 } else {
829 sxdp.sxdp_flags = xsk->config.bind_flags;
830 }
831
832 err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
833 if (err) {
834 err = -errno;
835 goto out_mmap_tx;
836 }
837
838 ctx->prog_fd = -1;
839
840 if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
841 err = xsk_setup_xdp_prog(xsk);
842 if (err)
843 goto out_mmap_tx;
844 }
845
846 *xsk_ptr = xsk;
847 return 0;
848
849 out_mmap_tx:
850 if (tx)
851 munmap(tx_map, off.tx.desc +
852 xsk->config.tx_size * sizeof(struct xdp_desc));
853 out_mmap_rx:
854 if (rx)
855 munmap(rx_map, off.rx.desc +
856 xsk->config.rx_size * sizeof(struct xdp_desc));
857 out_put_ctx:
858 xsk_put_ctx(ctx);
859 out_socket:
860 if (--umem->refcount)
861 close(xsk->fd);
862 out_xsk_alloc:
863 free(xsk);
864 return err;
865 }
866
xsk_socket__create(struct xsk_socket ** xsk_ptr,const char * ifname,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,const struct xsk_socket_config * usr_config)867 int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
868 __u32 queue_id, struct xsk_umem *umem,
869 struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
870 const struct xsk_socket_config *usr_config)
871 {
872 return xsk_socket__create_shared(xsk_ptr, ifname, queue_id, umem,
873 rx, tx, umem->fill_save,
874 umem->comp_save, usr_config);
875 }
876
xsk_umem__delete(struct xsk_umem * umem)877 int xsk_umem__delete(struct xsk_umem *umem)
878 {
879 if (!umem)
880 return 0;
881
882 if (umem->refcount)
883 return -EBUSY;
884
885 close(umem->fd);
886 free(umem);
887
888 return 0;
889 }
890
xsk_socket__delete(struct xsk_socket * xsk)891 void xsk_socket__delete(struct xsk_socket *xsk)
892 {
893 size_t desc_sz = sizeof(struct xdp_desc);
894 struct xdp_mmap_offsets off;
895 struct xsk_umem *umem;
896 struct xsk_ctx *ctx;
897 int err;
898
899 if (!xsk)
900 return;
901
902 ctx = xsk->ctx;
903 umem = ctx->umem;
904 if (ctx->prog_fd != -1) {
905 xsk_delete_bpf_maps(xsk);
906 close(ctx->prog_fd);
907 }
908
909 err = xsk_get_mmap_offsets(xsk->fd, &off);
910 if (!err) {
911 if (xsk->rx) {
912 munmap(xsk->rx->ring - off.rx.desc,
913 off.rx.desc + xsk->config.rx_size * desc_sz);
914 }
915 if (xsk->tx) {
916 munmap(xsk->tx->ring - off.tx.desc,
917 off.tx.desc + xsk->config.tx_size * desc_sz);
918 }
919 }
920
921 xsk_put_ctx(ctx);
922
923 umem->refcount--;
924 /* Do not close an fd that also has an associated umem connected
925 * to it.
926 */
927 if (xsk->fd != umem->fd)
928 close(xsk->fd);
929 free(xsk);
930 }
931
