1 #include <linux/kernel.h>
2 #include <linux/errno.h>
3 #include <linux/file.h>
4 #include <linux/slab.h>
5 #include <linux/net.h>
6 #include <linux/io_uring.h>
7
8 #include "io_uring.h"
9 #include "notif.h"
10 #include "rsrc.h"
11
__io_notif_complete_tw(struct io_kiocb * notif,bool * locked)12 static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)
13 {
14 struct io_notif_data *nd = io_notif_to_data(notif);
15 struct io_ring_ctx *ctx = notif->ctx;
16
17 if (nd->account_pages && ctx->user) {
18 __io_unaccount_mem(ctx->user, nd->account_pages);
19 nd->account_pages = 0;
20 }
21 io_req_task_complete(notif, locked);
22 }
23
io_uring_tx_zerocopy_callback(struct sk_buff * skb,struct ubuf_info * uarg,bool success)24 static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,
25 struct ubuf_info *uarg,
26 bool success)
27 {
28 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
29 struct io_kiocb *notif = cmd_to_io_kiocb(nd);
30
31 if (refcount_dec_and_test(&uarg->refcnt)) {
32 notif->io_task_work.func = __io_notif_complete_tw;
33 io_req_task_work_add(notif);
34 }
35 }
36
io_alloc_notif(struct io_ring_ctx * ctx)37 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
38 __must_hold(&ctx->uring_lock)
39 {
40 struct io_kiocb *notif;
41 struct io_notif_data *nd;
42
43 if (unlikely(!io_alloc_req_refill(ctx)))
44 return NULL;
45 notif = io_alloc_req(ctx);
46 notif->opcode = IORING_OP_NOP;
47 notif->flags = 0;
48 notif->file = NULL;
49 notif->task = current;
50 io_get_task_refs(1);
51 notif->rsrc_node = NULL;
52 io_req_set_rsrc_node(notif, ctx, 0);
53
54 nd = io_notif_to_data(notif);
55 nd->account_pages = 0;
56 nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
57 nd->uarg.callback = io_uring_tx_zerocopy_callback;
58 refcount_set(&nd->uarg.refcnt, 1);
59 return notif;
60 }
61
io_notif_flush(struct io_kiocb * notif)62 void io_notif_flush(struct io_kiocb *notif)
63 __must_hold(&slot->notif->ctx->uring_lock)
64 {
65 struct io_notif_data *nd = io_notif_to_data(notif);
66
67 /* drop slot's master ref */
68 if (refcount_dec_and_test(&nd->uarg.refcnt)) {
69 notif->io_task_work.func = __io_notif_complete_tw;
70 io_req_task_work_add(notif);
71 }
72 }
73
