1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "opdef.h"
13 #include "refs.h"
14 #include "tctx.h"
15 #include "sqpoll.h"
16 #include "fdinfo.h"
17 #include "kbuf.h"
18 #include "rsrc.h"
19
20 #include "xattr.h"
21 #include "nop.h"
22 #include "fs.h"
23 #include "splice.h"
24 #include "sync.h"
25 #include "advise.h"
26 #include "openclose.h"
27 #include "uring_cmd.h"
28 #include "epoll.h"
29 #include "statx.h"
30 #include "net.h"
31 #include "msg_ring.h"
32 #include "timeout.h"
33 #include "poll.h"
34 #include "cancel.h"
35 #include "rw.h"
36
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 {
39 WARN_ON_ONCE(1);
40 return -ECANCELED;
41 }
42
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44 const struct io_uring_sqe *sqe)
45 {
46 return -EOPNOTSUPP;
47 }
48
49 const struct io_op_def io_op_defs[] = {
50 [IORING_OP_NOP] = {
51 .audit_skip = 1,
52 .iopoll = 1,
53 .name = "NOP",
54 .prep = io_nop_prep,
55 .issue = io_nop,
56 },
57 [IORING_OP_READV] = {
58 .needs_file = 1,
59 .unbound_nonreg_file = 1,
60 .pollin = 1,
61 .buffer_select = 1,
62 .plug = 1,
63 .audit_skip = 1,
64 .ioprio = 1,
65 .iopoll = 1,
66 .async_size = sizeof(struct io_async_rw),
67 .name = "READV",
68 .prep = io_prep_rw,
69 .issue = io_read,
70 .prep_async = io_readv_prep_async,
71 .cleanup = io_readv_writev_cleanup,
72 .fail = io_rw_fail,
73 },
74 [IORING_OP_WRITEV] = {
75 .needs_file = 1,
76 .hash_reg_file = 1,
77 .unbound_nonreg_file = 1,
78 .pollout = 1,
79 .plug = 1,
80 .audit_skip = 1,
81 .ioprio = 1,
82 .iopoll = 1,
83 .async_size = sizeof(struct io_async_rw),
84 .name = "WRITEV",
85 .prep = io_prep_rw,
86 .issue = io_write,
87 .prep_async = io_writev_prep_async,
88 .cleanup = io_readv_writev_cleanup,
89 .fail = io_rw_fail,
90 },
91 [IORING_OP_FSYNC] = {
92 .needs_file = 1,
93 .audit_skip = 1,
94 .name = "FSYNC",
95 .prep = io_fsync_prep,
96 .issue = io_fsync,
97 },
98 [IORING_OP_READ_FIXED] = {
99 .needs_file = 1,
100 .unbound_nonreg_file = 1,
101 .pollin = 1,
102 .plug = 1,
103 .audit_skip = 1,
104 .ioprio = 1,
105 .iopoll = 1,
106 .async_size = sizeof(struct io_async_rw),
107 .name = "READ_FIXED",
108 .prep = io_prep_rw,
109 .issue = io_read,
110 .fail = io_rw_fail,
111 },
112 [IORING_OP_WRITE_FIXED] = {
113 .needs_file = 1,
114 .hash_reg_file = 1,
115 .unbound_nonreg_file = 1,
116 .pollout = 1,
117 .plug = 1,
118 .audit_skip = 1,
119 .ioprio = 1,
120 .iopoll = 1,
121 .async_size = sizeof(struct io_async_rw),
122 .name = "WRITE_FIXED",
123 .prep = io_prep_rw,
124 .issue = io_write,
125 .fail = io_rw_fail,
126 },
127 [IORING_OP_POLL_ADD] = {
128 .needs_file = 1,
129 .unbound_nonreg_file = 1,
130 .audit_skip = 1,
131 .name = "POLL_ADD",
132 .prep = io_poll_add_prep,
133 .issue = io_poll_add,
134 },
135 [IORING_OP_POLL_REMOVE] = {
136 .audit_skip = 1,
137 .name = "POLL_REMOVE",
138 .prep = io_poll_remove_prep,
139 .issue = io_poll_remove,
140 },
141 [IORING_OP_SYNC_FILE_RANGE] = {
142 .needs_file = 1,
143 .audit_skip = 1,
144 .name = "SYNC_FILE_RANGE",
145 .prep = io_sfr_prep,
146 .issue = io_sync_file_range,
147 },
148 [IORING_OP_SENDMSG] = {
149 .needs_file = 1,
150 .unbound_nonreg_file = 1,
151 .pollout = 1,
152 .ioprio = 1,
153 .manual_alloc = 1,
154 .name = "SENDMSG",
155 #if defined(CONFIG_NET)
156 .async_size = sizeof(struct io_async_msghdr),
157 .prep = io_sendmsg_prep,
158 .issue = io_sendmsg,
159 .prep_async = io_sendmsg_prep_async,
160 .cleanup = io_sendmsg_recvmsg_cleanup,
161 .fail = io_sendrecv_fail,
162 #else
163 .prep = io_eopnotsupp_prep,
164 #endif
165 },
166 [IORING_OP_RECVMSG] = {
167 .needs_file = 1,
168 .unbound_nonreg_file = 1,
169 .pollin = 1,
170 .buffer_select = 1,
171 .ioprio = 1,
172 .manual_alloc = 1,
173 .name = "RECVMSG",
174 #if defined(CONFIG_NET)
175 .async_size = sizeof(struct io_async_msghdr),
176 .prep = io_recvmsg_prep,
177 .issue = io_recvmsg,
178 .prep_async = io_recvmsg_prep_async,
179 .cleanup = io_sendmsg_recvmsg_cleanup,
180 .fail = io_sendrecv_fail,
181 #else
182 .prep = io_eopnotsupp_prep,
183 #endif
184 },
185 [IORING_OP_TIMEOUT] = {
186 .audit_skip = 1,
187 .async_size = sizeof(struct io_timeout_data),
188 .name = "TIMEOUT",
189 .prep = io_timeout_prep,
190 .issue = io_timeout,
191 },
192 [IORING_OP_TIMEOUT_REMOVE] = {
193 /* used by timeout updates' prep() */
194 .audit_skip = 1,
195 .name = "TIMEOUT_REMOVE",
196 .prep = io_timeout_remove_prep,
197 .issue = io_timeout_remove,
198 },
199 [IORING_OP_ACCEPT] = {
200 .needs_file = 1,
201 .unbound_nonreg_file = 1,
202 .pollin = 1,
203 .poll_exclusive = 1,
204 .ioprio = 1, /* used for flags */
205 .name = "ACCEPT",
206 #if defined(CONFIG_NET)
207 .prep = io_accept_prep,
208 .issue = io_accept,
209 #else
210 .prep = io_eopnotsupp_prep,
211 #endif
212 },
213 [IORING_OP_ASYNC_CANCEL] = {
214 .audit_skip = 1,
215 .name = "ASYNC_CANCEL",
216 .prep = io_async_cancel_prep,
217 .issue = io_async_cancel,
218 },
219 [IORING_OP_LINK_TIMEOUT] = {
220 .audit_skip = 1,
221 .async_size = sizeof(struct io_timeout_data),
222 .name = "LINK_TIMEOUT",
223 .prep = io_link_timeout_prep,
224 .issue = io_no_issue,
225 },
226 [IORING_OP_CONNECT] = {
227 .needs_file = 1,
228 .unbound_nonreg_file = 1,
229 .pollout = 1,
230 .name = "CONNECT",
231 #if defined(CONFIG_NET)
232 .async_size = sizeof(struct io_async_connect),
233 .prep = io_connect_prep,
234 .issue = io_connect,
235 .prep_async = io_connect_prep_async,
236 #else
237 .prep = io_eopnotsupp_prep,
238 #endif
239 },
240 [IORING_OP_FALLOCATE] = {
241 .needs_file = 1,
242 .name = "FALLOCATE",
243 .prep = io_fallocate_prep,
244 .issue = io_fallocate,
245 },
246 [IORING_OP_OPENAT] = {
247 .name = "OPENAT",
248 .prep = io_openat_prep,
249 .issue = io_openat,
250 .cleanup = io_open_cleanup,
251 },
252 [IORING_OP_CLOSE] = {
253 .name = "CLOSE",
254 .prep = io_close_prep,
255 .issue = io_close,
256 },
257 [IORING_OP_FILES_UPDATE] = {
258 .audit_skip = 1,
259 .iopoll = 1,
260 .name = "FILES_UPDATE",
261 .prep = io_files_update_prep,
262 .issue = io_files_update,
263 },
264 [IORING_OP_STATX] = {
265 .audit_skip = 1,
266 .name = "STATX",
267 .prep = io_statx_prep,
268 .issue = io_statx,
269 .cleanup = io_statx_cleanup,
270 },
271 [IORING_OP_READ] = {
272 .needs_file = 1,
273 .unbound_nonreg_file = 1,
274 .pollin = 1,
275 .buffer_select = 1,
276 .plug = 1,
277 .audit_skip = 1,
278 .ioprio = 1,
279 .iopoll = 1,
280 .async_size = sizeof(struct io_async_rw),
281 .name = "READ",
282 .prep = io_prep_rw,
283 .issue = io_read,
284 .fail = io_rw_fail,
285 },
286 [IORING_OP_WRITE] = {
287 .needs_file = 1,
288 .hash_reg_file = 1,
289 .unbound_nonreg_file = 1,
290 .pollout = 1,
291 .plug = 1,
292 .audit_skip = 1,
293 .ioprio = 1,
294 .iopoll = 1,
295 .async_size = sizeof(struct io_async_rw),
296 .name = "WRITE",
297 .prep = io_prep_rw,
298 .issue = io_write,
299 .fail = io_rw_fail,
300 },
301 [IORING_OP_FADVISE] = {
302 .needs_file = 1,
303 .audit_skip = 1,
304 .name = "FADVISE",
305 .prep = io_fadvise_prep,
306 .issue = io_fadvise,
307 },
308 [IORING_OP_MADVISE] = {
309 .name = "MADVISE",
310 .prep = io_madvise_prep,
311 .issue = io_madvise,
312 },
313 [IORING_OP_SEND] = {
314 .needs_file = 1,
315 .unbound_nonreg_file = 1,
316 .pollout = 1,
317 .audit_skip = 1,
318 .ioprio = 1,
319 .manual_alloc = 1,
320 .name = "SEND",
321 #if defined(CONFIG_NET)
322 .async_size = sizeof(struct io_async_msghdr),
323 .prep = io_sendmsg_prep,
324 .issue = io_send,
325 .fail = io_sendrecv_fail,
326 .prep_async = io_send_prep_async,
327 #else
328 .prep = io_eopnotsupp_prep,
329 #endif
330 },
331 [IORING_OP_RECV] = {
332 .needs_file = 1,
333 .unbound_nonreg_file = 1,
334 .pollin = 1,
335 .buffer_select = 1,
336 .audit_skip = 1,
337 .ioprio = 1,
338 .name = "RECV",
339 #if defined(CONFIG_NET)
340 .prep = io_recvmsg_prep,
341 .issue = io_recv,
342 .fail = io_sendrecv_fail,
343 #else
344 .prep = io_eopnotsupp_prep,
345 #endif
346 },
347 [IORING_OP_OPENAT2] = {
348 .name = "OPENAT2",
349 .prep = io_openat2_prep,
350 .issue = io_openat2,
351 .cleanup = io_open_cleanup,
352 },
353 [IORING_OP_EPOLL_CTL] = {
354 .unbound_nonreg_file = 1,
355 .audit_skip = 1,
356 .name = "EPOLL",
357 #if defined(CONFIG_EPOLL)
358 .prep = io_epoll_ctl_prep,
359 .issue = io_epoll_ctl,
360 #else
361 .prep = io_eopnotsupp_prep,
362 #endif
363 },
364 [IORING_OP_SPLICE] = {
365 .needs_file = 1,
366 .hash_reg_file = 1,
367 .unbound_nonreg_file = 1,
368 .audit_skip = 1,
369 .name = "SPLICE",
370 .prep = io_splice_prep,
371 .issue = io_splice,
372 },
373 [IORING_OP_PROVIDE_BUFFERS] = {
374 .audit_skip = 1,
375 .iopoll = 1,
376 .name = "PROVIDE_BUFFERS",
377 .prep = io_provide_buffers_prep,
378 .issue = io_provide_buffers,
379 },
380 [IORING_OP_REMOVE_BUFFERS] = {
381 .audit_skip = 1,
382 .iopoll = 1,
383 .name = "REMOVE_BUFFERS",
384 .prep = io_remove_buffers_prep,
385 .issue = io_remove_buffers,
386 },
387 [IORING_OP_TEE] = {
388 .needs_file = 1,
389 .hash_reg_file = 1,
390 .unbound_nonreg_file = 1,
391 .audit_skip = 1,
392 .name = "TEE",
393 .prep = io_tee_prep,
394 .issue = io_tee,
395 },
396 [IORING_OP_SHUTDOWN] = {
397 .needs_file = 1,
398 .name = "SHUTDOWN",
399 #if defined(CONFIG_NET)
400 .prep = io_shutdown_prep,
401 .issue = io_shutdown,
402 #else
403 .prep = io_eopnotsupp_prep,
404 #endif
405 },
406 [IORING_OP_RENAMEAT] = {
407 .name = "RENAMEAT",
408 .prep = io_renameat_prep,
409 .issue = io_renameat,
410 .cleanup = io_renameat_cleanup,
411 },
412 [IORING_OP_UNLINKAT] = {
413 .name = "UNLINKAT",
414 .prep = io_unlinkat_prep,
415 .issue = io_unlinkat,
416 .cleanup = io_unlinkat_cleanup,
417 },
418 [IORING_OP_MKDIRAT] = {
419 .name = "MKDIRAT",
420 .prep = io_mkdirat_prep,
421 .issue = io_mkdirat,
422 .cleanup = io_mkdirat_cleanup,
423 },
424 [IORING_OP_SYMLINKAT] = {
425 .name = "SYMLINKAT",
426 .prep = io_symlinkat_prep,
427 .issue = io_symlinkat,
428 .cleanup = io_link_cleanup,
429 },
430 [IORING_OP_LINKAT] = {
431 .name = "LINKAT",
432 .prep = io_linkat_prep,
433 .issue = io_linkat,
434 .cleanup = io_link_cleanup,
435 },
436 [IORING_OP_MSG_RING] = {
437 .needs_file = 1,
438 .iopoll = 1,
439 .name = "MSG_RING",
440 .prep = io_msg_ring_prep,
441 .issue = io_msg_ring,
442 },
443 [IORING_OP_FSETXATTR] = {
444 .needs_file = 1,
445 .name = "FSETXATTR",
446 .prep = io_fsetxattr_prep,
447 .issue = io_fsetxattr,
448 .cleanup = io_xattr_cleanup,
449 },
450 [IORING_OP_SETXATTR] = {
451 .name = "SETXATTR",
452 .prep = io_setxattr_prep,
453 .issue = io_setxattr,
454 .cleanup = io_xattr_cleanup,
455 },
456 [IORING_OP_FGETXATTR] = {
457 .needs_file = 1,
458 .name = "FGETXATTR",
459 .prep = io_fgetxattr_prep,
460 .issue = io_fgetxattr,
461 .cleanup = io_xattr_cleanup,
462 },
463 [IORING_OP_GETXATTR] = {
464 .name = "GETXATTR",
465 .prep = io_getxattr_prep,
466 .issue = io_getxattr,
467 .cleanup = io_xattr_cleanup,
468 },
469 [IORING_OP_SOCKET] = {
470 .audit_skip = 1,
471 .name = "SOCKET",
472 #if defined(CONFIG_NET)
473 .prep = io_socket_prep,
474 .issue = io_socket,
475 #else
476 .prep = io_eopnotsupp_prep,
477 #endif
478 },
479 [IORING_OP_URING_CMD] = {
480 .needs_file = 1,
481 .plug = 1,
482 .name = "URING_CMD",
483 .iopoll = 1,
484 .async_size = uring_cmd_pdu_size(1),
485 .prep = io_uring_cmd_prep,
486 .issue = io_uring_cmd,
487 .prep_async = io_uring_cmd_prep_async,
488 },
489 [IORING_OP_SEND_ZC] = {
490 .name = "SEND_ZC",
491 .needs_file = 1,
492 .unbound_nonreg_file = 1,
493 .pollout = 1,
494 .audit_skip = 1,
495 .ioprio = 1,
496 .manual_alloc = 1,
497 #if defined(CONFIG_NET)
498 .async_size = sizeof(struct io_async_msghdr),
499 .prep = io_send_zc_prep,
500 .issue = io_send_zc,
501 .prep_async = io_send_prep_async,
502 .cleanup = io_send_zc_cleanup,
503 .fail = io_sendrecv_fail,
504 #else
505 .prep = io_eopnotsupp_prep,
506 #endif
507 },
508 [IORING_OP_SENDMSG_ZC] = {
509 .name = "SENDMSG_ZC",
510 .needs_file = 1,
511 .unbound_nonreg_file = 1,
512 .pollout = 1,
513 .ioprio = 1,
514 .manual_alloc = 1,
515 #if defined(CONFIG_NET)
516 .async_size = sizeof(struct io_async_msghdr),
517 .prep = io_send_zc_prep,
518 .issue = io_sendmsg_zc,
519 .prep_async = io_sendmsg_prep_async,
520 .cleanup = io_send_zc_cleanup,
521 .fail = io_sendrecv_fail,
522 #else
523 .prep = io_eopnotsupp_prep,
524 #endif
525 },
526 };
527
io_uring_get_opcode(u8 opcode)528 const char *io_uring_get_opcode(u8 opcode)
529 {
530 if (opcode < IORING_OP_LAST)
531 return io_op_defs[opcode].name;
532 return "INVALID";
533 }
534
io_uring_optable_init(void)535 void __init io_uring_optable_init(void)
536 {
537 int i;
538
539 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
540
541 for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
542 BUG_ON(!io_op_defs[i].prep);
543 if (io_op_defs[i].prep != io_eopnotsupp_prep)
544 BUG_ON(!io_op_defs[i].issue);
545 WARN_ON_ONCE(!io_op_defs[i].name);
546 }
547 }
548
