1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "opdef.h"
13 #include "refs.h"
14 #include "tctx.h"
15 #include "sqpoll.h"
16 #include "fdinfo.h"
17 #include "kbuf.h"
18 #include "rsrc.h"
19
20 #include "xattr.h"
21 #include "nop.h"
22 #include "fs.h"
23 #include "splice.h"
24 #include "sync.h"
25 #include "advise.h"
26 #include "openclose.h"
27 #include "uring_cmd.h"
28 #include "epoll.h"
29 #include "statx.h"
30 #include "net.h"
31 #include "msg_ring.h"
32 #include "timeout.h"
33 #include "poll.h"
34 #include "cancel.h"
35 #include "rw.h"
36
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 {
39 WARN_ON_ONCE(1);
40 return -ECANCELED;
41 }
42
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44 const struct io_uring_sqe *sqe)
45 {
46 return -EOPNOTSUPP;
47 }
48
49 const struct io_op_def io_op_defs[] = {
50 [IORING_OP_NOP] = {
51 .audit_skip = 1,
52 .iopoll = 1,
53 .name = "NOP",
54 .prep = io_nop_prep,
55 .issue = io_nop,
56 },
57 [IORING_OP_READV] = {
58 .needs_file = 1,
59 .unbound_nonreg_file = 1,
60 .pollin = 1,
61 .buffer_select = 1,
62 .plug = 1,
63 .audit_skip = 1,
64 .ioprio = 1,
65 .iopoll = 1,
66 .iopoll_queue = 1,
67 .async_size = sizeof(struct io_async_rw),
68 .name = "READV",
69 .prep = io_prep_rw,
70 .issue = io_read,
71 .prep_async = io_readv_prep_async,
72 .cleanup = io_readv_writev_cleanup,
73 .fail = io_rw_fail,
74 },
75 [IORING_OP_WRITEV] = {
76 .needs_file = 1,
77 .hash_reg_file = 1,
78 .unbound_nonreg_file = 1,
79 .pollout = 1,
80 .plug = 1,
81 .audit_skip = 1,
82 .ioprio = 1,
83 .iopoll = 1,
84 .iopoll_queue = 1,
85 .async_size = sizeof(struct io_async_rw),
86 .name = "WRITEV",
87 .prep = io_prep_rw,
88 .issue = io_write,
89 .prep_async = io_writev_prep_async,
90 .cleanup = io_readv_writev_cleanup,
91 .fail = io_rw_fail,
92 },
93 [IORING_OP_FSYNC] = {
94 .needs_file = 1,
95 .audit_skip = 1,
96 .name = "FSYNC",
97 .prep = io_fsync_prep,
98 .issue = io_fsync,
99 },
100 [IORING_OP_READ_FIXED] = {
101 .needs_file = 1,
102 .unbound_nonreg_file = 1,
103 .pollin = 1,
104 .plug = 1,
105 .audit_skip = 1,
106 .ioprio = 1,
107 .iopoll = 1,
108 .iopoll_queue = 1,
109 .async_size = sizeof(struct io_async_rw),
110 .name = "READ_FIXED",
111 .prep = io_prep_rw,
112 .issue = io_read,
113 .fail = io_rw_fail,
114 },
115 [IORING_OP_WRITE_FIXED] = {
116 .needs_file = 1,
117 .hash_reg_file = 1,
118 .unbound_nonreg_file = 1,
119 .pollout = 1,
120 .plug = 1,
121 .audit_skip = 1,
122 .ioprio = 1,
123 .iopoll = 1,
124 .iopoll_queue = 1,
125 .async_size = sizeof(struct io_async_rw),
126 .name = "WRITE_FIXED",
127 .prep = io_prep_rw,
128 .issue = io_write,
129 .fail = io_rw_fail,
130 },
131 [IORING_OP_POLL_ADD] = {
132 .needs_file = 1,
133 .unbound_nonreg_file = 1,
134 .audit_skip = 1,
135 .name = "POLL_ADD",
136 .prep = io_poll_add_prep,
137 .issue = io_poll_add,
138 },
139 [IORING_OP_POLL_REMOVE] = {
140 .audit_skip = 1,
141 .name = "POLL_REMOVE",
142 .prep = io_poll_remove_prep,
143 .issue = io_poll_remove,
144 },
145 [IORING_OP_SYNC_FILE_RANGE] = {
146 .needs_file = 1,
147 .audit_skip = 1,
148 .name = "SYNC_FILE_RANGE",
149 .prep = io_sfr_prep,
150 .issue = io_sync_file_range,
151 },
152 [IORING_OP_SENDMSG] = {
153 .needs_file = 1,
154 .unbound_nonreg_file = 1,
155 .pollout = 1,
156 .ioprio = 1,
157 .manual_alloc = 1,
158 .name = "SENDMSG",
159 #if defined(CONFIG_NET)
160 .async_size = sizeof(struct io_async_msghdr),
161 .prep = io_sendmsg_prep,
162 .issue = io_sendmsg,
163 .prep_async = io_sendmsg_prep_async,
164 .cleanup = io_sendmsg_recvmsg_cleanup,
165 .fail = io_sendrecv_fail,
166 #else
167 .prep = io_eopnotsupp_prep,
168 #endif
169 },
170 [IORING_OP_RECVMSG] = {
171 .needs_file = 1,
172 .unbound_nonreg_file = 1,
173 .pollin = 1,
174 .buffer_select = 1,
175 .ioprio = 1,
176 .manual_alloc = 1,
177 .name = "RECVMSG",
178 #if defined(CONFIG_NET)
179 .async_size = sizeof(struct io_async_msghdr),
180 .prep = io_recvmsg_prep,
181 .issue = io_recvmsg,
182 .prep_async = io_recvmsg_prep_async,
183 .cleanup = io_sendmsg_recvmsg_cleanup,
184 .fail = io_sendrecv_fail,
185 #else
186 .prep = io_eopnotsupp_prep,
187 #endif
188 },
189 [IORING_OP_TIMEOUT] = {
190 .audit_skip = 1,
191 .async_size = sizeof(struct io_timeout_data),
192 .name = "TIMEOUT",
193 .prep = io_timeout_prep,
194 .issue = io_timeout,
195 },
196 [IORING_OP_TIMEOUT_REMOVE] = {
197 /* used by timeout updates' prep() */
198 .audit_skip = 1,
199 .name = "TIMEOUT_REMOVE",
200 .prep = io_timeout_remove_prep,
201 .issue = io_timeout_remove,
202 },
203 [IORING_OP_ACCEPT] = {
204 .needs_file = 1,
205 .unbound_nonreg_file = 1,
206 .pollin = 1,
207 .poll_exclusive = 1,
208 .ioprio = 1, /* used for flags */
209 .name = "ACCEPT",
210 #if defined(CONFIG_NET)
211 .prep = io_accept_prep,
212 .issue = io_accept,
213 #else
214 .prep = io_eopnotsupp_prep,
215 #endif
216 },
217 [IORING_OP_ASYNC_CANCEL] = {
218 .audit_skip = 1,
219 .name = "ASYNC_CANCEL",
220 .prep = io_async_cancel_prep,
221 .issue = io_async_cancel,
222 },
223 [IORING_OP_LINK_TIMEOUT] = {
224 .audit_skip = 1,
225 .async_size = sizeof(struct io_timeout_data),
226 .name = "LINK_TIMEOUT",
227 .prep = io_link_timeout_prep,
228 .issue = io_no_issue,
229 },
230 [IORING_OP_CONNECT] = {
231 .needs_file = 1,
232 .unbound_nonreg_file = 1,
233 .pollout = 1,
234 .name = "CONNECT",
235 #if defined(CONFIG_NET)
236 .async_size = sizeof(struct io_async_connect),
237 .prep = io_connect_prep,
238 .issue = io_connect,
239 .prep_async = io_connect_prep_async,
240 #else
241 .prep = io_eopnotsupp_prep,
242 #endif
243 },
244 [IORING_OP_FALLOCATE] = {
245 .needs_file = 1,
246 .name = "FALLOCATE",
247 .prep = io_fallocate_prep,
248 .issue = io_fallocate,
249 },
250 [IORING_OP_OPENAT] = {
251 .name = "OPENAT",
252 .prep = io_openat_prep,
253 .issue = io_openat,
254 .cleanup = io_open_cleanup,
255 },
256 [IORING_OP_CLOSE] = {
257 .name = "CLOSE",
258 .prep = io_close_prep,
259 .issue = io_close,
260 },
261 [IORING_OP_FILES_UPDATE] = {
262 .audit_skip = 1,
263 .iopoll = 1,
264 .name = "FILES_UPDATE",
265 .prep = io_files_update_prep,
266 .issue = io_files_update,
267 },
268 [IORING_OP_STATX] = {
269 .audit_skip = 1,
270 .name = "STATX",
271 .prep = io_statx_prep,
272 .issue = io_statx,
273 .cleanup = io_statx_cleanup,
274 },
275 [IORING_OP_READ] = {
276 .needs_file = 1,
277 .unbound_nonreg_file = 1,
278 .pollin = 1,
279 .buffer_select = 1,
280 .plug = 1,
281 .audit_skip = 1,
282 .ioprio = 1,
283 .iopoll = 1,
284 .iopoll_queue = 1,
285 .async_size = sizeof(struct io_async_rw),
286 .name = "READ",
287 .prep = io_prep_rw,
288 .issue = io_read,
289 .fail = io_rw_fail,
290 },
291 [IORING_OP_WRITE] = {
292 .needs_file = 1,
293 .hash_reg_file = 1,
294 .unbound_nonreg_file = 1,
295 .pollout = 1,
296 .plug = 1,
297 .audit_skip = 1,
298 .ioprio = 1,
299 .iopoll = 1,
300 .iopoll_queue = 1,
301 .async_size = sizeof(struct io_async_rw),
302 .name = "WRITE",
303 .prep = io_prep_rw,
304 .issue = io_write,
305 .fail = io_rw_fail,
306 },
307 [IORING_OP_FADVISE] = {
308 .needs_file = 1,
309 .audit_skip = 1,
310 .name = "FADVISE",
311 .prep = io_fadvise_prep,
312 .issue = io_fadvise,
313 },
314 [IORING_OP_MADVISE] = {
315 .name = "MADVISE",
316 .prep = io_madvise_prep,
317 .issue = io_madvise,
318 },
319 [IORING_OP_SEND] = {
320 .needs_file = 1,
321 .unbound_nonreg_file = 1,
322 .pollout = 1,
323 .audit_skip = 1,
324 .ioprio = 1,
325 .manual_alloc = 1,
326 .name = "SEND",
327 #if defined(CONFIG_NET)
328 .async_size = sizeof(struct io_async_msghdr),
329 .prep = io_sendmsg_prep,
330 .issue = io_send,
331 .fail = io_sendrecv_fail,
332 .prep_async = io_send_prep_async,
333 #else
334 .prep = io_eopnotsupp_prep,
335 #endif
336 },
337 [IORING_OP_RECV] = {
338 .needs_file = 1,
339 .unbound_nonreg_file = 1,
340 .pollin = 1,
341 .buffer_select = 1,
342 .audit_skip = 1,
343 .ioprio = 1,
344 .name = "RECV",
345 #if defined(CONFIG_NET)
346 .prep = io_recvmsg_prep,
347 .issue = io_recv,
348 .fail = io_sendrecv_fail,
349 #else
350 .prep = io_eopnotsupp_prep,
351 #endif
352 },
353 [IORING_OP_OPENAT2] = {
354 .name = "OPENAT2",
355 .prep = io_openat2_prep,
356 .issue = io_openat2,
357 .cleanup = io_open_cleanup,
358 },
359 [IORING_OP_EPOLL_CTL] = {
360 .unbound_nonreg_file = 1,
361 .audit_skip = 1,
362 .name = "EPOLL",
363 #if defined(CONFIG_EPOLL)
364 .prep = io_epoll_ctl_prep,
365 .issue = io_epoll_ctl,
366 #else
367 .prep = io_eopnotsupp_prep,
368 #endif
369 },
370 [IORING_OP_SPLICE] = {
371 .needs_file = 1,
372 .hash_reg_file = 1,
373 .unbound_nonreg_file = 1,
374 .audit_skip = 1,
375 .name = "SPLICE",
376 .prep = io_splice_prep,
377 .issue = io_splice,
378 },
379 [IORING_OP_PROVIDE_BUFFERS] = {
380 .audit_skip = 1,
381 .iopoll = 1,
382 .name = "PROVIDE_BUFFERS",
383 .prep = io_provide_buffers_prep,
384 .issue = io_provide_buffers,
385 },
386 [IORING_OP_REMOVE_BUFFERS] = {
387 .audit_skip = 1,
388 .iopoll = 1,
389 .name = "REMOVE_BUFFERS",
390 .prep = io_remove_buffers_prep,
391 .issue = io_remove_buffers,
392 },
393 [IORING_OP_TEE] = {
394 .needs_file = 1,
395 .hash_reg_file = 1,
396 .unbound_nonreg_file = 1,
397 .audit_skip = 1,
398 .name = "TEE",
399 .prep = io_tee_prep,
400 .issue = io_tee,
401 },
402 [IORING_OP_SHUTDOWN] = {
403 .needs_file = 1,
404 .name = "SHUTDOWN",
405 #if defined(CONFIG_NET)
406 .prep = io_shutdown_prep,
407 .issue = io_shutdown,
408 #else
409 .prep = io_eopnotsupp_prep,
410 #endif
411 },
412 [IORING_OP_RENAMEAT] = {
413 .name = "RENAMEAT",
414 .prep = io_renameat_prep,
415 .issue = io_renameat,
416 .cleanup = io_renameat_cleanup,
417 },
418 [IORING_OP_UNLINKAT] = {
419 .name = "UNLINKAT",
420 .prep = io_unlinkat_prep,
421 .issue = io_unlinkat,
422 .cleanup = io_unlinkat_cleanup,
423 },
424 [IORING_OP_MKDIRAT] = {
425 .name = "MKDIRAT",
426 .prep = io_mkdirat_prep,
427 .issue = io_mkdirat,
428 .cleanup = io_mkdirat_cleanup,
429 },
430 [IORING_OP_SYMLINKAT] = {
431 .name = "SYMLINKAT",
432 .prep = io_symlinkat_prep,
433 .issue = io_symlinkat,
434 .cleanup = io_link_cleanup,
435 },
436 [IORING_OP_LINKAT] = {
437 .name = "LINKAT",
438 .prep = io_linkat_prep,
439 .issue = io_linkat,
440 .cleanup = io_link_cleanup,
441 },
442 [IORING_OP_MSG_RING] = {
443 .needs_file = 1,
444 .iopoll = 1,
445 .name = "MSG_RING",
446 .prep = io_msg_ring_prep,
447 .issue = io_msg_ring,
448 },
449 [IORING_OP_FSETXATTR] = {
450 .needs_file = 1,
451 .name = "FSETXATTR",
452 .prep = io_fsetxattr_prep,
453 .issue = io_fsetxattr,
454 .cleanup = io_xattr_cleanup,
455 },
456 [IORING_OP_SETXATTR] = {
457 .name = "SETXATTR",
458 .prep = io_setxattr_prep,
459 .issue = io_setxattr,
460 .cleanup = io_xattr_cleanup,
461 },
462 [IORING_OP_FGETXATTR] = {
463 .needs_file = 1,
464 .name = "FGETXATTR",
465 .prep = io_fgetxattr_prep,
466 .issue = io_fgetxattr,
467 .cleanup = io_xattr_cleanup,
468 },
469 [IORING_OP_GETXATTR] = {
470 .name = "GETXATTR",
471 .prep = io_getxattr_prep,
472 .issue = io_getxattr,
473 .cleanup = io_xattr_cleanup,
474 },
475 [IORING_OP_SOCKET] = {
476 .audit_skip = 1,
477 .name = "SOCKET",
478 #if defined(CONFIG_NET)
479 .prep = io_socket_prep,
480 .issue = io_socket,
481 #else
482 .prep = io_eopnotsupp_prep,
483 #endif
484 },
485 [IORING_OP_URING_CMD] = {
486 .needs_file = 1,
487 .plug = 1,
488 .name = "URING_CMD",
489 .iopoll = 1,
490 .iopoll_queue = 1,
491 .async_size = uring_cmd_pdu_size(1),
492 .prep = io_uring_cmd_prep,
493 .issue = io_uring_cmd,
494 .prep_async = io_uring_cmd_prep_async,
495 },
496 [IORING_OP_SEND_ZC] = {
497 .name = "SEND_ZC",
498 .needs_file = 1,
499 .unbound_nonreg_file = 1,
500 .pollout = 1,
501 .audit_skip = 1,
502 .ioprio = 1,
503 .manual_alloc = 1,
504 #if defined(CONFIG_NET)
505 .async_size = sizeof(struct io_async_msghdr),
506 .prep = io_send_zc_prep,
507 .issue = io_send_zc,
508 .prep_async = io_send_prep_async,
509 .cleanup = io_send_zc_cleanup,
510 .fail = io_sendrecv_fail,
511 #else
512 .prep = io_eopnotsupp_prep,
513 #endif
514 },
515 [IORING_OP_SENDMSG_ZC] = {
516 .name = "SENDMSG_ZC",
517 .needs_file = 1,
518 .unbound_nonreg_file = 1,
519 .pollout = 1,
520 .ioprio = 1,
521 .manual_alloc = 1,
522 #if defined(CONFIG_NET)
523 .async_size = sizeof(struct io_async_msghdr),
524 .prep = io_send_zc_prep,
525 .issue = io_sendmsg_zc,
526 .prep_async = io_sendmsg_prep_async,
527 .cleanup = io_send_zc_cleanup,
528 .fail = io_sendrecv_fail,
529 #else
530 .prep = io_eopnotsupp_prep,
531 #endif
532 },
533 };
534
io_uring_get_opcode(u8 opcode)535 const char *io_uring_get_opcode(u8 opcode)
536 {
537 if (opcode < IORING_OP_LAST)
538 return io_op_defs[opcode].name;
539 return "INVALID";
540 }
541
io_uring_optable_init(void)542 void __init io_uring_optable_init(void)
543 {
544 int i;
545
546 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
547
548 for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
549 BUG_ON(!io_op_defs[i].prep);
550 if (io_op_defs[i].prep != io_eopnotsupp_prep)
551 BUG_ON(!io_op_defs[i].issue);
552 WARN_ON_ONCE(!io_op_defs[i].name);
553 }
554 }
555