1/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
2/*
3 * Header file for the io_uring interface.
4 *
5 * Copyright (C) 2019 Jens Axboe
6 * Copyright (C) 2019 Christoph Hellwig
7 */
8#ifndef LINUX_IO_URING_H
9#define LINUX_IO_URING_H
10
11#include <linux/fs.h>
12#include <linux/types.h>
13#include <linux/io_uring/zcrx.h>
14
15/*
16 * this file is shared with liburing and that has to autodetect
17 * if linux/time_types.h is available or not, it can
18 * define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
19 * if linux/time_types.h is not available
20 */
21#ifndef UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
22#include <linux/time_types.h>
23#endif
24
25#ifdef __cplusplus
26extern "C" {
27#endif
28
29/*
30 * IO submission data structure (Submission Queue Entry)
31 */
32struct io_uring_sqe {
33 __u8 opcode; /* type of operation for this sqe */
34 __u8 flags; /* IOSQE_ flags */
35 __u16 ioprio; /* ioprio for the request */
36 __s32 fd; /* file descriptor to do IO on */
37 union {
38 __u64 off; /* offset into file */
39 __u64 addr2;
40 struct {
41 __u32 cmd_op;
42 __u32 __pad1;
43 };
44 };
45 union {
46 __u64 addr; /* pointer to buffer or iovecs */
47 __u64 splice_off_in;
48 struct {
49 __u32 level;
50 __u32 optname;
51 };
52 };
53 __u32 len; /* buffer size or number of iovecs */
54 union {
55 __u32 rw_flags;
56 __u32 fsync_flags;
57 __u16 poll_events; /* compatibility */
58 __u32 poll32_events; /* word-reversed for BE */
59 __u32 sync_range_flags;
60 __u32 msg_flags;
61 __u32 timeout_flags;
62 __u32 accept_flags;
63 __u32 cancel_flags;
64 __u32 open_flags;
65 __u32 statx_flags;
66 __u32 fadvise_advice;
67 __u32 splice_flags;
68 __u32 rename_flags;
69 __u32 unlink_flags;
70 __u32 hardlink_flags;
71 __u32 xattr_flags;
72 __u32 msg_ring_flags;
73 __u32 uring_cmd_flags;
74 __u32 waitid_flags;
75 __u32 futex_flags;
76 __u32 install_fd_flags;
77 __u32 nop_flags;
78 __u32 pipe_flags;
79 };
80 __u64 user_data; /* data to be passed back at completion time */
81 /* pack this to avoid bogus arm OABI complaints */
82 union {
83 /* index into fixed buffers, if used */
84 __u16 buf_index;
85 /* for grouped buffer selection */
86 __u16 buf_group;
87 } __attribute__((packed));
88 /* personality to use, if used */
89 __u16 personality;
90 union {
91 __s32 splice_fd_in;
92 __u32 file_index;
93 __u32 zcrx_ifq_idx;
94 __u32 optlen;
95 struct {
96 __u16 addr_len;
97 __u16 __pad3[1];
98 };
99 struct {
100 __u8 write_stream;
101 __u8 __pad4[3];
102 };
103 };
104 union {
105 struct {
106 __u64 addr3;
107 __u64 __pad2[1];
108 };
109 struct {
110 __u64 attr_ptr; /* pointer to attribute information */
111 __u64 attr_type_mask; /* bit mask of attributes */
112 };
113 __u64 optval;
114 /*
115 * If the ring is initialized with IORING_SETUP_SQE128, then
116 * this field is used for 80 bytes of arbitrary command data
117 */
118 __u8 cmd[0];
119 };
120};
121
122/* sqe->attr_type_mask flags */
123#define IORING_RW_ATTR_FLAG_PI (1U << 0)
124/* PI attribute information */
125struct io_uring_attr_pi {
126 __u16 flags;
127 __u16 app_tag;
128 __u32 len;
129 __u64 addr;
130 __u64 seed;
131 __u64 rsvd;
132};
133
134/*
135 * If sqe->file_index is set to this for opcodes that instantiate a new
136 * direct descriptor (like openat/openat2/accept), then io_uring will allocate
137 * an available direct descriptor instead of having the application pass one
138 * in. The picked direct descriptor will be returned in cqe->res, or -ENFILE
139 * if the space is full.
140 */
141#define IORING_FILE_INDEX_ALLOC (~0U)
142
143enum io_uring_sqe_flags_bit {
144 IOSQE_FIXED_FILE_BIT,
145 IOSQE_IO_DRAIN_BIT,
146 IOSQE_IO_LINK_BIT,
147 IOSQE_IO_HARDLINK_BIT,
148 IOSQE_ASYNC_BIT,
149 IOSQE_BUFFER_SELECT_BIT,
150 IOSQE_CQE_SKIP_SUCCESS_BIT,
151};
152
153/*
154 * sqe->flags
155 */
156/* use fixed fileset */
157#define IOSQE_FIXED_FILE (1U << IOSQE_FIXED_FILE_BIT)
158/* issue after inflight IO */
159#define IOSQE_IO_DRAIN (1U << IOSQE_IO_DRAIN_BIT)
160/* links next sqe */
161#define IOSQE_IO_LINK (1U << IOSQE_IO_LINK_BIT)
162/* like LINK, but stronger */
163#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT)
164/* always go async */
165#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT)
166/* select buffer from sqe->buf_group */
167#define IOSQE_BUFFER_SELECT (1U << IOSQE_BUFFER_SELECT_BIT)
168/* don't post CQE if request succeeded */
169#define IOSQE_CQE_SKIP_SUCCESS (1U << IOSQE_CQE_SKIP_SUCCESS_BIT)
170
171/*
172 * io_uring_setup() flags
173 */
174#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */
175#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */
176#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */
177#define IORING_SETUP_CQSIZE (1U << 3) /* app defines CQ size */
178#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */
179#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
180#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */
181#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */
182/*
183 * Cooperative task running. When requests complete, they often require
184 * forcing the submitter to transition to the kernel to complete. If this
185 * flag is set, work will be done when the task transitions anyway, rather
186 * than force an inter-processor interrupt reschedule. This avoids interrupting
187 * a task running in userspace, and saves an IPI.
188 */
189#define IORING_SETUP_COOP_TASKRUN (1U << 8)
190/*
191 * If COOP_TASKRUN is set, get notified if task work is available for
192 * running and a kernel transition would be needed to run it. This sets
193 * IORING_SQ_TASKRUN in the sq ring flags. Not valid without COOP_TASKRUN
194 * or DEFER_TASKRUN.
195 */
196#define IORING_SETUP_TASKRUN_FLAG (1U << 9)
197#define IORING_SETUP_SQE128 (1U << 10) /* SQEs are 128 byte */
198#define IORING_SETUP_CQE32 (1U << 11) /* CQEs are 32 byte */
199/*
200 * Only one task is allowed to submit requests
201 */
202#define IORING_SETUP_SINGLE_ISSUER (1U << 12)
203
204/*
205 * Defer running task work to get events.
206 * Rather than running bits of task work whenever the task transitions
207 * try to do it just before it is needed.
208 */
209#define IORING_SETUP_DEFER_TASKRUN (1U << 13)
210
211/*
212 * Application provides the memory for the rings
213 */
214#define IORING_SETUP_NO_MMAP (1U << 14)
215
216/*
217 * Register the ring fd in itself for use with
218 * IORING_REGISTER_USE_REGISTERED_RING; return a registered fd index rather
219 * than an fd.
220 */
221#define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15)
222
223/*
224 * Removes indirection through the SQ index array.
225 */
226#define IORING_SETUP_NO_SQARRAY (1U << 16)
227
228/* Use hybrid poll in iopoll process */
229#define IORING_SETUP_HYBRID_IOPOLL (1U << 17)
230
231/*
232 * Allow both 16b and 32b CQEs. If a 32b CQE is posted, it will have
233 * IORING_CQE_F_32 set in cqe->flags.
234 */
235#define IORING_SETUP_CQE_MIXED (1U << 18)
236
237/*
238 * Allow both 64b and 128b SQEs. If a 128b SQE is posted, it will have
239 * a 128b opcode.
240 */
241#define IORING_SETUP_SQE_MIXED (1U << 19)
242
243/*
244 * When set, io_uring ignores SQ head and tail and fetches SQEs to submit
245 * starting from index 0 instead from the index stored in the head pointer.
246 * IOW, the user should place all SQE at the beginning of the SQ memory
247 * before issuing a submission syscall.
248 *
249 * It requires IORING_SETUP_NO_SQARRAY and is incompatible with
250 * IORING_SETUP_SQPOLL. The user must also never change the SQ head and tail
251 * values and keep it set to 0. Any other value is undefined behaviour.
252 */
253#define IORING_SETUP_SQ_REWIND (1U << 20)
254
255enum io_uring_op {
256 IORING_OP_NOP,
257 IORING_OP_READV,
258 IORING_OP_WRITEV,
259 IORING_OP_FSYNC,
260 IORING_OP_READ_FIXED,
261 IORING_OP_WRITE_FIXED,
262 IORING_OP_POLL_ADD,
263 IORING_OP_POLL_REMOVE,
264 IORING_OP_SYNC_FILE_RANGE,
265 IORING_OP_SENDMSG,
266 IORING_OP_RECVMSG,
267 IORING_OP_TIMEOUT,
268 IORING_OP_TIMEOUT_REMOVE,
269 IORING_OP_ACCEPT,
270 IORING_OP_ASYNC_CANCEL,
271 IORING_OP_LINK_TIMEOUT,
272 IORING_OP_CONNECT,
273 IORING_OP_FALLOCATE,
274 IORING_OP_OPENAT,
275 IORING_OP_CLOSE,
276 IORING_OP_FILES_UPDATE,
277 IORING_OP_STATX,
278 IORING_OP_READ,
279 IORING_OP_WRITE,
280 IORING_OP_FADVISE,
281 IORING_OP_MADVISE,
282 IORING_OP_SEND,
283 IORING_OP_RECV,
284 IORING_OP_OPENAT2,
285 IORING_OP_EPOLL_CTL,
286 IORING_OP_SPLICE,
287 IORING_OP_PROVIDE_BUFFERS,
288 IORING_OP_REMOVE_BUFFERS,
289 IORING_OP_TEE,
290 IORING_OP_SHUTDOWN,
291 IORING_OP_RENAMEAT,
292 IORING_OP_UNLINKAT,
293 IORING_OP_MKDIRAT,
294 IORING_OP_SYMLINKAT,
295 IORING_OP_LINKAT,
296 IORING_OP_MSG_RING,
297 IORING_OP_FSETXATTR,
298 IORING_OP_SETXATTR,
299 IORING_OP_FGETXATTR,
300 IORING_OP_GETXATTR,
301 IORING_OP_SOCKET,
302 IORING_OP_URING_CMD,
303 IORING_OP_SEND_ZC,
304 IORING_OP_SENDMSG_ZC,
305 IORING_OP_READ_MULTISHOT,
306 IORING_OP_WAITID,
307 IORING_OP_FUTEX_WAIT,
308 IORING_OP_FUTEX_WAKE,
309 IORING_OP_FUTEX_WAITV,
310 IORING_OP_FIXED_FD_INSTALL,
311 IORING_OP_FTRUNCATE,
312 IORING_OP_BIND,
313 IORING_OP_LISTEN,
314 IORING_OP_RECV_ZC,
315 IORING_OP_EPOLL_WAIT,
316 IORING_OP_READV_FIXED,
317 IORING_OP_WRITEV_FIXED,
318 IORING_OP_PIPE,
319 IORING_OP_NOP128,
320 IORING_OP_URING_CMD128,
321
322 /* this goes last, obviously */
323 IORING_OP_LAST,
324};
325
326/*
327 * sqe->uring_cmd_flags top 8bits aren't available for userspace
328 * IORING_URING_CMD_FIXED use registered buffer; pass this flag
329 * along with setting sqe->buf_index.
330 * IORING_URING_CMD_MULTISHOT must be used with buffer select, like other
331 * multishot commands. Not compatible with
332 * IORING_URING_CMD_FIXED, for now.
333 */
334#define IORING_URING_CMD_FIXED (1U << 0)
335#define IORING_URING_CMD_MULTISHOT (1U << 1)
336#define IORING_URING_CMD_MASK (IORING_URING_CMD_FIXED | IORING_URING_CMD_MULTISHOT)
337
338
339/*
340 * sqe->fsync_flags
341 */
342#define IORING_FSYNC_DATASYNC (1U << 0)
343
344/*
345 * sqe->timeout_flags
346 *
347 * IORING_TIMEOUT_IMMEDIATE_ARG: If set, sqe->addr stores the timeout
348 * value in nanoseconds instead of
349 * pointing to a timespec.
350 */
351#define IORING_TIMEOUT_ABS (1U << 0)
352#define IORING_TIMEOUT_UPDATE (1U << 1)
353#define IORING_TIMEOUT_BOOTTIME (1U << 2)
354#define IORING_TIMEOUT_REALTIME (1U << 3)
355#define IORING_LINK_TIMEOUT_UPDATE (1U << 4)
356#define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5)
357#define IORING_TIMEOUT_MULTISHOT (1U << 6)
358#define IORING_TIMEOUT_IMMEDIATE_ARG (1U << 7)
359#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
360#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
361/*
362 * sqe->splice_flags
363 * extends splice(2) flags
364 */
365#define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */
366
367/*
368 * POLL_ADD flags. Note that since sqe->poll_events is the flag space, the
369 * command flags for POLL_ADD are stored in sqe->len.
370 *
371 * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if
372 * the poll handler will continue to report
373 * CQEs on behalf of the same SQE.
374 *
375 * IORING_POLL_UPDATE Update existing poll request, matching
376 * sqe->addr as the old user_data field.
377 *
378 * IORING_POLL_LEVEL Level triggered poll.
379 */
380#define IORING_POLL_ADD_MULTI (1U << 0)
381#define IORING_POLL_UPDATE_EVENTS (1U << 1)
382#define IORING_POLL_UPDATE_USER_DATA (1U << 2)
383#define IORING_POLL_ADD_LEVEL (1U << 3)
384
385/*
386 * ASYNC_CANCEL flags.
387 *
388 * IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key
389 * IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the
390 * request 'user_data'
391 * IORING_ASYNC_CANCEL_ANY Match any request
392 * IORING_ASYNC_CANCEL_FD_FIXED 'fd' passed in is a fixed descriptor
393 * IORING_ASYNC_CANCEL_USERDATA Match on user_data, default for no other key
394 * IORING_ASYNC_CANCEL_OP Match request based on opcode
395 */
396#define IORING_ASYNC_CANCEL_ALL (1U << 0)
397#define IORING_ASYNC_CANCEL_FD (1U << 1)
398#define IORING_ASYNC_CANCEL_ANY (1U << 2)
399#define IORING_ASYNC_CANCEL_FD_FIXED (1U << 3)
400#define IORING_ASYNC_CANCEL_USERDATA (1U << 4)
401#define IORING_ASYNC_CANCEL_OP (1U << 5)
402
403/*
404 * send/sendmsg and recv/recvmsg flags (sqe->ioprio)
405 *
406 * IORING_RECVSEND_POLL_FIRST If set, instead of first attempting to send
407 * or receive and arm poll if that yields an
408 * -EAGAIN result, arm poll upfront and skip
409 * the initial transfer attempt.
410 *
411 * IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if
412 * the handler will continue to report
413 * CQEs on behalf of the same SQE.
414 *
415 * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in
416 * the buf_index field.
417 *
418 * IORING_SEND_ZC_REPORT_USAGE
419 * If set, SEND[MSG]_ZC should report
420 * the zerocopy usage in cqe.res
421 * for the IORING_CQE_F_NOTIF cqe.
422 * 0 is reported if zerocopy was actually possible.
423 * IORING_NOTIF_USAGE_ZC_COPIED if data was copied
424 * (at least partially).
425 *
426 * IORING_RECVSEND_BUNDLE Used with IOSQE_BUFFER_SELECT. If set, send or
427 * recv will grab as many buffers from the buffer
428 * group ID given and send them all. The completion
429 * result will be the number of buffers send, with
430 * the starting buffer ID in cqe->flags as per
431 * usual for provided buffer usage. The buffers
432 * will be contiguous from the starting buffer ID.
433 *
434 * IORING_SEND_VECTORIZED If set, SEND[_ZC] will take a pointer to a io_vec
435 * to allow vectorized send operations.
436 */
437#define IORING_RECVSEND_POLL_FIRST (1U << 0)
438#define IORING_RECV_MULTISHOT (1U << 1)
439#define IORING_RECVSEND_FIXED_BUF (1U << 2)
440#define IORING_SEND_ZC_REPORT_USAGE (1U << 3)
441#define IORING_RECVSEND_BUNDLE (1U << 4)
442#define IORING_SEND_VECTORIZED (1U << 5)
443
444/*
445 * cqe.res for IORING_CQE_F_NOTIF if
446 * IORING_SEND_ZC_REPORT_USAGE was requested
447 *
448 * It should be treated as a flag, all other
449 * bits of cqe.res should be treated as reserved!
450 */
451#define IORING_NOTIF_USAGE_ZC_COPIED (1U << 31)
452
453/*
454 * accept flags stored in sqe->ioprio
455 */
456#define IORING_ACCEPT_MULTISHOT (1U << 0)
457#define IORING_ACCEPT_DONTWAIT (1U << 1)
458#define IORING_ACCEPT_POLL_FIRST (1U << 2)
459
460/*
461 * IORING_OP_MSG_RING command types, stored in sqe->addr
462 */
463enum io_uring_msg_ring_flags {
464 IORING_MSG_DATA, /* pass sqe->len as 'res' and off as user_data */
465 IORING_MSG_SEND_FD, /* send a registered fd to another ring */
466};
467
468/*
469 * IORING_OP_MSG_RING flags (sqe->msg_ring_flags)
470 *
471 * IORING_MSG_RING_CQE_SKIP Don't post a CQE to the target ring. Not
472 * applicable for IORING_MSG_DATA, obviously.
473 */
474#define IORING_MSG_RING_CQE_SKIP (1U << 0)
475/* Pass through the flags from sqe->file_index to cqe->flags */
476#define IORING_MSG_RING_FLAGS_PASS (1U << 1)
477
478/*
479 * IORING_OP_FIXED_FD_INSTALL flags (sqe->install_fd_flags)
480 *
481 * IORING_FIXED_FD_NO_CLOEXEC Don't mark the fd as O_CLOEXEC
482 */
483#define IORING_FIXED_FD_NO_CLOEXEC (1U << 0)
484
485/*
486 * IORING_OP_NOP flags (sqe->nop_flags)
487 *
488 * IORING_NOP_INJECT_RESULT Inject result from sqe->result
489 */
490#define IORING_NOP_INJECT_RESULT (1U << 0)
491#define IORING_NOP_FILE (1U << 1)
492#define IORING_NOP_FIXED_FILE (1U << 2)
493#define IORING_NOP_FIXED_BUFFER (1U << 3)
494#define IORING_NOP_TW (1U << 4)
495#define IORING_NOP_CQE32 (1U << 5)
496
497/*
498 * IO completion data structure (Completion Queue Entry)
499 */
500struct io_uring_cqe {
501 __u64 user_data; /* sqe->user_data value passed back */
502 __s32 res; /* result code for this event */
503 __u32 flags;
504
505 /*
506 * If the ring is initialized with IORING_SETUP_CQE32, then this field
507 * contains 16-bytes of padding, doubling the size of the CQE.
508 */
509 __u64 big_cqe[];
510};
511
512/*
513 * cqe->flags
514 *
515 * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
516 * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries
517 * IORING_CQE_F_SOCK_NONEMPTY If set, more data to read after socket recv
518 * IORING_CQE_F_NOTIF Set for notification CQEs. Can be used to distinct
519 * them from sends.
520 * IORING_CQE_F_BUF_MORE If set, the buffer ID set in the completion will get
521 * more completions. In other words, the buffer is being
522 * partially consumed, and will be used by the kernel for
523 * more completions. This is only set for buffers used via
524 * the incremental buffer consumption, as provided by
525 * a ring buffer setup with IOU_PBUF_RING_INC. For any
526 * other provided buffer type, all completions with a
527 * buffer passed back is automatically returned to the
528 * application.
529 * IORING_CQE_F_SKIP If set, then the application/liburing must ignore this
530 * CQE. It's only purpose is to fill a gap in the ring,
531 * if a large CQE is attempted posted when the ring has
532 * just a single small CQE worth of space left before
533 * wrapping.
534 * IORING_CQE_F_32 If set, this is a 32b/big-cqe posting. Use with rings
535 * setup in a mixed CQE mode, where both 16b and 32b
536 * CQEs may be posted to the CQ ring.
537 */
538#define IORING_CQE_F_BUFFER (1U << 0)
539#define IORING_CQE_F_MORE (1U << 1)
540#define IORING_CQE_F_SOCK_NONEMPTY (1U << 2)
541#define IORING_CQE_F_NOTIF (1U << 3)
542#define IORING_CQE_F_BUF_MORE (1U << 4)
543#define IORING_CQE_F_SKIP (1U << 5)
544#define IORING_CQE_F_32 (1U << 15)
545
546#define IORING_CQE_BUFFER_SHIFT 16
547
548/*
549 * Magic offsets for the application to mmap the data it needs
550 */
551#define IORING_OFF_SQ_RING 0ULL
552#define IORING_OFF_CQ_RING 0x8000000ULL
553#define IORING_OFF_SQES 0x10000000ULL
554#define IORING_OFF_PBUF_RING 0x80000000ULL
555#define IORING_OFF_PBUF_SHIFT 16
556#define IORING_OFF_MMAP_MASK 0xf8000000ULL
557
558/*
559 * Filled with the offset for mmap(2)
560 */
561struct io_sqring_offsets {
562 __u32 head;
563 __u32 tail;
564 __u32 ring_mask;
565 __u32 ring_entries;
566 __u32 flags;
567 __u32 dropped;
568 __u32 array;
569 __u32 resv1;
570 __u64 user_addr;
571};
572
573/*
574 * sq_ring->flags
575 */
576#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
577#define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */
578#define IORING_SQ_TASKRUN (1U << 2) /* task should enter the kernel */
579
580struct io_cqring_offsets {
581 __u32 head;
582 __u32 tail;
583 __u32 ring_mask;
584 __u32 ring_entries;
585 __u32 overflow;
586 __u32 cqes;
587 __u32 flags;
588 __u32 resv1;
589 __u64 user_addr;
590};
591
592/*
593 * cq_ring->flags
594 */
595
596/* disable eventfd notifications */
597#define IORING_CQ_EVENTFD_DISABLED (1U << 0)
598
599/*
600 * io_uring_enter(2) flags
601 */
602#define IORING_ENTER_GETEVENTS (1U << 0)
603#define IORING_ENTER_SQ_WAKEUP (1U << 1)
604#define IORING_ENTER_SQ_WAIT (1U << 2)
605#define IORING_ENTER_EXT_ARG (1U << 3)
606#define IORING_ENTER_REGISTERED_RING (1U << 4)
607#define IORING_ENTER_ABS_TIMER (1U << 5)
608#define IORING_ENTER_EXT_ARG_REG (1U << 6)
609#define IORING_ENTER_NO_IOWAIT (1U << 7)
610
611/*
612 * Passed in for io_uring_setup(2). Copied back with updated info on success
613 */
614struct io_uring_params {
615 __u32 sq_entries;
616 __u32 cq_entries;
617 __u32 flags;
618 __u32 sq_thread_cpu;
619 __u32 sq_thread_idle;
620 __u32 features;
621 __u32 wq_fd;
622 __u32 resv[3];
623 struct io_sqring_offsets sq_off;
624 struct io_cqring_offsets cq_off;
625};
626
627/*
628 * io_uring_params->features flags
629 */
630#define IORING_FEAT_SINGLE_MMAP (1U << 0)
631#define IORING_FEAT_NODROP (1U << 1)
632#define IORING_FEAT_SUBMIT_STABLE (1U << 2)
633#define IORING_FEAT_RW_CUR_POS (1U << 3)
634#define IORING_FEAT_CUR_PERSONALITY (1U << 4)
635#define IORING_FEAT_FAST_POLL (1U << 5)
636#define IORING_FEAT_POLL_32BITS (1U << 6)
637#define IORING_FEAT_SQPOLL_NONFIXED (1U << 7)
638#define IORING_FEAT_EXT_ARG (1U << 8)
639#define IORING_FEAT_NATIVE_WORKERS (1U << 9)
640#define IORING_FEAT_RSRC_TAGS (1U << 10)
641#define IORING_FEAT_CQE_SKIP (1U << 11)
642#define IORING_FEAT_LINKED_FILE (1U << 12)
643#define IORING_FEAT_REG_REG_RING (1U << 13)
644#define IORING_FEAT_RECVSEND_BUNDLE (1U << 14)
645#define IORING_FEAT_MIN_TIMEOUT (1U << 15)
646#define IORING_FEAT_RW_ATTR (1U << 16)
647#define IORING_FEAT_NO_IOWAIT (1U << 17)
648
649/*
650 * io_uring_register(2) opcodes and arguments
651 */
652enum io_uring_register_op {
653 IORING_REGISTER_BUFFERS = 0,
654 IORING_UNREGISTER_BUFFERS = 1,
655 IORING_REGISTER_FILES = 2,
656 IORING_UNREGISTER_FILES = 3,
657 IORING_REGISTER_EVENTFD = 4,
658 IORING_UNREGISTER_EVENTFD = 5,
659 IORING_REGISTER_FILES_UPDATE = 6,
660 IORING_REGISTER_EVENTFD_ASYNC = 7,
661 IORING_REGISTER_PROBE = 8,
662 IORING_REGISTER_PERSONALITY = 9,
663 IORING_UNREGISTER_PERSONALITY = 10,
664 IORING_REGISTER_RESTRICTIONS = 11,
665 IORING_REGISTER_ENABLE_RINGS = 12,
666
667 /* extended with tagging */
668 IORING_REGISTER_FILES2 = 13,
669 IORING_REGISTER_FILES_UPDATE2 = 14,
670 IORING_REGISTER_BUFFERS2 = 15,
671 IORING_REGISTER_BUFFERS_UPDATE = 16,
672
673 /* set/clear io-wq thread affinities */
674 IORING_REGISTER_IOWQ_AFF = 17,
675 IORING_UNREGISTER_IOWQ_AFF = 18,
676
677 /* set/get max number of io-wq workers */
678 IORING_REGISTER_IOWQ_MAX_WORKERS = 19,
679
680 /* register/unregister io_uring fd with the ring */
681 IORING_REGISTER_RING_FDS = 20,
682 IORING_UNREGISTER_RING_FDS = 21,
683
684 /* register ring based provide buffer group */
685 IORING_REGISTER_PBUF_RING = 22,
686 IORING_UNREGISTER_PBUF_RING = 23,
687
688 /* sync cancelation API */
689 IORING_REGISTER_SYNC_CANCEL = 24,
690
691 /* register a range of fixed file slots for automatic slot allocation */
692 IORING_REGISTER_FILE_ALLOC_RANGE = 25,
693
694 /* return status information for a buffer group */
695 IORING_REGISTER_PBUF_STATUS = 26,
696
697 /* set/clear busy poll settings */
698 IORING_REGISTER_NAPI = 27,
699 IORING_UNREGISTER_NAPI = 28,
700
701 IORING_REGISTER_CLOCK = 29,
702
703 /* clone registered buffers from source ring to current ring */
704 IORING_REGISTER_CLONE_BUFFERS = 30,
705
706 /* send MSG_RING without having a ring */
707 IORING_REGISTER_SEND_MSG_RING = 31,
708
709 /* register a netdev hw rx queue for zerocopy */
710 IORING_REGISTER_ZCRX_IFQ = 32,
711
712 /* resize CQ ring */
713 IORING_REGISTER_RESIZE_RINGS = 33,
714
715 IORING_REGISTER_MEM_REGION = 34,
716
717 /* query various aspects of io_uring, see linux/io_uring/query.h */
718 IORING_REGISTER_QUERY = 35,
719
720 /* auxiliary zcrx configuration, see enum zcrx_ctrl_op */
721 IORING_REGISTER_ZCRX_CTRL = 36,
722
723 /* register bpf filtering programs */
724 IORING_REGISTER_BPF_FILTER = 37,
725
726 /* this goes last */
727 IORING_REGISTER_LAST,
728
729 /* flag added to the opcode to use a registered ring fd */
730 IORING_REGISTER_USE_REGISTERED_RING = 1U << 31
731};
732
733/* io-wq worker categories */
734enum io_wq_type {
735 IO_WQ_BOUND,
736 IO_WQ_UNBOUND,
737};
738
739/* deprecated, see struct io_uring_rsrc_update */
740struct io_uring_files_update {
741 __u32 offset;
742 __u32 resv;
743 __aligned_u64 /* __s32 * */ fds;
744};
745
746enum {
747 /* initialise with user provided memory pointed by user_addr */
748 IORING_MEM_REGION_TYPE_USER = 1,
749};
750
751struct io_uring_region_desc {
752 __u64 user_addr;
753 __u64 size;
754 __u32 flags;
755 __u32 id;
756 __u64 mmap_offset;
757 __u64 __resv[4];
758};
759
760enum {
761 /* expose the region as registered wait arguments */
762 IORING_MEM_REGION_REG_WAIT_ARG = 1,
763};
764
765struct io_uring_mem_region_reg {
766 __u64 region_uptr; /* struct io_uring_region_desc * */
767 __u64 flags;
768 __u64 __resv[2];
769};
770
771/*
772 * Register a fully sparse file space, rather than pass in an array of all
773 * -1 file descriptors.
774 */
775#define IORING_RSRC_REGISTER_SPARSE (1U << 0)
776
777struct io_uring_rsrc_register {
778 __u32 nr;
779 __u32 flags;
780 __u64 resv2;
781 __aligned_u64 data;
782 __aligned_u64 tags;
783};
784
785struct io_uring_rsrc_update {
786 __u32 offset;
787 __u32 resv;
788 __aligned_u64 data;
789};
790
791struct io_uring_rsrc_update2 {
792 __u32 offset;
793 __u32 resv;
794 __aligned_u64 data;
795 __aligned_u64 tags;
796 __u32 nr;
797 __u32 resv2;
798};
799
800/* Skip updating fd indexes set to this value in the fd table */
801#define IORING_REGISTER_FILES_SKIP (-2)
802
803#define IO_URING_OP_SUPPORTED (1U << 0)
804
805struct io_uring_probe_op {
806 __u8 op;
807 __u8 resv;
808 __u16 flags; /* IO_URING_OP_* flags */
809 __u32 resv2;
810};
811
812struct io_uring_probe {
813 __u8 last_op; /* last opcode supported */
814 __u8 ops_len; /* length of ops[] array below */
815 __u16 resv;
816 __u32 resv2[3];
817 struct io_uring_probe_op ops[];
818};
819
820struct io_uring_restriction {
821 __u16 opcode;
822 union {
823 __u8 register_op; /* IORING_RESTRICTION_REGISTER_OP */
824 __u8 sqe_op; /* IORING_RESTRICTION_SQE_OP */
825 __u8 sqe_flags; /* IORING_RESTRICTION_SQE_FLAGS_* */
826 };
827 __u8 resv;
828 __u32 resv2[3];
829};
830
831struct io_uring_task_restriction {
832 __u16 flags;
833 __u16 nr_res;
834 __u32 resv[3];
835 __DECLARE_FLEX_ARRAY(struct io_uring_restriction, restrictions);
836};
837
838struct io_uring_clock_register {
839 __u32 clockid;
840 __u32 __resv[3];
841};
842
843enum {
844 IORING_REGISTER_SRC_REGISTERED = (1U << 0),
845 IORING_REGISTER_DST_REPLACE = (1U << 1),
846};
847
848struct io_uring_clone_buffers {
849 __u32 src_fd;
850 __u32 flags;
851 __u32 src_off;
852 __u32 dst_off;
853 __u32 nr;
854 __u32 pad[3];
855};
856
857struct io_uring_buf {
858 __u64 addr;
859 __u32 len;
860 __u16 bid;
861 __u16 resv;
862};
863
864struct io_uring_buf_ring {
865 union {
866 /*
867 * To avoid spilling into more pages than we need to, the
868 * ring tail is overlaid with the io_uring_buf->resv field.
869 */
870 struct {
871 __u64 resv1;
872 __u32 resv2;
873 __u16 resv3;
874 __u16 tail;
875 };
876 __DECLARE_FLEX_ARRAY(struct io_uring_buf, bufs);
877 };
878};
879
880/*
881 * Flags for IORING_REGISTER_PBUF_RING.
882 *
883 * IOU_PBUF_RING_MMAP: If set, kernel will allocate the memory for the ring.
884 * The application must not set a ring_addr in struct
885 * io_uring_buf_reg, instead it must subsequently call
886 * mmap(2) with the offset set as:
887 * IORING_OFF_PBUF_RING | (bgid << IORING_OFF_PBUF_SHIFT)
888 * to get a virtual mapping for the ring.
889 * IOU_PBUF_RING_INC: If set, buffers consumed from this buffer ring can be
890 * consumed incrementally. Normally one (or more) buffers
891 * are fully consumed. With incremental consumptions, it's
892 * feasible to register big ranges of buffers, and each
893 * use of it will consume only as much as it needs. This
894 * requires that both the kernel and application keep
895 * track of where the current read/recv index is at.
896 */
897enum io_uring_register_pbuf_ring_flags {
898 IOU_PBUF_RING_MMAP = 1,
899 IOU_PBUF_RING_INC = 2,
900};
901
902/* argument for IORING_(UN)REGISTER_PBUF_RING */
903struct io_uring_buf_reg {
904 __u64 ring_addr;
905 __u32 ring_entries;
906 __u16 bgid;
907 __u16 flags;
908 __u32 min_left;
909 __u32 resv[5];
910};
911
912/* argument for IORING_REGISTER_PBUF_STATUS */
913struct io_uring_buf_status {
914 __u32 buf_group; /* input */
915 __u32 head; /* output */
916 __u32 resv[8];
917};
918
919enum io_uring_napi_op {
920 /* register/ungister backward compatible opcode */
921 IO_URING_NAPI_REGISTER_OP = 0,
922
923 /* opcodes to update napi_list when static tracking is used */
924 IO_URING_NAPI_STATIC_ADD_ID = 1,
925 IO_URING_NAPI_STATIC_DEL_ID = 2
926};
927
928enum io_uring_napi_tracking_strategy {
929 /* value must be 0 for backward compatibility */
930 IO_URING_NAPI_TRACKING_DYNAMIC = 0,
931 IO_URING_NAPI_TRACKING_STATIC = 1,
932 IO_URING_NAPI_TRACKING_INACTIVE = 255
933};
934
935/* argument for IORING_(UN)REGISTER_NAPI */
936struct io_uring_napi {
937 __u32 busy_poll_to;
938 __u8 prefer_busy_poll;
939
940 /* a io_uring_napi_op value */
941 __u8 opcode;
942 __u8 pad[2];
943
944 /*
945 * for IO_URING_NAPI_REGISTER_OP, it is a
946 * io_uring_napi_tracking_strategy value.
947 *
948 * for IO_URING_NAPI_STATIC_ADD_ID/IO_URING_NAPI_STATIC_DEL_ID
949 * it is the napi id to add/del from napi_list.
950 */
951 __u32 op_param;
952 __u32 resv;
953};
954
955/*
956 * io_uring_restriction->opcode values
957 */
958enum io_uring_register_restriction_op {
959 /* Allow an io_uring_register(2) opcode */
960 IORING_RESTRICTION_REGISTER_OP = 0,
961
962 /* Allow an sqe opcode */
963 IORING_RESTRICTION_SQE_OP = 1,
964
965 /* Allow sqe flags */
966 IORING_RESTRICTION_SQE_FLAGS_ALLOWED = 2,
967
968 /* Require sqe flags (these flags must be set on each submission) */
969 IORING_RESTRICTION_SQE_FLAGS_REQUIRED = 3,
970
971 IORING_RESTRICTION_LAST
972};
973
974enum {
975 IORING_REG_WAIT_TS = (1U << 0),
976};
977
978/*
979 * Argument for io_uring_enter(2) with
980 * IORING_GETEVENTS | IORING_ENTER_EXT_ARG_REG set, where the actual argument
981 * is an index into a previously registered fixed wait region described by
982 * the below structure.
983 */
984struct io_uring_reg_wait {
985 struct __kernel_timespec ts;
986 __u32 min_wait_usec;
987 __u32 flags;
988 __u64 sigmask;
989 __u32 sigmask_sz;
990 __u32 pad[3];
991 __u64 pad2[2];
992};
993
994/*
995 * Argument for io_uring_enter(2) with IORING_GETEVENTS | IORING_ENTER_EXT_ARG
996 */
997struct io_uring_getevents_arg {
998 __u64 sigmask;
999 __u32 sigmask_sz;
1000 __u32 min_wait_usec;
1001 __u64 ts;
1002};
1003
1004/*
1005 * Argument for IORING_REGISTER_SYNC_CANCEL
1006 */
1007struct io_uring_sync_cancel_reg {
1008 __u64 addr;
1009 __s32 fd;
1010 __u32 flags;
1011 struct __kernel_timespec timeout;
1012 __u8 opcode;
1013 __u8 pad[7];
1014 __u64 pad2[3];
1015};
1016
1017/*
1018 * Argument for IORING_REGISTER_FILE_ALLOC_RANGE
1019 * The range is specified as [off, off + len)
1020 */
1021struct io_uring_file_index_range {
1022 __u32 off;
1023 __u32 len;
1024 __u64 resv;
1025};
1026
1027struct io_uring_recvmsg_out {
1028 __u32 namelen;
1029 __u32 controllen;
1030 __u32 payloadlen;
1031 __u32 flags;
1032};
1033
1034/*
1035 * Argument for IORING_OP_URING_CMD when file is a socket
1036 */
1037enum io_uring_socket_op {
1038 SOCKET_URING_OP_SIOCINQ = 0,
1039 SOCKET_URING_OP_SIOCOUTQ,
1040 SOCKET_URING_OP_GETSOCKOPT,
1041 SOCKET_URING_OP_SETSOCKOPT,
1042 SOCKET_URING_OP_TX_TIMESTAMP,
1043 SOCKET_URING_OP_GETSOCKNAME,
1044};
1045
1046/*
1047 * SOCKET_URING_OP_TX_TIMESTAMP definitions
1048 */
1049
1050#define IORING_TIMESTAMP_HW_SHIFT 16
1051/* The cqe->flags bit from which the timestamp type is stored */
1052#define IORING_TIMESTAMP_TYPE_SHIFT (IORING_TIMESTAMP_HW_SHIFT + 1)
1053/* The cqe->flags flag signifying whether it's a hardware timestamp */
1054#define IORING_CQE_F_TSTAMP_HW ((__u32)1 << IORING_TIMESTAMP_HW_SHIFT)
1055
1056struct io_timespec {
1057 __u64 tv_sec;
1058 __u64 tv_nsec;
1059};
1060
1061#ifdef __cplusplus
1062}
1063#endif
1064
1065#endif