summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/bpf-ops.c2
-rw-r--r--io_uring/epoll.c18
-rw-r--r--io_uring/fs.c3
-rw-r--r--io_uring/kbuf.c5
-rw-r--r--io_uring/memmap.c4
-rw-r--r--io_uring/mpscq.h9
-rw-r--r--io_uring/msg_ring.c34
-rw-r--r--io_uring/nop.c8
-rw-r--r--io_uring/opdef.c1
-rw-r--r--io_uring/query.c12
-rw-r--r--io_uring/register.c31
-rw-r--r--io_uring/rw.c54
-rw-r--r--io_uring/tw.c30
-rw-r--r--io_uring/uring_cmd.c4
-rw-r--r--io_uring/zcrx.c42
-rw-r--r--io_uring/zcrx.h7
16 files changed, 170 insertions, 94 deletions
diff --git a/io_uring/bpf-ops.c b/io_uring/bpf-ops.c
index 5a50f0675fe5..cf2bd068e331 100644
--- a/io_uring/bpf-ops.c
+++ b/io_uring/bpf-ops.c
@@ -168,6 +168,8 @@ static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops)
if (ctx->bpf_ops)
return -EBUSY;
+ if (ops->priv)
+ return -EBUSY;
if (WARN_ON_ONCE(!ops->loop_step))
return -EINVAL;
diff --git a/io_uring/epoll.c b/io_uring/epoll.c
index 8d4610246ba0..eecd748cad01 100644
--- a/io_uring/epoll.c
+++ b/io_uring/epoll.c
@@ -51,10 +51,24 @@ int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_epoll *ie = io_kiocb_to_cmd(req, struct io_epoll);
- int ret;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
+ struct epoll_key key;
+ int ret;
+
+ CLASS(fd, f)(ie->epfd);
+ if (fd_empty(f))
+ return -EBADF;
+
+ CLASS(fd, tf)(ie->fd);
+ if (fd_empty(tf))
+ return -EBADF;
+ /* disallow adding an epoll context to another epoll context */
+ if (ie->op == EPOLL_CTL_ADD && is_file_epoll(fd_file(tf)))
+ return -EINVAL;
- ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
+ key.file = fd_file(tf);
+ key.fd = ie->fd;
+ ret = do_epoll_ctl_file(fd_file(f), ie->op, &key, &ie->event, force_nonblock);
if (force_nonblock && ret == -EAGAIN)
return -EAGAIN;
diff --git a/io_uring/fs.c b/io_uring/fs.c
index d0580c754bf8..26ea841a22e7 100644
--- a/io_uring/fs.c
+++ b/io_uring/fs.c
@@ -110,7 +110,8 @@ int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
const char __user *fname;
int err;
- if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in)
+ if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in ||
+ sqe->addr3 || sqe->__pad2[0])
return -EINVAL;
if (unlikely(req->flags & REQ_F_FIXED_FILE))
return -EBADF;
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 3cd29477fff2..de0129bceaba 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -287,8 +287,6 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
iov = kmalloc_objs(struct iovec, nr_avail);
if (unlikely(!iov))
return -ENOMEM;
- if (arg->mode & KBUF_MODE_FREE)
- kfree(arg->iovs);
arg->iovs = iov;
nr_iovs = nr_avail;
} else if (nr_avail < nr_iovs) {
@@ -330,6 +328,9 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
buf = io_ring_head_to_buf(br, ++head, bl->mask);
} while (--nr_iovs);
+ if (arg->iovs != org_iovs && (arg->mode & KBUF_MODE_FREE))
+ kfree(org_iovs);
+
if (head == tail)
req->flags |= REQ_F_BL_EMPTY;
diff --git a/io_uring/memmap.c b/io_uring/memmap.c
index 4f9b439319c4..23e8a85111bc 100644
--- a/io_uring/memmap.c
+++ b/io_uring/memmap.c
@@ -53,7 +53,7 @@ struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages)
nr_pages = end - start;
if (WARN_ON_ONCE(!nr_pages))
return ERR_PTR(-EINVAL);
- if (WARN_ON_ONCE(nr_pages > INT_MAX))
+ if (nr_pages > INT_MAX / sizeof(struct page *))
return ERR_PTR(-EOVERFLOW);
pages = kvmalloc_objs(struct page *, nr_pages, GFP_KERNEL_ACCOUNT);
@@ -337,7 +337,7 @@ unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr,
ptr = io_uring_validate_mmap_request(filp, pgoff);
if (IS_ERR(ptr))
- return -ENOMEM;
+ return PTR_ERR(ptr);
/*
* Some architectures have strong cache aliasing requirements.
diff --git a/io_uring/mpscq.h b/io_uring/mpscq.h
index c801384c6a0a..f910526766fd 100644
--- a/io_uring/mpscq.h
+++ b/io_uring/mpscq.h
@@ -122,4 +122,13 @@ static inline struct llist_node *mpscq_pop(struct mpscq *q,
return NULL;
}
+/*
+ * Returns true if the most recent mpscq_pop() that returned a node also
+ * emptied the queue. Consumer must be serialized.
+ */
+static inline bool mpscq_pop_emptied(struct mpscq *q, struct llist_node *head)
+{
+ return head == &q->stub;
+}
+
#endif /* IOU_MPSCQ_H */
diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 3ff9098573db..3067c9343991 100644
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -93,19 +93,38 @@ static void io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req,
io_req_task_work_add_remote(req, IOU_F_TWQ_LAZY_WAKE);
}
+static int io_msg_ring_cqe_flags(struct io_ring_ctx *target_ctx,
+ const struct io_msg *msg, u32 *flags)
+{
+ *flags = 0;
+
+ if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS))
+ return 0;
+
+ *flags = msg->cqe_flags;
+ if ((*flags & IORING_CQE_F_32) &&
+ !(target_ctx->flags & (IORING_SETUP_CQE32 |
+ IORING_SETUP_CQE_MIXED)))
+ return -EINVAL;
+
+ return 0;
+}
+
static int io_msg_data_remote(struct io_ring_ctx *target_ctx,
struct io_msg *msg)
{
struct io_kiocb *target;
- u32 flags = 0;
+ u32 flags;
+ int ret;
- target = kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO) ;
+ ret = io_msg_ring_cqe_flags(target_ctx, msg, &flags);
+ if (ret)
+ return ret;
+
+ target = kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
if (unlikely(!target))
return -ENOMEM;
- if (msg->flags & IORING_MSG_RING_FLAGS_PASS)
- flags = msg->cqe_flags;
-
io_msg_remote_post(target_ctx, target, msg->len, flags, msg->user_data);
return 0;
}
@@ -130,8 +149,9 @@ static int __io_msg_ring_data(struct io_ring_ctx *target_ctx,
if (io_msg_need_remote(target_ctx))
return io_msg_data_remote(target_ctx, msg);
- if (msg->flags & IORING_MSG_RING_FLAGS_PASS)
- flags = msg->cqe_flags;
+ ret = io_msg_ring_cqe_flags(target_ctx, msg, &flags);
+ if (ret)
+ return ret;
ret = -EOVERFLOW;
if (target_ctx->flags & IORING_SETUP_IOPOLL) {
diff --git a/io_uring/nop.c b/io_uring/nop.c
index 91ae0b2e7e55..60ab19604b36 100644
--- a/io_uring/nop.c
+++ b/io_uring/nop.c
@@ -40,6 +40,8 @@ int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
nop->fd = READ_ONCE(sqe->fd);
else
nop->fd = -1;
+ if (nop->flags & IORING_NOP_FIXED_FILE)
+ req->flags |= REQ_F_FIXED_FILE;
if (nop->flags & IORING_NOP_FIXED_BUFFER)
req->buf_index = READ_ONCE(sqe->buf_index);
if (nop->flags & IORING_NOP_CQE32) {
@@ -59,12 +61,10 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
int ret = nop->result;
if (nop->flags & IORING_NOP_FILE) {
- if (nop->flags & IORING_NOP_FIXED_FILE) {
+ if (req->flags & REQ_F_FIXED_FILE)
req->file = io_file_get_fixed(req, nop->fd, issue_flags);
- req->flags |= REQ_F_FIXED_FILE;
- } else {
+ else
req->file = io_file_get_normal(req, nop->fd);
- }
if (!req->file) {
ret = -EBADF;
goto done;
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index 88a45c7d897f..4e58eb1344ea 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -520,6 +520,7 @@ const struct io_issue_def io_issue_defs[] = {
#endif
},
[IORING_OP_RECV_ZC] = {
+ .audit_skip = 1,
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
diff --git a/io_uring/query.c b/io_uring/query.c
index d529d94aa8f4..88a325736992 100644
--- a/io_uring/query.c
+++ b/io_uring/query.c
@@ -9,7 +9,7 @@
union io_query_data {
struct io_uring_query_opcode opcodes;
struct io_uring_query_zcrx zcrx;
- struct io_uring_query_zcrx_notif zcrx_notif;
+ struct io_uring_query_zcrx_event zcrx_notif;
struct io_uring_query_scq scq;
};
@@ -47,11 +47,11 @@ static ssize_t io_query_zcrx(union io_query_data *data)
static ssize_t io_query_zcrx_notif(union io_query_data *data)
{
- struct io_uring_query_zcrx_notif *e = &data->zcrx_notif;
+ struct io_uring_query_zcrx_event *e = &data->zcrx_notif;
- e->notif_flags = ZCRX_NOTIF_TYPE_MASK;
- e->notif_stats_size = sizeof(struct zcrx_notif_stats);
- e->notif_stats_off_alignment = __alignof__(struct zcrx_notif_stats);
+ e->event_flags = ZCRX_EVENT_TYPE_MASK;
+ e->stats_size = sizeof(struct zcrx_stats);
+ e->stats_off_alignment = __alignof__(struct zcrx_stats);
e->__resv1 = 0;
memset(&e->__resv2, 0, sizeof(e->__resv2));
return sizeof(*e);
@@ -96,7 +96,7 @@ static int io_handle_query_entry(union io_query_data *data, void __user *uhdr,
case IO_URING_QUERY_ZCRX:
ret = io_query_zcrx(data);
break;
- case IO_URING_QUERY_ZCRX_NOTIF:
+ case IO_URING_QUERY_ZCRX_EVENT:
ret = io_query_zcrx_notif(data);
break;
case IO_URING_QUERY_SCQ:
diff --git a/io_uring/register.c b/io_uring/register.c
index dce5e2f9cf77..02bc103bcc9d 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -503,6 +503,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
unsigned i, tail, old_head;
struct io_uring_params *p = &config.p;
struct io_rings_layout *rl = &config.layout;
+ u32 *o_sq_array, *n_sq_array = NULL;
int ret;
memset(&config, 0, sizeof(config));
@@ -589,6 +590,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
ctx->rings = NULL;
o.sq_sqes = ctx->sq_sqes;
ctx->sq_sqes = NULL;
+ o_sq_array = ctx->sq_array;
+ if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
+ n_sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
/*
* Now copy SQ and CQ entries, if any. If either of the destination
@@ -599,20 +603,27 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
if (tail - old_head > p->sq_entries)
goto overflow;
for (i = old_head; i < tail; i++) {
- unsigned index, dst_mask, src_mask;
+ unsigned int dst, src;
size_t sq_size;
- index = i;
+ dst = i & (p->sq_entries - 1);
+ src = i & (ctx->sq_entries - 1);
+ if (n_sq_array) {
+ src = READ_ONCE(o_sq_array[src]);
+ if (unlikely(src >= ctx->sq_entries)) {
+ WRITE_ONCE(n_sq_array[dst], UINT_MAX);
+ continue;
+ }
+ WRITE_ONCE(n_sq_array[dst], dst);
+ }
+
sq_size = sizeof(struct io_uring_sqe);
- src_mask = ctx->sq_entries - 1;
- dst_mask = p->sq_entries - 1;
if (ctx->flags & IORING_SETUP_SQE128) {
- index <<= 1;
+ dst <<= 1;
+ src <<= 1;
sq_size <<= 1;
- src_mask = (ctx->sq_entries << 1) - 1;
- dst_mask = (p->sq_entries << 1) - 1;
}
- memcpy(&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size);
+ memcpy(&n.sq_sqes[dst], &o.sq_sqes[src], sq_size);
}
WRITE_ONCE(n.rings->sq.head, old_head);
WRITE_ONCE(n.rings->sq.tail, tail);
@@ -655,8 +666,8 @@ overflow:
WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow));
/* all done, store old pointers and assign new ones */
- if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
- ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
+ if (n_sq_array)
+ ctx->sq_array = n_sq_array;
ctx->sq_entries = p->sq_entries;
ctx->cq_entries = p->cq_entries;
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 0c4834645279..95038cfda615 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -601,20 +601,38 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
{
struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
struct io_kiocb *req = cmd_to_io_kiocb(rw);
+ int final_res = io_fixup_rw_res(req, res);
if (kiocb->ki_flags & IOCB_WRITE)
io_req_end_write(req);
- if (unlikely(res != req->cqe.res)) {
- if (res == -EAGAIN && io_rw_should_reissue(req))
- req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE;
- else
- req->cqe.res = res;
- }
+
+ if (res == -EAGAIN && io_rw_should_reissue(req))
+ req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE;
+ else if (unlikely(final_res != req->cqe.res))
+ req->cqe.res = final_res;
/* order with io_iopoll_complete() checking ->iopoll_completed */
smp_store_release(&req->iopoll_completed, 1);
}
+static inline ssize_t io_fixup_restart_res(ssize_t ret)
+{
+ switch (ret) {
+ case -ERESTARTSYS:
+ case -ERESTARTNOINTR:
+ case -ERESTARTNOHAND:
+ case -ERESTART_RESTARTBLOCK:
+ /*
+ * We can't just restart the syscall, since previously
+ * submitted sqes may already be in progress. Just fail
+ * this IO with EINTR.
+ */
+ return -EINTR;
+ default:
+ return ret;
+ }
+}
+
static inline void io_rw_done(struct io_kiocb *req, ssize_t ret)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -624,21 +642,8 @@ static inline void io_rw_done(struct io_kiocb *req, ssize_t ret)
return;
/* transform internal restart error codes */
- if (unlikely(ret < 0)) {
- switch (ret) {
- case -ERESTARTSYS:
- case -ERESTARTNOINTR:
- case -ERESTARTNOHAND:
- case -ERESTART_RESTARTBLOCK:
- /*
- * We can't just restart the syscall, since previously
- * submitted sqes may already be in progress. Just fail
- * this IO with EINTR.
- */
- ret = -EINTR;
- break;
- }
- }
+ if (unlikely(ret < 0))
+ ret = io_fixup_restart_res(ret);
if (req->flags & REQ_F_IOPOLL)
io_complete_rw_iopoll(&rw->kiocb, ret);
@@ -1034,7 +1039,8 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags)
if (req->flags & REQ_F_BUFFERS_COMMIT)
io_kbuf_recycle(req, sel.buf_list, issue_flags);
- return ret;
+
+ return io_fixup_restart_res(ret);
}
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
@@ -1068,8 +1074,10 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
return IOU_RETRY;
} else if (ret <= 0) {
io_kbuf_recycle(req, sel.buf_list, issue_flags);
- if (ret < 0)
+ if (ret < 0) {
+ ret = io_fixup_restart_res(ret);
req_set_fail(req);
+ }
} else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
cflags = io_put_kbuf(req, ret, sel.buf_list);
} else {
diff --git a/io_uring/tw.c b/io_uring/tw.c
index e74372233f40..e6ee15571e85 100644
--- a/io_uring/tw.c
+++ b/io_uring/tw.c
@@ -34,10 +34,6 @@ void io_tctx_fallback_work(struct work_struct *work)
fallback_work);
unsigned int count = 0;
- /* see tctx_task_work() - a set bit must always have a run coming */
- clear_bit(0, &tctx->tw_pending);
- smp_mb__after_atomic();
-
/*
* Run the entries directly. We're in PF_KTHRED context, hence
* io_should_terminate_tw() is true and they will be marked as
@@ -55,7 +51,7 @@ static void io_fallback_tw(struct io_uring_task *tctx)
* the queued work) stay around until the drain has run.
*/
get_task_struct(tctx->task);
- if (!queue_work(system_unbound_wq, &tctx->fallback_work))
+ if (!queue_work(system_dfl_wq, &tctx->fallback_work))
put_task_struct(tctx->task);
}
@@ -101,6 +97,13 @@ void tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries,
io_poll_task_func, io_req_rw_complete,
(struct io_tw_req){req}, ts);
(*count)++;
+ /*
+ * Break if most recent pop emptied the queue. This helps
+ * bound task_work run, and also protects the regular
+ * task_work addition.
+ */
+ if (mpscq_pop_emptied(&tctx->task_list, tctx->task_head))
+ break;
if (unlikely(need_resched())) {
ctx_flush_and_put(ctx, ts);
ctx = NULL;
@@ -127,8 +130,6 @@ void tctx_task_work(struct callback_head *cb)
unsigned int count = 0;
tctx = container_of(cb, struct io_uring_task, task_work);
- clear_bit(0, &tctx->tw_pending);
- smp_mb__after_atomic();
tctx_task_work_run(tctx, UINT_MAX, &count);
}
@@ -138,11 +139,11 @@ void tctx_task_work(struct callback_head *cb)
*/
static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx)
{
+ lockdep_assert_in_rcu_read_lock();
+
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) {
- struct io_rings *rings;
+ struct io_rings *rings = rcu_dereference(ctx->rings_rcu);
- guard(rcu)();
- rings = rcu_dereference(ctx->rings_rcu);
atomic_or(IORING_SQ_TASKRUN, &rings->sq_flags);
}
}
@@ -152,6 +153,9 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
struct io_ring_ctx *ctx = req->ctx;
int nr_wait;
+ /* pairs with synchronize_rcu() in io_ring_exit_work() */
+ guard(rcu)();
+
/*
* We don't know how many requests there are in the link and whether
* they can even be queued lazily, fall back to non-lazy.
@@ -206,7 +210,7 @@ void io_req_normal_work_add(struct io_kiocb *req)
struct io_uring_task *tctx = req->tctx;
struct io_ring_ctx *ctx = req->ctx;
- /* task_work already pending, we're done */
+ /* tw run already pending, nothing else to do */
if (!mpscq_push(&tctx->task_list, &req->io_task_work.node))
return;
@@ -223,10 +227,6 @@ void io_req_normal_work_add(struct io_kiocb *req)
return;
}
- /* task_work must only be added once */
- if (test_and_set_bit(0, &tctx->tw_pending))
- return;
-
if (likely(!task_work_add(tctx->task, &tctx->task_work, ctx->notify_method)))
return;
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 7b25dcd9d05f..c14c22cff49e 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -90,7 +90,7 @@ static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
}
/*
- * Mark this command as concelable, then io_uring_try_cancel_uring_cmd()
+ * Mark this command as cancelable, then io_uring_try_cancel_uring_cmd()
* will try to cancel this issued command by sending ->uring_cmd() with
* issue_flags of IO_URING_F_CANCEL.
*
@@ -168,7 +168,7 @@ void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
}
io_req_uring_cleanup(req, issue_flags);
if (req->flags & REQ_F_IOPOLL) {
- /* order with io_iopoll_req_issued() checking ->iopoll_complete */
+ /* order with io_do_iopoll() checking ->iopoll_completed */
smp_store_release(&req->iopoll_completed, 1);
} else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
if (WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED))
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 49163f9c39df..76b9b0d54af9 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -766,7 +766,7 @@ static int import_zcrx(struct io_ring_ctx *ctx,
return -EINVAL;
if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
return -EINVAL;
- if (reg->notif_desc)
+ if (reg->event_desc)
return -EINVAL;
if (reg->flags & ~ZCRX_REG_IMPORT)
return -EINVAL;
@@ -855,7 +855,7 @@ netdev_put_unlock:
static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
const struct io_uring_zcrx_ifq_reg *reg,
- const struct zcrx_notification_desc *notif)
+ const struct zcrx_event_desc *notif)
{
size_t stats_off = notif->stats_offset;
size_t used, end;
@@ -863,12 +863,12 @@ static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
used = reg->offsets.rqes +
sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
- if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_notif_stats)))
+ if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_stats)))
return -EINVAL;
if (stats_off < used)
return -ERANGE;
if (check_add_overflow(stats_off,
- sizeof(struct zcrx_notif_stats),
+ sizeof(struct zcrx_stats),
&end))
return -ERANGE;
if (end > io_region_size(&ifq->rq_region))
@@ -883,7 +883,7 @@ static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
int io_register_zcrx(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg)
{
- struct zcrx_notification_desc notif;
+ struct zcrx_event_desc notif;
struct io_uring_zcrx_area_reg area;
struct io_uring_zcrx_ifq_reg reg;
struct io_uring_region_desc rd;
@@ -928,14 +928,14 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
return -EFAULT;
memset(&notif, 0, sizeof(notif));
- if (reg.notif_desc && copy_from_user(&notif, u64_to_user_ptr(reg.notif_desc),
+ if (reg.event_desc && copy_from_user(&notif, u64_to_user_ptr(reg.event_desc),
sizeof(notif)))
return -EFAULT;
- if (notif.type_mask & ~ZCRX_NOTIF_TYPE_MASK)
+ if (notif.type_mask & ~ZCRX_EVENT_TYPE_MASK)
return -EINVAL;
- if (notif.flags & ~ZCRX_NOTIF_DESC_FLAG_STATS)
+ if (notif.flags & ~ZCRX_EVENT_DESC_FLAG_STATS)
return -EINVAL;
- if (!(notif.flags & ZCRX_NOTIF_DESC_FLAG_STATS)) {
+ if (!(notif.flags & ZCRX_EVENT_DESC_FLAG_STATS)) {
if (notif.stats_offset)
return -EINVAL;
}
@@ -970,7 +970,7 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
if (ret)
goto err;
- if (notif.flags & ZCRX_NOTIF_DESC_FLAG_STATS) {
+ if (notif.flags & ZCRX_EVENT_DESC_FLAG_STATS) {
ret = zcrx_validate_notif_stats(ifq, &reg, &notif);
if (ret)
goto err;
@@ -1244,7 +1244,7 @@ static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp)
allocated = io_zcrx_refill_slow(pp, ifq, netmems, to_alloc);
if (!allocated) {
- zcrx_send_notif(ifq, ZCRX_NOTIF_NO_BUFFERS);
+ zcrx_send_notif(ifq, ZCRX_EVENT_ALLOC_FAIL);
return 0;
}
out_return:
@@ -1294,6 +1294,7 @@ static void io_pp_zc_destroy(struct page_pool *pp)
static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
struct netdev_rx_queue *rxq)
{
+ struct io_zcrx_ifq *ifq = mp_priv;
struct nlattr *nest;
int type;
@@ -1301,6 +1302,13 @@ static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
nest = nla_nest_start(rsp, type);
if (!nest)
return -EMSGSIZE;
+
+ if (nla_put_uint(rsp, NETDEV_A_IO_URING_PROVIDER_INFO_RX_BUF_LEN,
+ 1ULL << ifq->niov_shift)) {
+ nla_nest_cancel(rsp, nest);
+ return -EMSGSIZE;
+ }
+
nla_nest_end(rsp, nest);
return 0;
@@ -1398,16 +1406,16 @@ static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
struct zcrx_ctrl *ctrl)
{
- const struct zcrx_ctrl_arm_notif *an = &ctrl->zc_arm_notif;
+ const struct zcrx_ctrl_arm_event *an = &ctrl->zc_arm_event;
unsigned type_mask;
- if (an->notif_type >= __ZCRX_NOTIF_TYPE_LAST)
+ if (an->event_type >= __ZCRX_EVENT_TYPE_LAST)
return -EINVAL;
if (!mem_is_zero(&an->__resv, sizeof(an->__resv)))
return -EINVAL;
guard(spinlock_bh)(&zcrx->ctx_lock);
- type_mask = 1U << an->notif_type;
+ type_mask = 1U << an->event_type;
if (type_mask & ~zcrx->fired_notifs)
return -EINVAL;
zcrx->fired_notifs &= ~type_mask;
@@ -1420,7 +1428,7 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
struct io_zcrx_ifq *zcrx;
BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_flush));
- BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_notif));
+ BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_event));
if (nr_args)
return -EINVAL;
@@ -1438,7 +1446,7 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
return zcrx_flush_rq(ctx, zcrx, &ctrl);
case ZCRX_CTRL_EXPORT:
return zcrx_export(ctx, zcrx, &ctrl, arg);
- case ZCRX_CTRL_ARM_NOTIFICATION:
+ case ZCRX_CTRL_ARM_EVENT:
return zcrx_arm_notif(ctx, zcrx, &ctrl);
}
@@ -1584,7 +1592,7 @@ static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
zcrx_stat_add(&ifq->notif_stats->copy_count, 1);
zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret);
}
- zcrx_send_notif(ifq, ZCRX_NOTIF_COPY);
+ zcrx_send_notif(ifq, ZCRX_EVENT_COPY);
}
return ret;
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index fa00900e479e..c1005f23caff 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -10,8 +10,9 @@
#define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
#define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE |\
- ZCRX_FEATURE_NOTIFICATION)
-#define ZCRX_NOTIF_TYPE_MASK ((1U << ZCRX_NOTIF_NO_BUFFERS) | (1U << ZCRX_NOTIF_COPY))
+ ZCRX_FEATURE_EVENT)
+#define ZCRX_EVENT_TYPE_MASK ((1U << ZCRX_EVENT_ALLOC_FAIL) |\
+ (1U << ZCRX_EVENT_COPY))
struct io_zcrx_mem {
unsigned long size;
@@ -80,7 +81,7 @@ struct io_zcrx_ifq {
u32 allowed_notif_mask;
u32 fired_notifs;
u64 notif_data;
- struct zcrx_notif_stats *notif_stats;
+ struct zcrx_stats *notif_stats;
};
#if defined(CONFIG_IO_URING_ZCRX)
generated by cgit v1.3.1 (git 2.54.0) at 2026-07-26 08:18:52 +0000