summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/uapi/linux/io_uring/query.h14
-rw-r--r--include/uapi/linux/io_uring/zcrx.h32
-rw-r--r--io_uring/query.c12
-rw-r--r--io_uring/rw.c42
-rw-r--r--io_uring/zcrx.c34
-rw-r--r--io_uring/zcrx.h7
6 files changed, 75 insertions, 66 deletions
diff --git a/include/uapi/linux/io_uring/query.h b/include/uapi/linux/io_uring/query.h
index 1a68eca7c6b4..e6493e4a7a4d 100644
--- a/include/uapi/linux/io_uring/query.h
+++ b/include/uapi/linux/io_uring/query.h
@@ -23,7 +23,7 @@ enum {
IO_URING_QUERY_OPCODES = 0,
IO_URING_QUERY_ZCRX = 1,
IO_URING_QUERY_SCQ = 2,
- IO_URING_QUERY_ZCRX_NOTIF = 3,
+ IO_URING_QUERY_ZCRX_EVENT = 3,
__IO_URING_QUERY_MAX,
};
@@ -63,13 +63,13 @@ struct io_uring_query_zcrx {
__u64 __resv2;
};
-struct io_uring_query_zcrx_notif {
- /* Bitmask of supported ZCRX_NOTIF_* flags */
- __u32 notif_flags;
- /* Size of io_uring_zcrx_notif_stats */
- __u32 notif_stats_size;
+struct io_uring_query_zcrx_event {
+ /* Bitmask of supported ZCRX_EVENT_* flags */
+ __u32 event_flags;
+ /* Size of zcrx_stats */
+ __u32 stats_size;
/* Required alignment for the stats struct within the region (ie stats_offset) */
- __u32 notif_stats_off_alignment;
+ __u32 stats_off_alignment;
__u32 __resv1;
__u64 __resv2[4];
};
diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h
index 15c05c45ce36..e01bc0e34b24 100644
--- a/include/uapi/linux/io_uring/zcrx.h
+++ b/include/uapi/linux/io_uring/zcrx.h
@@ -65,30 +65,30 @@ enum zcrx_features {
* value in struct io_uring_zcrx_ifq_reg::rx_buf_len.
*/
ZCRX_FEATURE_RX_PAGE_SIZE = 1 << 0,
- ZCRX_FEATURE_NOTIFICATION = 1 << 1,
+ ZCRX_FEATURE_EVENT = 1 << 1,
};
-enum zcrx_notification_type {
- ZCRX_NOTIF_NO_BUFFERS,
- ZCRX_NOTIF_COPY,
+enum zcrx_event_type {
+ ZCRX_EVENT_ALLOC_FAIL,
+ ZCRX_EVENT_COPY,
- __ZCRX_NOTIF_TYPE_LAST,
+ __ZCRX_EVENT_TYPE_LAST,
};
-enum zcrx_notification_desc_flags {
- /* If set, stats_offset holds a valid offset to a notif_stats struct */
- ZCRX_NOTIF_DESC_FLAG_STATS = 1 << 0,
+enum zcrx_event_desc_flags {
+ /* If set, stats_offset holds a valid offset to a zcrx_stats struct */
+ ZCRX_EVENT_DESC_FLAG_STATS = 1 << 0,
};
-struct zcrx_notif_stats {
+struct zcrx_stats {
__u64 copy_count; /* cumulative copy-fallback CQEs */
__u64 copy_bytes; /* cumulative bytes copied */
};
-struct zcrx_notification_desc {
+struct zcrx_event_desc {
__u64 user_data;
__u32 type_mask;
- __u32 flags; /* see enum zcrx_notification_desc_flags */
+ __u32 flags; /* see enum zcrx_event_desc_flags */
__u64 stats_offset; /* offset from the beginning of refill ring region for stats */
__u64 __resv2[9];
};
@@ -108,14 +108,14 @@ struct io_uring_zcrx_ifq_reg {
struct io_uring_zcrx_offsets offsets;
__u32 zcrx_id;
__u32 rx_buf_len;
- __u64 notif_desc; /* see struct zcrx_notification_desc */
+ __u64 event_desc; /* see struct zcrx_event_desc */
__u64 __resv[2];
};
enum zcrx_ctrl_op {
ZCRX_CTRL_FLUSH_RQ,
ZCRX_CTRL_EXPORT,
- ZCRX_CTRL_ARM_NOTIFICATION,
+ ZCRX_CTRL_ARM_EVENT,
__ZCRX_CTRL_LAST,
};
@@ -129,8 +129,8 @@ struct zcrx_ctrl_export {
__u32 __resv1[11];
};
-struct zcrx_ctrl_arm_notif {
- __u32 notif_type;
+struct zcrx_ctrl_arm_event {
+ __u32 event_type; /* see enum zcrx_event_type */
__u32 __resv[11];
};
@@ -142,7 +142,7 @@ struct zcrx_ctrl {
union {
struct zcrx_ctrl_export zc_export;
struct zcrx_ctrl_flush_rq zc_flush;
- struct zcrx_ctrl_arm_notif zc_arm_notif;
+ struct zcrx_ctrl_arm_event zc_arm_event;
};
};
diff --git a/io_uring/query.c b/io_uring/query.c
index d529d94aa8f4..88a325736992 100644
--- a/io_uring/query.c
+++ b/io_uring/query.c
@@ -9,7 +9,7 @@
union io_query_data {
struct io_uring_query_opcode opcodes;
struct io_uring_query_zcrx zcrx;
- struct io_uring_query_zcrx_notif zcrx_notif;
+ struct io_uring_query_zcrx_event zcrx_notif;
struct io_uring_query_scq scq;
};
@@ -47,11 +47,11 @@ static ssize_t io_query_zcrx(union io_query_data *data)
static ssize_t io_query_zcrx_notif(union io_query_data *data)
{
- struct io_uring_query_zcrx_notif *e = &data->zcrx_notif;
+ struct io_uring_query_zcrx_event *e = &data->zcrx_notif;
- e->notif_flags = ZCRX_NOTIF_TYPE_MASK;
- e->notif_stats_size = sizeof(struct zcrx_notif_stats);
- e->notif_stats_off_alignment = __alignof__(struct zcrx_notif_stats);
+ e->event_flags = ZCRX_EVENT_TYPE_MASK;
+ e->stats_size = sizeof(struct zcrx_stats);
+ e->stats_off_alignment = __alignof__(struct zcrx_stats);
e->__resv1 = 0;
memset(&e->__resv2, 0, sizeof(e->__resv2));
return sizeof(*e);
@@ -96,7 +96,7 @@ static int io_handle_query_entry(union io_query_data *data, void __user *uhdr,
case IO_URING_QUERY_ZCRX:
ret = io_query_zcrx(data);
break;
- case IO_URING_QUERY_ZCRX_NOTIF:
+ case IO_URING_QUERY_ZCRX_EVENT:
ret = io_query_zcrx_notif(data);
break;
case IO_URING_QUERY_SCQ:
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 63b6519e498c..95038cfda615 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -615,6 +615,24 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
smp_store_release(&req->iopoll_completed, 1);
}
+static inline ssize_t io_fixup_restart_res(ssize_t ret)
+{
+ switch (ret) {
+ case -ERESTARTSYS:
+ case -ERESTARTNOINTR:
+ case -ERESTARTNOHAND:
+ case -ERESTART_RESTARTBLOCK:
+ /*
+ * We can't just restart the syscall, since previously
+ * submitted sqes may already be in progress. Just fail
+ * this IO with EINTR.
+ */
+ return -EINTR;
+ default:
+ return ret;
+ }
+}
+
static inline void io_rw_done(struct io_kiocb *req, ssize_t ret)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -624,21 +642,8 @@ static inline void io_rw_done(struct io_kiocb *req, ssize_t ret)
return;
/* transform internal restart error codes */
- if (unlikely(ret < 0)) {
- switch (ret) {
- case -ERESTARTSYS:
- case -ERESTARTNOINTR:
- case -ERESTARTNOHAND:
- case -ERESTART_RESTARTBLOCK:
- /*
- * We can't just restart the syscall, since previously
- * submitted sqes may already be in progress. Just fail
- * this IO with EINTR.
- */
- ret = -EINTR;
- break;
- }
- }
+ if (unlikely(ret < 0))
+ ret = io_fixup_restart_res(ret);
if (req->flags & REQ_F_IOPOLL)
io_complete_rw_iopoll(&rw->kiocb, ret);
@@ -1034,7 +1039,8 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags)
if (req->flags & REQ_F_BUFFERS_COMMIT)
io_kbuf_recycle(req, sel.buf_list, issue_flags);
- return ret;
+
+ return io_fixup_restart_res(ret);
}
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
@@ -1068,8 +1074,10 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
return IOU_RETRY;
} else if (ret <= 0) {
io_kbuf_recycle(req, sel.buf_list, issue_flags);
- if (ret < 0)
+ if (ret < 0) {
+ ret = io_fixup_restart_res(ret);
req_set_fail(req);
+ }
} else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
cflags = io_put_kbuf(req, ret, sel.buf_list);
} else {
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 7ad52f499f87..76b9b0d54af9 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -766,7 +766,7 @@ static int import_zcrx(struct io_ring_ctx *ctx,
return -EINVAL;
if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
return -EINVAL;
- if (reg->notif_desc)
+ if (reg->event_desc)
return -EINVAL;
if (reg->flags & ~ZCRX_REG_IMPORT)
return -EINVAL;
@@ -855,7 +855,7 @@ netdev_put_unlock:
static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
const struct io_uring_zcrx_ifq_reg *reg,
- const struct zcrx_notification_desc *notif)
+ const struct zcrx_event_desc *notif)
{
size_t stats_off = notif->stats_offset;
size_t used, end;
@@ -863,12 +863,12 @@ static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
used = reg->offsets.rqes +
sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
- if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_notif_stats)))
+ if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_stats)))
return -EINVAL;
if (stats_off < used)
return -ERANGE;
if (check_add_overflow(stats_off,
- sizeof(struct zcrx_notif_stats),
+ sizeof(struct zcrx_stats),
&end))
return -ERANGE;
if (end > io_region_size(&ifq->rq_region))
@@ -883,7 +883,7 @@ static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
int io_register_zcrx(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg)
{
- struct zcrx_notification_desc notif;
+ struct zcrx_event_desc notif;
struct io_uring_zcrx_area_reg area;
struct io_uring_zcrx_ifq_reg reg;
struct io_uring_region_desc rd;
@@ -928,14 +928,14 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
return -EFAULT;
memset(&notif, 0, sizeof(notif));
- if (reg.notif_desc && copy_from_user(&notif, u64_to_user_ptr(reg.notif_desc),
+ if (reg.event_desc && copy_from_user(&notif, u64_to_user_ptr(reg.event_desc),
sizeof(notif)))
return -EFAULT;
- if (notif.type_mask & ~ZCRX_NOTIF_TYPE_MASK)
+ if (notif.type_mask & ~ZCRX_EVENT_TYPE_MASK)
return -EINVAL;
- if (notif.flags & ~ZCRX_NOTIF_DESC_FLAG_STATS)
+ if (notif.flags & ~ZCRX_EVENT_DESC_FLAG_STATS)
return -EINVAL;
- if (!(notif.flags & ZCRX_NOTIF_DESC_FLAG_STATS)) {
+ if (!(notif.flags & ZCRX_EVENT_DESC_FLAG_STATS)) {
if (notif.stats_offset)
return -EINVAL;
}
@@ -970,7 +970,7 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
if (ret)
goto err;
- if (notif.flags & ZCRX_NOTIF_DESC_FLAG_STATS) {
+ if (notif.flags & ZCRX_EVENT_DESC_FLAG_STATS) {
ret = zcrx_validate_notif_stats(ifq, &reg, &notif);
if (ret)
goto err;
@@ -1244,7 +1244,7 @@ static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp)
allocated = io_zcrx_refill_slow(pp, ifq, netmems, to_alloc);
if (!allocated) {
- zcrx_send_notif(ifq, ZCRX_NOTIF_NO_BUFFERS);
+ zcrx_send_notif(ifq, ZCRX_EVENT_ALLOC_FAIL);
return 0;
}
out_return:
@@ -1406,16 +1406,16 @@ static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
struct zcrx_ctrl *ctrl)
{
- const struct zcrx_ctrl_arm_notif *an = &ctrl->zc_arm_notif;
+ const struct zcrx_ctrl_arm_event *an = &ctrl->zc_arm_event;
unsigned type_mask;
- if (an->notif_type >= __ZCRX_NOTIF_TYPE_LAST)
+ if (an->event_type >= __ZCRX_EVENT_TYPE_LAST)
return -EINVAL;
if (!mem_is_zero(&an->__resv, sizeof(an->__resv)))
return -EINVAL;
guard(spinlock_bh)(&zcrx->ctx_lock);
- type_mask = 1U << an->notif_type;
+ type_mask = 1U << an->event_type;
if (type_mask & ~zcrx->fired_notifs)
return -EINVAL;
zcrx->fired_notifs &= ~type_mask;
@@ -1428,7 +1428,7 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
struct io_zcrx_ifq *zcrx;
BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_flush));
- BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_notif));
+ BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_event));
if (nr_args)
return -EINVAL;
@@ -1446,7 +1446,7 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
return zcrx_flush_rq(ctx, zcrx, &ctrl);
case ZCRX_CTRL_EXPORT:
return zcrx_export(ctx, zcrx, &ctrl, arg);
- case ZCRX_CTRL_ARM_NOTIFICATION:
+ case ZCRX_CTRL_ARM_EVENT:
return zcrx_arm_notif(ctx, zcrx, &ctrl);
}
@@ -1592,7 +1592,7 @@ static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
zcrx_stat_add(&ifq->notif_stats->copy_count, 1);
zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret);
}
- zcrx_send_notif(ifq, ZCRX_NOTIF_COPY);
+ zcrx_send_notif(ifq, ZCRX_EVENT_COPY);
}
return ret;
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index fa00900e479e..c1005f23caff 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -10,8 +10,9 @@
#define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
#define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE |\
- ZCRX_FEATURE_NOTIFICATION)
-#define ZCRX_NOTIF_TYPE_MASK ((1U << ZCRX_NOTIF_NO_BUFFERS) | (1U << ZCRX_NOTIF_COPY))
+ ZCRX_FEATURE_EVENT)
+#define ZCRX_EVENT_TYPE_MASK ((1U << ZCRX_EVENT_ALLOC_FAIL) |\
+ (1U << ZCRX_EVENT_COPY))
struct io_zcrx_mem {
unsigned long size;
@@ -80,7 +81,7 @@ struct io_zcrx_ifq {
u32 allowed_notif_mask;
u32 fired_notifs;
u64 notif_data;
- struct zcrx_notif_stats *notif_stats;
+ struct zcrx_stats *notif_stats;
};
#if defined(CONFIG_IO_URING_ZCRX)
generated by cgit v1.3.1 (git 2.54.0) at 2026-07-25 23:52:39 +0000