io_uring: move uring_cmd handling to its own file
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
cd40cae29e
commit
99f15d8d61
@ -4,5 +4,5 @@
|
|||||||
|
|
||||||
obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \
|
obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \
|
||||||
sync.o advise.o filetable.o \
|
sync.o advise.o filetable.o \
|
||||||
openclose.o
|
openclose.o uring_cmd.o
|
||||||
obj-$(CONFIG_IO_WQ) += io-wq.o
|
obj-$(CONFIG_IO_WQ) += io-wq.o
|
||||||
|
@ -99,6 +99,7 @@
|
|||||||
#include "sync.h"
|
#include "sync.h"
|
||||||
#include "advise.h"
|
#include "advise.h"
|
||||||
#include "openclose.h"
|
#include "openclose.h"
|
||||||
|
#include "uring_cmd.h"
|
||||||
|
|
||||||
#define IORING_MAX_ENTRIES 32768
|
#define IORING_MAX_ENTRIES 32768
|
||||||
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
|
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
|
||||||
@ -473,14 +474,6 @@ struct io_cancel_data {
|
|||||||
int seq;
|
int seq;
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
|
||||||
* The URING_CMD payload starts at 'cmd' in the first sqe, and continues into
|
|
||||||
* the following sqe if SQE128 is used.
|
|
||||||
*/
|
|
||||||
#define uring_cmd_pdu_size(is_sqe128) \
|
|
||||||
((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) - \
|
|
||||||
offsetof(struct io_uring_sqe, cmd))
|
|
||||||
|
|
||||||
struct io_op_def {
|
struct io_op_def {
|
||||||
/* needs req->file assigned */
|
/* needs req->file assigned */
|
||||||
unsigned needs_file : 1;
|
unsigned needs_file : 1;
|
||||||
@ -988,11 +981,6 @@ static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
|
|||||||
return matched;
|
return matched;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool req_has_async_data(struct io_kiocb *req)
|
|
||||||
{
|
|
||||||
return req->flags & REQ_F_ASYNC_DATA;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline void req_fail_link_node(struct io_kiocb *req, int res)
|
static inline void req_fail_link_node(struct io_kiocb *req, int res)
|
||||||
{
|
{
|
||||||
req_set_fail(req);
|
req_set_fail(req);
|
||||||
@ -1743,7 +1731,7 @@ static void io_req_complete_post(struct io_kiocb *req)
|
|||||||
io_cqring_ev_posted(ctx);
|
io_cqring_ev_posted(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)
|
inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)
|
||||||
{
|
{
|
||||||
if (issue_flags & IO_URING_F_COMPLETE_DEFER)
|
if (issue_flags & IO_URING_F_COMPLETE_DEFER)
|
||||||
req->flags |= REQ_F_COMPLETE_INLINE;
|
req->flags |= REQ_F_COMPLETE_INLINE;
|
||||||
@ -2151,7 +2139,7 @@ static void __io_req_task_work_add(struct io_kiocb *req,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void io_req_task_work_add(struct io_kiocb *req)
|
void io_req_task_work_add(struct io_kiocb *req)
|
||||||
{
|
{
|
||||||
struct io_uring_task *tctx = req->task->io_uring;
|
struct io_uring_task *tctx = req->task->io_uring;
|
||||||
|
|
||||||
@ -3268,7 +3256,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool io_alloc_async_data(struct io_kiocb *req)
|
bool io_alloc_async_data(struct io_kiocb *req)
|
||||||
{
|
{
|
||||||
WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
|
WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
|
||||||
req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
|
req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
|
||||||
@ -3714,111 +3702,6 @@ out_free:
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
|
|
||||||
{
|
|
||||||
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
|
||||||
|
|
||||||
ioucmd->task_work_cb(ioucmd);
|
|
||||||
}
|
|
||||||
|
|
||||||
void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
|
|
||||||
void (*task_work_cb)(struct io_uring_cmd *))
|
|
||||||
{
|
|
||||||
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
|
|
||||||
|
|
||||||
ioucmd->task_work_cb = task_work_cb;
|
|
||||||
req->io_task_work.func = io_uring_cmd_work;
|
|
||||||
io_req_task_work_add(req);
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);
|
|
||||||
|
|
||||||
static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
|
|
||||||
u64 extra1, u64 extra2)
|
|
||||||
{
|
|
||||||
req->extra1 = extra1;
|
|
||||||
req->extra2 = extra2;
|
|
||||||
req->flags |= REQ_F_CQE32_INIT;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Called by consumers of io_uring_cmd, if they originally returned
|
|
||||||
* -EIOCBQUEUED upon receiving the command.
|
|
||||||
*/
|
|
||||||
void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
|
|
||||||
{
|
|
||||||
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
|
|
||||||
|
|
||||||
if (ret < 0)
|
|
||||||
req_set_fail(req);
|
|
||||||
|
|
||||||
io_req_set_res(req, 0, ret);
|
|
||||||
if (req->ctx->flags & IORING_SETUP_CQE32)
|
|
||||||
io_req_set_cqe32_extra(req, res2, 0);
|
|
||||||
__io_req_complete(req, 0);
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
|
|
||||||
|
|
||||||
static int io_uring_cmd_prep_async(struct io_kiocb *req)
|
|
||||||
{
|
|
||||||
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
|
||||||
size_t cmd_size;
|
|
||||||
|
|
||||||
cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
|
|
||||||
|
|
||||||
memcpy(req->async_data, ioucmd->cmd, cmd_size);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int io_uring_cmd_prep(struct io_kiocb *req,
|
|
||||||
const struct io_uring_sqe *sqe)
|
|
||||||
{
|
|
||||||
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
|
||||||
|
|
||||||
if (sqe->rw_flags || sqe->__pad1)
|
|
||||||
return -EINVAL;
|
|
||||||
ioucmd->cmd = sqe->cmd;
|
|
||||||
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
|
|
||||||
{
|
|
||||||
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
|
||||||
struct io_ring_ctx *ctx = req->ctx;
|
|
||||||
struct file *file = req->file;
|
|
||||||
int ret;
|
|
||||||
|
|
||||||
if (!req->file->f_op->uring_cmd)
|
|
||||||
return -EOPNOTSUPP;
|
|
||||||
|
|
||||||
if (ctx->flags & IORING_SETUP_SQE128)
|
|
||||||
issue_flags |= IO_URING_F_SQE128;
|
|
||||||
if (ctx->flags & IORING_SETUP_CQE32)
|
|
||||||
issue_flags |= IO_URING_F_CQE32;
|
|
||||||
if (ctx->flags & IORING_SETUP_IOPOLL)
|
|
||||||
issue_flags |= IO_URING_F_IOPOLL;
|
|
||||||
|
|
||||||
if (req_has_async_data(req))
|
|
||||||
ioucmd->cmd = req->async_data;
|
|
||||||
|
|
||||||
ret = file->f_op->uring_cmd(ioucmd, issue_flags);
|
|
||||||
if (ret == -EAGAIN) {
|
|
||||||
if (!req_has_async_data(req)) {
|
|
||||||
if (io_alloc_async_data(req))
|
|
||||||
return -ENOMEM;
|
|
||||||
io_uring_cmd_prep_async(req);
|
|
||||||
}
|
|
||||||
return -EAGAIN;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ret != -EIOCBQUEUED) {
|
|
||||||
io_uring_cmd_done(ioucmd, ret, 0);
|
|
||||||
return IOU_OK;
|
|
||||||
}
|
|
||||||
|
|
||||||
return IOU_ISSUE_SKIP_COMPLETE;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int io_msg_ring_prep(struct io_kiocb *req,
|
static int io_msg_ring_prep(struct io_kiocb *req,
|
||||||
const struct io_uring_sqe *sqe)
|
const struct io_uring_sqe *sqe)
|
||||||
{
|
{
|
||||||
@ -11533,8 +11416,6 @@ static int __init io_uring_init(void)
|
|||||||
|
|
||||||
BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
|
BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
|
||||||
|
|
||||||
BUILD_BUG_ON(sizeof(struct io_uring_cmd) > 64);
|
|
||||||
|
|
||||||
for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
|
for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
|
||||||
BUG_ON(!io_op_defs[i].prep);
|
BUG_ON(!io_op_defs[i].prep);
|
||||||
BUG_ON(!io_op_defs[i].issue);
|
BUG_ON(!io_op_defs[i].issue);
|
||||||
|
@ -25,6 +25,11 @@ static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
|
|||||||
req->cqe.flags = cflags;
|
req->cqe.flags = cflags;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline bool req_has_async_data(struct io_kiocb *req)
|
||||||
|
{
|
||||||
|
return req->flags & REQ_F_ASYNC_DATA;
|
||||||
|
}
|
||||||
|
|
||||||
static inline void io_put_file(struct file *file)
|
static inline void io_put_file(struct file *file)
|
||||||
{
|
{
|
||||||
if (file)
|
if (file)
|
||||||
@ -53,6 +58,8 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,
|
|||||||
lockdep_assert_held(&ctx->uring_lock);
|
lockdep_assert_held(&ctx->uring_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
|
||||||
|
|
||||||
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
|
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
|
||||||
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
|
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
|
||||||
unsigned issue_flags);
|
unsigned issue_flags);
|
||||||
@ -65,5 +72,7 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
|
|||||||
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
|
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
|
||||||
struct io_rsrc_data *data_to_kill);
|
struct io_rsrc_data *data_to_kill);
|
||||||
bool io_is_uring_fops(struct file *file);
|
bool io_is_uring_fops(struct file *file);
|
||||||
|
bool io_alloc_async_data(struct io_kiocb *req);
|
||||||
|
void io_req_task_work_add(struct io_kiocb *req);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
115
io_uring/uring_cmd.c
Normal file
115
io_uring/uring_cmd.c
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
// SPDX-License-Identifier: GPL-2.0
|
||||||
|
#include <linux/kernel.h>
|
||||||
|
#include <linux/errno.h>
|
||||||
|
#include <linux/file.h>
|
||||||
|
#include <linux/io_uring.h>
|
||||||
|
|
||||||
|
#include <uapi/linux/io_uring.h>
|
||||||
|
|
||||||
|
#include "io_uring_types.h"
|
||||||
|
#include "io_uring.h"
|
||||||
|
#include "uring_cmd.h"
|
||||||
|
|
||||||
|
static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
|
||||||
|
{
|
||||||
|
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
||||||
|
|
||||||
|
ioucmd->task_work_cb(ioucmd);
|
||||||
|
}
|
||||||
|
|
||||||
|
void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
|
||||||
|
void (*task_work_cb)(struct io_uring_cmd *))
|
||||||
|
{
|
||||||
|
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
|
||||||
|
|
||||||
|
ioucmd->task_work_cb = task_work_cb;
|
||||||
|
req->io_task_work.func = io_uring_cmd_work;
|
||||||
|
io_req_task_work_add(req);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);
|
||||||
|
|
||||||
|
static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
|
||||||
|
u64 extra1, u64 extra2)
|
||||||
|
{
|
||||||
|
req->extra1 = extra1;
|
||||||
|
req->extra2 = extra2;
|
||||||
|
req->flags |= REQ_F_CQE32_INIT;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Called by consumers of io_uring_cmd, if they originally returned
|
||||||
|
* -EIOCBQUEUED upon receiving the command.
|
||||||
|
*/
|
||||||
|
void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
|
||||||
|
{
|
||||||
|
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
|
||||||
|
|
||||||
|
if (ret < 0)
|
||||||
|
req_set_fail(req);
|
||||||
|
|
||||||
|
io_req_set_res(req, 0, ret);
|
||||||
|
if (req->ctx->flags & IORING_SETUP_CQE32)
|
||||||
|
io_req_set_cqe32_extra(req, res2, 0);
|
||||||
|
__io_req_complete(req, 0);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
|
||||||
|
|
||||||
|
int io_uring_cmd_prep_async(struct io_kiocb *req)
|
||||||
|
{
|
||||||
|
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
||||||
|
size_t cmd_size;
|
||||||
|
|
||||||
|
cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
|
||||||
|
|
||||||
|
memcpy(req->async_data, ioucmd->cmd, cmd_size);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
||||||
|
{
|
||||||
|
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
||||||
|
|
||||||
|
if (sqe->rw_flags || sqe->__pad1)
|
||||||
|
return -EINVAL;
|
||||||
|
ioucmd->cmd = sqe->cmd;
|
||||||
|
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
|
||||||
|
{
|
||||||
|
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
|
||||||
|
struct io_ring_ctx *ctx = req->ctx;
|
||||||
|
struct file *file = req->file;
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
if (!req->file->f_op->uring_cmd)
|
||||||
|
return -EOPNOTSUPP;
|
||||||
|
|
||||||
|
if (ctx->flags & IORING_SETUP_SQE128)
|
||||||
|
issue_flags |= IO_URING_F_SQE128;
|
||||||
|
if (ctx->flags & IORING_SETUP_CQE32)
|
||||||
|
issue_flags |= IO_URING_F_CQE32;
|
||||||
|
if (ctx->flags & IORING_SETUP_IOPOLL)
|
||||||
|
issue_flags |= IO_URING_F_IOPOLL;
|
||||||
|
|
||||||
|
if (req_has_async_data(req))
|
||||||
|
ioucmd->cmd = req->async_data;
|
||||||
|
|
||||||
|
ret = file->f_op->uring_cmd(ioucmd, issue_flags);
|
||||||
|
if (ret == -EAGAIN) {
|
||||||
|
if (!req_has_async_data(req)) {
|
||||||
|
if (io_alloc_async_data(req))
|
||||||
|
return -ENOMEM;
|
||||||
|
io_uring_cmd_prep_async(req);
|
||||||
|
}
|
||||||
|
return -EAGAIN;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ret != -EIOCBQUEUED) {
|
||||||
|
io_uring_cmd_done(ioucmd, ret, 0);
|
||||||
|
return IOU_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
return IOU_ISSUE_SKIP_COMPLETE;
|
||||||
|
}
|
13
io_uring/uring_cmd.h
Normal file
13
io_uring/uring_cmd.h
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
// SPDX-License-Identifier: GPL-2.0
|
||||||
|
|
||||||
|
int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);
|
||||||
|
int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
|
||||||
|
int io_uring_cmd_prep_async(struct io_kiocb *req);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The URING_CMD payload starts at 'cmd' in the first sqe, and continues into
|
||||||
|
* the following sqe if SQE128 is used.
|
||||||
|
*/
|
||||||
|
#define uring_cmd_pdu_size(is_sqe128) \
|
||||||
|
((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) - \
|
||||||
|
offsetof(struct io_uring_sqe, cmd))
|
Loading…
Reference in New Issue
Block a user