cpp/liburing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to newer API

Browse Source 
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe
2019-01-09 15:26:20 -07:00
parent 66a7d05bd8
commit 7bf7e8e8e5
5 changed files with 129 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/io_uring.c
View File

@@ -10,18 +10,18 @@
#include "barrier.h" #include "barrier.h"
static int __io_uring_get_completion(int fd, struct io_uring_cq *cq, static int __io_uring_get_completion(int fd, struct io_uring_cq *cq,
struct io_uring_event **ev_ptr, int wait) struct io_uring_cqe **cqe_ptr, int wait)
{ {
const unsigned mask = *cq->kring_mask; const unsigned mask = *cq->kring_mask;
unsigned head; unsigned head;
int ret; int ret;
*ev_ptr = NULL; *cqe_ptr = NULL;
head = *cq->khead; head = *cq->khead;
do { do {
read_barrier(); read_barrier();
if (head != *cq->ktail) { if (head != *cq->ktail) {
*ev_ptr = &cq->events[head & mask]; *cqe_ptr = &cq->cqes[head & mask];
break; break;
} }
if (!wait) if (!wait)
@@ -31,7 +31,7 @@ static int __io_uring_get_completion(int fd, struct io_uring_cq *cq,
return -errno; return -errno;
} while (1); } while (1);
if (*ev_ptr) { if (*cqe_ptr) {
*cq->khead = head + 1; *cq->khead = head + 1;
write_barrier(); write_barrier();
} }
@@ -43,24 +43,24 @@ static int __io_uring_get_completion(int fd, struct io_uring_cq *cq,
* Return an IO completion, if one is readily available * Return an IO completion, if one is readily available
*/ */
int io_uring_get_completion(struct io_uring *ring, int io_uring_get_completion(struct io_uring *ring,
struct io_uring_event **ev_ptr) struct io_uring_cqe **cqe_ptr)
{ {
return __io_uring_get_completion(ring->ring_fd, &ring->cq, ev_ptr, 0); return __io_uring_get_completion(ring->ring_fd, &ring->cq, cqe_ptr, 0);
} }
/* /*
* Return an IO completion, waiting for it if necessary * Return an IO completion, waiting for it if necessary
*/ */
int io_uring_wait_completion(struct io_uring *ring, int io_uring_wait_completion(struct io_uring *ring,
struct io_uring_event **ev_ptr) struct io_uring_cqe **cqe_ptr)
{ {
return __io_uring_get_completion(ring->ring_fd, &ring->cq, ev_ptr, 1); return __io_uring_get_completion(ring->ring_fd, &ring->cq, cqe_ptr, 1);
} }
/* /*
* Submit iocbs acquired from io_uring_get_iocb() to the kernel. * Submit sqes acquired from io_uring_get_sqe() to the kernel.
* *
* Returns number of iocbs submitted * Returns number of sqes submitted
*/ */
int io_uring_submit(struct io_uring *ring) int io_uring_submit(struct io_uring *ring)
{ {
@@ -77,24 +77,24 @@ int io_uring_submit(struct io_uring *ring)
goto submit; goto submit;
} }
if (sq->iocb_head == sq->iocb_tail) if (sq->sqe_head == sq->sqe_tail)
return 0; return 0;
/* /*
* Fill in iocbs that we have queued up, adding them to the kernel ring * Fill in sqes that we have queued up, adding them to the kernel ring
*/ */
submitted = 0; submitted = 0;
ktail = ktail_next = *sq->ktail; ktail = ktail_next = *sq->ktail;
while (sq->iocb_head < sq->iocb_tail) { while (sq->sqe_head < sq->sqe_tail) {
ktail_next++; ktail_next++;
read_barrier(); read_barrier();
if (ktail_next == *sq->khead) if (ktail_next == *sq->khead)
break; break;
sq->array[ktail & mask] = sq->iocb_head & mask; sq->array[ktail & mask] = sq->sqe_head & mask;
ktail = ktail_next; ktail = ktail_next;
sq->iocb_head++; sq->sqe_head++;
submitted++; submitted++;
} }
@@ -113,27 +113,27 @@ submit:
} }
/* /*
* Return an iocb to fill. Application must later call io_uring_submit() * Return an sqe to fill. Application must later call io_uring_submit()
* when it's ready to tell the kernel about it. The caller may call this * when it's ready to tell the kernel about it. The caller may call this
* function multiple times before calling io_uring_submit(). * function multiple times before calling io_uring_submit().
* *
* Returns a vacant iocb, or NULL if we're full. * Returns a vacant sqe, or NULL if we're full.
*/ */
struct io_uring_iocb *io_uring_get_iocb(struct io_uring *ring) struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
{ {
struct io_uring_sq *sq = &ring->sq; struct io_uring_sq *sq = &ring->sq;
unsigned next = sq->iocb_tail + 1; unsigned next = sq->sqe_tail + 1;
struct io_uring_iocb *iocb; struct io_uring_sqe *sqe;
/* /*
* All iocbs are used * All sqes are used
*/ */
if (next - sq->iocb_head > *sq->kring_entries) if (next - sq->sqe_head > *sq->kring_entries)
return NULL; return NULL;
iocb = &sq->iocbs[sq->iocb_tail & *sq->kring_mask]; sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
sq->iocb_tail = next; sq->sqe_tail = next;
return iocb; return sqe;
} }
static int io_uring_mmap(int fd, struct io_uring_params *p, static int io_uring_mmap(int fd, struct io_uring_params *p,
@@ -156,23 +156,23 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
sq->kdropped = ptr + p->sq_off.dropped; sq->kdropped = ptr + p->sq_off.dropped;
sq->array = ptr + p->sq_off.array; sq->array = ptr + p->sq_off.array;
size = p->sq_entries * sizeof(struct io_uring_iocb); size = p->sq_entries * sizeof(struct io_uring_sqe),
sq->iocbs = mmap(0, size, PROT_READ | PROT_WRITE, sq->sqes = mmap(0, size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, MAP_SHARED | MAP_POPULATE, fd,
IORING_OFF_IOCB); IORING_OFF_SQES);
if (sq->iocbs == MAP_FAILED) { if (sq->sqes == MAP_FAILED) {
ret = -errno; ret = -errno;
err: err:
munmap(sq->khead, sq->ring_sz); munmap(sq->khead, sq->ring_sz);
return ret; return ret;
} }
cq->ring_sz = p->cq_off.events + p->cq_entries * sizeof(struct io_uring_event); cq->ring_sz = p->cq_off.cqes + p->cq_entries * sizeof(struct io_uring_cqe);
ptr = mmap(0, cq->ring_sz, PROT_READ | PROT_WRITE, ptr = mmap(0, cq->ring_sz, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING); MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
if (ptr == MAP_FAILED) { if (ptr == MAP_FAILED) {
ret = -errno; ret = -errno;
munmap(sq->iocbs, p->sq_entries * sizeof(struct io_uring_iocb)); munmap(sq->sqes, p->sq_entries * sizeof(struct io_uring_sqe));
goto err; goto err;
} }
cq->khead = ptr + p->cq_off.head; cq->khead = ptr + p->cq_off.head;
@@ -180,7 +180,7 @@ err:
cq->kring_mask = ptr + p->cq_off.ring_mask; cq->kring_mask = ptr + p->cq_off.ring_mask;
cq->kring_entries = ptr + p->cq_off.ring_entries; cq->kring_entries = ptr + p->cq_off.ring_entries;
cq->koverflow = ptr + p->cq_off.overflow; cq->koverflow = ptr + p->cq_off.overflow;
cq->events = ptr + p->cq_off.events; cq->cqes = ptr + p->cq_off.cqes;
return 0; return 0;
} }
@@ -209,7 +209,7 @@ void io_uring_queue_exit(struct io_uring *ring)
struct io_uring_sq *sq = &ring->sq; struct io_uring_sq *sq = &ring->sq;
struct io_uring_cq *cq = &ring->cq; struct io_uring_cq *cq = &ring->cq;
munmap(sq->iocbs, *sq->kring_entries * sizeof(struct io_uring_iocb)); munmap(sq->sqes, *sq->kring_entries * sizeof(struct io_uring_sqe));
munmap(sq->khead, sq->ring_sz); munmap(sq->khead, sq->ring_sz);
munmap(cq->khead, cq->ring_sz); munmap(cq->khead, cq->ring_sz);
close(ring->ring_fd); close(ring->ring_fd);

27
src/io_uring.h
View File

@@ -12,9 +12,9 @@
#include <linux/types.h> #include <linux/types.h>
/* /*
* IO submission data structure * IO submission data structure (Submission Queue Entry)
*/ */
struct io_uring_iocb { struct io_uring_sqe {
__u8 opcode; __u8 opcode;
__u8 flags; __u8 flags;
__u16 ioprio; __u16 ioprio;
@@ -35,23 +35,22 @@ struct io_uring_iocb {
* io_uring_setup() flags * io_uring_setup() flags
*/ */
#define IORING_SETUP_IOPOLL (1 << 0) /* io_context is polled */ #define IORING_SETUP_IOPOLL (1 << 0) /* io_context is polled */
#define IORING_SETUP_FIXEDBUFS (1 << 1) /* IO buffers are fixed */ #define IORING_SETUP_SQTHREAD (1 << 1) /* Use SQ thread */
#define IORING_SETUP_SQTHREAD (1 << 2) /* Use SQ thread */ #define IORING_SETUP_SQWQ (1 << 2) /* Use SQ workqueue */
#define IORING_SETUP_SQWQ (1 << 3) /* Use SQ workqueue */ #define IORING_SETUP_SQPOLL (1 << 3) /* SQ thread polls */
#define IORING_SETUP_SQPOLL (1 << 4) /* SQ thread polls */
#define IORING_OP_READ 1 #define IORING_OP_READV 1
#define IORING_OP_WRITE 2 #define IORING_OP_WRITEV 2
#define IORING_OP_FSYNC 3 #define IORING_OP_FSYNC 3
#define IORING_OP_FDSYNC 4 #define IORING_OP_FDSYNC 4
#define IORING_OP_READ_FIXED 5 #define IORING_OP_READ_FIXED 5
#define IORING_OP_WRITE_FIXED 6 #define IORING_OP_WRITE_FIXED 6
/* /*
* IO completion data structure * IO completion data structure (Completion Queue Entry)
*/ */
struct io_uring_event { struct io_uring_cqe {
__u64 index; /* what iocb this event came from */ __u64 index; /* what sqe this event came from */
__s32 res; /* result code for this event */ __s32 res; /* result code for this event */
__u32 flags; __u32 flags;
}; };
@@ -59,14 +58,14 @@ struct io_uring_event {
/* /*
* io_uring_event->flags * io_uring_event->flags
*/ */
#define IOEV_FLAG_CACHEHIT (1 << 0) /* IO did not hit media */ #define IOCQE_FLAG_CACHEHIT (1 << 0) /* IO did not hit media */
/* /*
* Magic offsets for the application to mmap the data it needs * Magic offsets for the application to mmap the data it needs
*/ */
#define IORING_OFF_SQ_RING 0ULL #define IORING_OFF_SQ_RING 0ULL
#define IORING_OFF_CQ_RING 0x8000000ULL #define IORING_OFF_CQ_RING 0x8000000ULL
#define IORING_OFF_IOCB 0x10000000ULL #define IORING_OFF_SQES 0x10000000ULL
/* /*
* Filled with the offset for mmap(2) * Filled with the offset for mmap(2)
@@ -90,7 +89,7 @@ struct io_cqring_offsets {
__u32 ring_mask; __u32 ring_mask;
__u32 ring_entries; __u32 ring_entries;
__u32 overflow; __u32 overflow;
__u32 events; __u32 cqes;
__u32 resv[4]; __u32 resv[4];
}; };

20
src/liburing.h
View File

@@ -15,10 +15,10 @@ struct io_uring_sq {
unsigned *kflags; unsigned *kflags;
unsigned *kdropped; unsigned *kdropped;
unsigned *array; unsigned *array;
struct io_uring_iocb *iocbs; struct io_uring_sqe *sqes;
unsigned iocb_head; unsigned sqe_head;
unsigned iocb_tail; unsigned sqe_tail;
size_t ring_sz; size_t ring_sz;
}; };
@@ -29,7 +29,7 @@ struct io_uring_cq {
unsigned *kring_mask; unsigned *kring_mask;
unsigned *kring_entries; unsigned *kring_entries;
unsigned *koverflow; unsigned *koverflow;
struct io_uring_event *events; struct io_uring_cqe *cqes;
size_t ring_sz; size_t ring_sz;
}; };
@@ -55,16 +55,16 @@ extern int io_uring_queue_init(unsigned entries, struct io_uring_params *p,
struct iovec *iovecs, struct io_uring *ring); struct iovec *iovecs, struct io_uring *ring);
extern void io_uring_queue_exit(struct io_uring *ring); extern void io_uring_queue_exit(struct io_uring *ring);
extern int io_uring_get_completion(struct io_uring *ring, extern int io_uring_get_completion(struct io_uring *ring,
struct io_uring_event **ev_ptr); struct io_uring_cqe **cqe_ptr);
extern int io_uring_wait_completion(struct io_uring *ring, extern int io_uring_wait_completion(struct io_uring *ring,
struct io_uring_event **ev_ptr); struct io_uring_cqe **cqe_ptr);
extern int io_uring_submit(struct io_uring *ring); extern int io_uring_submit(struct io_uring *ring);
extern struct io_uring_iocb *io_uring_get_iocb(struct io_uring *ring); extern struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring);
static inline struct io_uring_iocb * static inline struct io_uring_sqe *
io_uring_iocb_from_ev(struct io_uring *ring, struct io_uring_event *ev) io_uring_sqe_from_cqe(struct io_uring *ring, struct io_uring_cqe *cqe)
{ {
return &ring->sq.iocbs[ev->index]; return &ring->sq.sqes[cqe->index];
} }
#endif #endif

85
test/io_uring-cp.c
View File

@@ -16,7 +16,7 @@
static struct io_uring in_ring; static struct io_uring in_ring;
static struct io_uring out_ring; static struct io_uring out_ring;
static void *bufs[QD]; static struct iovec iovecs[QD];
static int setup_context(unsigned entries, struct io_uring *ring, int offload) static int setup_context(unsigned entries, struct io_uring *ring, int offload)
{ {
@@ -50,26 +50,27 @@ static int get_file_size(int fd, off_t *size)
return -1; return -1;
} }
static unsigned iocb_index(struct io_uring_iocb *iocb) static unsigned sqe_index(struct io_uring_sqe *sqe)
{ {
return iocb - in_ring.sq.iocbs; return sqe - in_ring.sq.sqes;
} }
static int queue_read(int fd, off_t size, off_t offset) static int queue_read(int fd, off_t size, off_t offset)
{ {
struct io_uring_iocb *iocb; struct io_uring_sqe *sqe;
iocb = io_uring_get_iocb(&in_ring); sqe = io_uring_get_sqe(&in_ring);
if (!iocb) if (!sqe)
return 1; return 1;
iocb->opcode = IORING_OP_READ; sqe->opcode = IORING_OP_READV;
iocb->flags = 0; sqe->flags = 0;
iocb->ioprio = 0; sqe->ioprio = 0;
iocb->fd = fd; sqe->fd = fd;
iocb->off = offset; sqe->off = offset;
iocb->addr = bufs[iocb_index(iocb)]; sqe->addr = &iovecs[sqe_index(sqe)];
iocb->len = size; iovecs[sqe_index(sqe)].iov_len = size;
sqe->len = 1;
return 0; return 0;
} }
@@ -85,16 +86,16 @@ static int complete_writes(unsigned *writes)
nr = ret; nr = ret;
while (nr) { while (nr) {
struct io_uring_event *ev = NULL; struct io_uring_cqe *cqe;
ret = io_uring_wait_completion(&out_ring, &ev); ret = io_uring_wait_completion(&out_ring, &cqe);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "io_uring_wait_completion: %s\n", fprintf(stderr, "io_uring_wait_completion: %s\n",
strerror(-ret)); strerror(-ret));
return 1; return 1;
} }
if (ev->res < 0) { if (cqe->res < 0) {
fprintf(stderr, "ev failed: %s\n", strerror(-ev->res)); fprintf(stderr, "cqe failed: %s\n", strerror(-cqe->res));
return 1; return 1;
} }
(*writes)--; (*writes)--;
@@ -106,22 +107,23 @@ static int complete_writes(unsigned *writes)
static void queue_write(int fd, off_t size, off_t offset, unsigned index) static void queue_write(int fd, off_t size, off_t offset, unsigned index)
{ {
struct io_uring_iocb *iocb; struct io_uring_sqe *sqe;
iocb = io_uring_get_iocb(&out_ring); sqe = io_uring_get_sqe(&out_ring);
iocb->opcode = IORING_OP_WRITE; sqe->opcode = IORING_OP_WRITEV;
iocb->flags = 0; sqe->flags = 0;
iocb->ioprio = 0; sqe->ioprio = 0;
iocb->fd = fd; sqe->fd = fd;
iocb->off = offset; sqe->off = offset;
iocb->addr = bufs[index]; sqe->addr = &iovecs[index];
iocb->len = size; iovecs[index].iov_len = size;
sqe->len = 1;
} }
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
struct io_uring_event *ev;
off_t read_left, write_left, offset; off_t read_left, write_left, offset;
struct io_uring_cqe *cqe;
int i, infd, outfd, ret; int i, infd, outfd, ret;
unsigned reads, writes; unsigned reads, writes;
@@ -141,9 +143,14 @@ int main(int argc, char *argv[])
return 1; return 1;
} }
for (i = 0; i < QD; i++) for (i = 0; i < QD; i++) {
if (posix_memalign(&bufs[i], BS, BS)) void *buf;
if (posix_memalign(&buf, BS, BS))
return 1; return 1;
iovecs[i].iov_base = buf;
iovecs[i].iov_len = BS;
}
if (setup_context(QD, &in_ring, 1)) if (setup_context(QD, &in_ring, 1))
return 1; return 1;
@@ -157,7 +164,7 @@ int main(int argc, char *argv[])
write_left = read_left; write_left = read_left;
while (read_left || write_left) { while (read_left || write_left) {
off_t this_size = read_left; off_t this_size = read_left;
struct io_uring_iocb *iocb; struct io_uring_sqe *sqe;
if (this_size > BS) if (this_size > BS)
this_size = BS; this_size = BS;
@@ -186,25 +193,25 @@ skip_read:
*/ */
while (reads || write_left) { while (reads || write_left) {
if (reads) if (reads)
ret = io_uring_wait_completion(&in_ring, &ev); ret = io_uring_wait_completion(&in_ring, &cqe);
else else
ret = io_uring_get_completion(&in_ring, &ev); ret = io_uring_get_completion(&in_ring, &cqe);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "io_uring_get_completion: %s\n", fprintf(stderr, "io_uring_get_completion: %s\n",
strerror(-ret)); strerror(-ret));
return 1; return 1;
} }
if (!ev) if (!cqe)
break; break;
reads--; reads--;
if (ev->res < 0) { if (cqe->res < 0) {
fprintf(stderr, "ev failed: %s\n", fprintf(stderr, "cqe failed: %s\n",
strerror(-ev->res)); strerror(-cqe->res));
return 1; return 1;
} }
iocb = io_uring_iocb_from_ev(&in_ring, ev); sqe = io_uring_sqe_from_cqe(&in_ring, cqe);
queue_write(outfd, ev->res, iocb->off, ev->index); queue_write(outfd, cqe->res, sqe->off, cqe->index);
write_left -= ev->res; write_left -= cqe->res;
writes++; writes++;
}; };
if (complete_writes(&writes)) if (complete_writes(&writes))

46
test/io_uring-test.c
View File

@@ -11,13 +11,16 @@
#include <unistd.h> #include <unistd.h>
#include "../src/liburing.h" #include "../src/liburing.h"
#define QD 4
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
struct io_uring_params p; struct io_uring_params p;
struct io_uring ring; struct io_uring ring;
int i, fd, ret, pending, done; int i, fd, ret, pending, done;
struct io_uring_iocb *iocb; struct io_uring_sqe *sqe;
struct io_uring_event *ev; struct io_uring_cqe *cqe;
struct iovec *iovecs;
off_t offset; off_t offset;
void *buf; void *buf;
@@ -29,7 +32,7 @@ int main(int argc, char *argv[])
memset(&p, 0, sizeof(p)); memset(&p, 0, sizeof(p));
p.flags = IORING_SETUP_IOPOLL; p.flags = IORING_SETUP_IOPOLL;
ret = io_uring_queue_init(4, &p, NULL, &ring); ret = io_uring_queue_init(QD, &p, NULL, &ring);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "queue_init: %s\n", strerror(-ret)); fprintf(stderr, "queue_init: %s\n", strerror(-ret));
return 1; return 1;
@@ -41,22 +44,28 @@ int main(int argc, char *argv[])
return 1; return 1;
} }
if (posix_memalign(&buf, 4096, 4096)) iovecs = calloc(QD, sizeof(struct iovec));
return 1; for (i = 0; i < QD; i++) {
if (posix_memalign(&buf, 4096, 4096))
return 1;
iovecs[i].iov_base = buf;
iovecs[i].iov_len = 4096;
}
offset = 0; offset = 0;
i = 0;
do { do {
iocb = io_uring_get_iocb(&ring); sqe = io_uring_get_sqe(&ring);
if (!iocb) if (!sqe)
break; break;
iocb->opcode = IORING_OP_READ; sqe->opcode = IORING_OP_READV;
iocb->flags = 0; sqe->flags = 0;
iocb->ioprio = 0; sqe->ioprio = 0;
iocb->fd = fd; sqe->fd = fd;
iocb->off = offset; sqe->off = offset;
iocb->addr = buf; sqe->addr = &iovecs[i];
iocb->len = 4096; sqe->len = 1;
offset += 4096; offset += iovecs[i].iov_len;
} while (1); } while (1);
ret = io_uring_submit(&ring); ret = io_uring_submit(&ring);
@@ -68,16 +77,15 @@ int main(int argc, char *argv[])
done = 0; done = 0;
pending = ret; pending = ret;
for (i = 0; i < pending; i++) { for (i = 0; i < pending; i++) {
ev = NULL; ret = io_uring_get_completion(&ring, &cqe);
ret = io_uring_get_completion(&ring, &ev);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "io_uring_get_completion: %s\n", strerror(-ret)); fprintf(stderr, "io_uring_get_completion: %s\n", strerror(-ret));
return 1; return 1;
} }
done++; done++;
if (ev->res != 4096) { if (cqe->res != 4096) {
fprintf(stderr, "ret=%d, wanted 4096\n", ev->res); fprintf(stderr, "ret=%d, wanted 4096\n", cqe->res);
return 1; return 1;
} }
} }