Read/write loop test working

.clang-format

@@ -0,0 +1,151 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories: 
+  - Regex:           '^<ext/.*\.h>'
+    Priority:        2
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+RawStringFormats: 
+  - Language:        Cpp
+    Delimiters:      
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+  - Language:        TextProto
+    Delimiters:      
+      - pb
+      - PB
+      - proto
+      - PROTO
+    EnclosingFunctions: 
+      - EqualsProto
+      - EquivToProto
+      - PARSE_PARTIAL_TEXT_PROTO
+      - PARSE_TEST_PROTO
+      - PARSE_TEXT_PROTO
+      - ParseTextOrDie
+      - ParseTextProtoOrDie
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+StatementMacros: 
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseTab:          Never
+...
+

.gitignore

@@ -0,0 +1,4 @@
+*.a
+*.so
+*.o
+*.swp

Makefile

@@ -0,0 +1,38 @@
+FIRE_CXX ?= clang++
+FIRE_CXXFLAGS ?= -O3 -std=gnu++2a -Wall -Werror -Wextra -fPIE -fPIC -fstack-protector-strong -fsanitize=safe-stack -fsanitize=safe-stack
+FIRE_LDFLAGS ?= -fuse-ld=gold -flto -Wl,-z,relro -Wl,-z,now
+FIRE_LDLIBS ?= -lglog -lgflags -luring
+
+all: liburing++.a liburing++.o liburing++.so test_loop
+
+objects = uring.o
+
+liburing++.a: $(objects)
+	ar rcs $@ $^
+
+liburing++.o: $(objects)
+	gold -z relro -z now -r --output=$@ $+
+
+liburing++.so: $(objects)
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -shared -o $@ $+ $(FIRE_LDLIBS)
+
+test_loop: test_loop.o uring.o
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -pie -o $@ $+ $(FIRE_LDLIBS)
+
+%.o: %.cc *.h Makefile
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) -c -o $@ $<
+
+clean:
+	rm --force *.so *.o *.a
+
+asan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=address -fno-omit-frame-pointer -std=gnu++2a -fPIE -fPIC" $(MAKE) all
+
+tsan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=thread -std=gnu++2a -fPIE -fPIC" $(MAKE) all
+
+ubsan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=undefined -std=gnu++2a -fPIE -fPIC" $(MAKE) all

README.md

@@ -0,0 +1 @@
+Idiomatic C++ wrapper around [liburing](http://git.kernel.dk/cgit/liburing/)

test_loop.cc

@@ -0,0 +1,40 @@
+#include "uring.h"
+
+#include <glog/logging.h>
+#include <sys/socket.h>
+
+int main(int argc, char* argv[]) {
+  google::InitGoogleLogging(argv[0]);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  uring::URing uring(1);
+
+  int sv[2];
+  PCHECK(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
+
+  std::string foo("foo\n");
+
+  bool fired = false;
+  uring.write(sv[0], foo.data(), foo.size(), [&](int32_t res) {
+    fired = true;
+    LOG(INFO) << "write() res=" << res;
+    CHECK_EQ(res, 4);
+  });
+  uring.Submit();
+  uring.Wait();
+  CHECK(fired);
+
+  char buf[10];
+  fired = false;
+  uring.read(sv[1], buf, 10, [&](int32_t res) {
+    fired = true;
+    LOG(INFO) << "read() res=" << res;
+    CHECK_EQ(res, 4);
+    CHECK_EQ(memcmp(buf, foo.data(), foo.size()), 0);
+  });
+  uring.Submit();
+  uring.Wait();
+  CHECK(fired);
+
+  CHECK_EQ(uring.Try(), false);
+}

uring.cc

@@ -0,0 +1,122 @@
+#include "uring.h"
+
+#include <glog/logging.h>
+
+namespace uring {
+
+URing::URing(uint32_t num_entries)
+    : ring_fd_(io_uring_queue_init(num_entries, &ring_, 0)),
+      entries_(num_entries * 3),
+      free_head_(&entries_.at(0)) {
+  PCHECK(ring_fd_ >= 0) << "io_uring_queue_init()";
+
+  // Chain the entries together into a list that we'll use as a free list for
+  // future allocation.
+  Entry *prev = nullptr;
+  for (auto &entry : entries_) {
+    entry.free_next = prev;
+    prev = &entry;
+  }
+}
+
+URing::~URing() { io_uring_queue_exit(&ring_); }
+
+void URing::Submit() { io_uring_submit(&ring_); }
+
+void URing::Wait() {
+  io_uring_cqe *cqe;
+  PCHECK(io_uring_wait_cqe(&ring_, &cqe) == 0);
+  ProcessCQE(cqe);
+}
+
+bool URing::Try() {
+  io_uring_cqe *cqe = nullptr;
+  PCHECK(io_uring_peek_cqe(&ring_, &cqe) == 0);
+  if (cqe) {
+    ProcessCQE(cqe);
+    return true;
+  }
+  return false;
+}
+
+void URing::write(int fd, const void *buf, size_t count,
+                  const std::function<void(int32_t res)> &callback) {
+  std::vector<iovec> vecs{
+      {
+          .iov_base = const_cast<void *>(buf),
+          .iov_len = count,
+      },
+  };
+  writev(fd, vecs, callback);
+}
+
+void URing::writev(int fd, const std::vector<iovec> &vecs,
+                   const std::function<void(int32_t res)> &callback) {
+  pwritev(fd, vecs, 0, callback);
+}
+
+void URing::pwritev(int fd, const std::vector<iovec> &vecs, off_t offset,
+                    const std::function<void(int32_t res)> &callback) {
+  auto *entry = GetEntry(callback);
+  entry->vecs = vecs;
+
+  auto *sqe = GetSQE();
+  io_uring_prep_writev(sqe, fd, entry->vecs.data(), entry->vecs.size(), offset);
+  io_uring_sqe_set_data(sqe, reinterpret_cast<void *>(entry));
+}
+
+void URing::read(int fd, const void *buf, size_t count,
+                 const std::function<void(int32_t res)> &callback) {
+  std::vector<iovec> vecs{
+      {
+          .iov_base = const_cast<void *>(buf),
+          .iov_len = count,
+      },
+  };
+  readv(fd, vecs, callback);
+}
+
+void URing::readv(int fd, const std::vector<iovec> &vecs,
+                  const std::function<void(int32_t res)> &callback) {
+  preadv(fd, vecs, 0, callback);
+}
+
+void URing::preadv(int fd, const std::vector<iovec> &vecs, off_t offset,
+                   const std::function<void(int32_t res)> &callback) {
+  auto *entry = GetEntry(callback);
+  entry->vecs = vecs;
+
+  auto *sqe = GetSQE();
+  io_uring_prep_readv(sqe, fd, entry->vecs.data(), entry->vecs.size(), offset);
+  io_uring_sqe_set_data(sqe, reinterpret_cast<void *>(entry));
+}
+
+URing::Entry *URing::GetEntry(
+    const std::function<void(int32_t res)> &callback) {
+  CHECK(free_head_);
+  auto *entry = free_head_;
+  free_head_ = entry->free_next;
+  entry->callback = callback;
+  return entry;
+}
+
+void URing::PutEntry(Entry *entry, int32_t res) {
+  entry->callback(res);
+  entry->free_next = free_head_;
+  free_head_ = entry;
+}
+
+io_uring_sqe *URing::GetSQE() {
+  auto *sqe = io_uring_get_sqe(&ring_);
+  // TODO: automatically Submit() on full submit queue? spin? something else?
+  CHECK(sqe);
+  return sqe;
+}
+
+void URing::ProcessCQE(io_uring_cqe *cqe) {
+  Entry *entry = reinterpret_cast<Entry *>(io_uring_cqe_get_data(cqe));
+  PutEntry(entry, cqe->res);
+  io_uring_cqe_seen(&ring_, cqe);
+}
+
+}  // namespace uring

uring.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <liburing.h>
+#include <cstdint>
+#include <functional>
+#include <vector>
+
+namespace uring {
+
+// Not thread safe. Instantiate one per thread instead.
+class URing {
+ public:
+  URing(uint32_t num_entries);
+  ~URing();
+
+  // TODO: write/writev don't work for files because they always use offset 0
+  void write(int fd, const void *buf, size_t count,
+             const std::function<void(int32_t res)> &callback);
+  void writev(int fd, const std::vector<iovec> &vecs,
+              const std::function<void(int32_t res)> &callback);
+  void pwritev(int fd, const std::vector<iovec> &vecs, off_t offset,
+               const std::function<void(int32_t res)> &callback);
+
+  // TODO: read/readv don't work for files because they always use offset 0
+  void read(int fd, const void *buf, size_t count,
+            const std::function<void(int32_t res)> &callback);
+  void readv(int fd, const std::vector<iovec> &vecs,
+             const std::function<void(int32_t res)> &callback);
+  void preadv(int fd, const std::vector<iovec> &vecs, off_t offset,
+              const std::function<void(int32_t res)> &callback);
+
+  // Submit all operations queued since the last Submit(). They must all be in
+  // a valid state when this is called.
+  void Submit();
+
+  // Wait for one operation to complete and synchronously call its callback.
+  void Wait();
+
+  // If an operation is complete, synchronously call its callback and return
+  // true.
+  bool Try();
+
+ private:
+  io_uring ring_;
+  int ring_fd_;
+
+  struct Entry {
+    Entry *free_next;
+    std::vector<iovec> vecs;
+    std::function<void(int32_t)> callback;
+  };
+
+  std::vector<Entry> entries_;
+  Entry *free_head_ = nullptr;
+
+  Entry *GetEntry(const std::function<void(int32_t res)> &callback);
+  void PutEntry(Entry *entry, int32_t res);
+
+  io_uring_sqe *GetSQE();
+
+  void ProcessCQE(io_uring_cqe *cqe);
+};
+
+}  // namespace uring