commit 3556a7bed5bbe67d4f33084529758e705ccc1782 Author: Ian Gulliver Date: Sat May 18 23:19:01 2019 -0700 Read/write loop test working diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..3c316f8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,151 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseTab: Never +... + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3e990f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.a +*.so +*.o +*.swp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..154c42e --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +FIRE_CXX ?= clang++ +FIRE_CXXFLAGS ?= -O3 -std=gnu++2a -Wall -Werror -Wextra -fPIE -fPIC -fstack-protector-strong -fsanitize=safe-stack -fsanitize=safe-stack +FIRE_LDFLAGS ?= -fuse-ld=gold -flto -Wl,-z,relro -Wl,-z,now +FIRE_LDLIBS ?= -lglog -lgflags -luring + +all: liburing++.a liburing++.o liburing++.so test_loop + +objects = uring.o + +liburing++.a: $(objects) + ar rcs $@ $^ + +liburing++.o: $(objects) + gold -z relro -z now -r --output=$@ $+ + +liburing++.so: $(objects) + $(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -shared -o $@ $+ $(FIRE_LDLIBS) + +test_loop: test_loop.o uring.o + $(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -pie -o $@ $+ $(FIRE_LDLIBS) + +%.o: %.cc *.h Makefile + $(FIRE_CXX) $(FIRE_CXXFLAGS) -c -o $@ $< + +clean: + rm --force *.so *.o *.a + +asan: + $(MAKE) clean + FIRE_CXXFLAGS="-O1 -g -fsanitize=address -fno-omit-frame-pointer -std=gnu++2a -fPIE -fPIC" $(MAKE) all + +tsan: + $(MAKE) clean + FIRE_CXXFLAGS="-O1 -g -fsanitize=thread -std=gnu++2a -fPIE -fPIC" $(MAKE) all + +ubsan: + $(MAKE) clean + FIRE_CXXFLAGS="-O1 -g -fsanitize=undefined -std=gnu++2a -fPIE -fPIC" $(MAKE) all diff --git a/README.md b/README.md new file mode 100644 index 0000000..c39f1b7 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +Idiomatic C++ wrapper around [liburing](http://git.kernel.dk/cgit/liburing/) diff --git a/test_loop b/test_loop new file mode 100755 index 0000000..155c2aa Binary files /dev/null and b/test_loop differ diff --git a/test_loop.cc b/test_loop.cc new file mode 100644 index 0000000..2794d7e --- /dev/null +++ b/test_loop.cc @@ -0,0 +1,40 @@ +#include "uring.h" + +#include +#include + +int main(int argc, char* argv[]) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + + uring::URing uring(1); + + int sv[2]; + PCHECK(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0); + + std::string foo("foo\n"); + + bool fired = false; + uring.write(sv[0], foo.data(), foo.size(), [&](int32_t res) { + fired = true; + LOG(INFO) << "write() res=" << res; + CHECK_EQ(res, 4); + }); + uring.Submit(); + uring.Wait(); + CHECK(fired); + + char buf[10]; + fired = false; + uring.read(sv[1], buf, 10, [&](int32_t res) { + fired = true; + LOG(INFO) << "read() res=" << res; + CHECK_EQ(res, 4); + CHECK_EQ(memcmp(buf, foo.data(), foo.size()), 0); + }); + uring.Submit(); + uring.Wait(); + CHECK(fired); + + CHECK_EQ(uring.Try(), false); +} diff --git a/uring.cc b/uring.cc new file mode 100644 index 0000000..292a5d7 --- /dev/null +++ b/uring.cc @@ -0,0 +1,122 @@ +#include "uring.h" + +#include + +namespace uring { + +URing::URing(uint32_t num_entries) + : ring_fd_(io_uring_queue_init(num_entries, &ring_, 0)), + entries_(num_entries * 3), + free_head_(&entries_.at(0)) { + PCHECK(ring_fd_ >= 0) << "io_uring_queue_init()"; + + // Chain the entries together into a list that we'll use as a free list for + // future allocation. + Entry *prev = nullptr; + for (auto &entry : entries_) { + entry.free_next = prev; + prev = &entry; + } +} + +URing::~URing() { io_uring_queue_exit(&ring_); } + +void URing::Submit() { io_uring_submit(&ring_); } + +void URing::Wait() { + io_uring_cqe *cqe; + PCHECK(io_uring_wait_cqe(&ring_, &cqe) == 0); + ProcessCQE(cqe); +} + +bool URing::Try() { + io_uring_cqe *cqe = nullptr; + PCHECK(io_uring_peek_cqe(&ring_, &cqe) == 0); + if (cqe) { + ProcessCQE(cqe); + return true; + } + return false; +} + +void URing::write(int fd, const void *buf, size_t count, + const std::function &callback) { + std::vector vecs{ + { + .iov_base = const_cast(buf), + .iov_len = count, + }, + }; + writev(fd, vecs, callback); +} + +void URing::writev(int fd, const std::vector &vecs, + const std::function &callback) { + pwritev(fd, vecs, 0, callback); +} + +void URing::pwritev(int fd, const std::vector &vecs, off_t offset, + const std::function &callback) { + auto *entry = GetEntry(callback); + entry->vecs = vecs; + + auto *sqe = GetSQE(); + io_uring_prep_writev(sqe, fd, entry->vecs.data(), entry->vecs.size(), offset); + io_uring_sqe_set_data(sqe, reinterpret_cast(entry)); +} + +void URing::read(int fd, const void *buf, size_t count, + const std::function &callback) { + std::vector vecs{ + { + .iov_base = const_cast(buf), + .iov_len = count, + }, + }; + readv(fd, vecs, callback); +} + +void URing::readv(int fd, const std::vector &vecs, + const std::function &callback) { + preadv(fd, vecs, 0, callback); +} + +void URing::preadv(int fd, const std::vector &vecs, off_t offset, + const std::function &callback) { + auto *entry = GetEntry(callback); + entry->vecs = vecs; + + auto *sqe = GetSQE(); + io_uring_prep_readv(sqe, fd, entry->vecs.data(), entry->vecs.size(), offset); + io_uring_sqe_set_data(sqe, reinterpret_cast(entry)); +} + +URing::Entry *URing::GetEntry( + const std::function &callback) { + CHECK(free_head_); + auto *entry = free_head_; + free_head_ = entry->free_next; + entry->callback = callback; + return entry; +} + +void URing::PutEntry(Entry *entry, int32_t res) { + entry->callback(res); + entry->free_next = free_head_; + free_head_ = entry; +} + +io_uring_sqe *URing::GetSQE() { + auto *sqe = io_uring_get_sqe(&ring_); + // TODO: automatically Submit() on full submit queue? spin? something else? + CHECK(sqe); + return sqe; +} + +void URing::ProcessCQE(io_uring_cqe *cqe) { + Entry *entry = reinterpret_cast(io_uring_cqe_get_data(cqe)); + PutEntry(entry, cqe->res); + io_uring_cqe_seen(&ring_, cqe); +} + +} // namespace uring diff --git a/uring.h b/uring.h new file mode 100644 index 0000000..ab12ad9 --- /dev/null +++ b/uring.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include + +namespace uring { + +// Not thread safe. Instantiate one per thread instead. +class URing { + public: + URing(uint32_t num_entries); + ~URing(); + + // TODO: write/writev don't work for files because they always use offset 0 + void write(int fd, const void *buf, size_t count, + const std::function &callback); + void writev(int fd, const std::vector &vecs, + const std::function &callback); + void pwritev(int fd, const std::vector &vecs, off_t offset, + const std::function &callback); + + // TODO: read/readv don't work for files because they always use offset 0 + void read(int fd, const void *buf, size_t count, + const std::function &callback); + void readv(int fd, const std::vector &vecs, + const std::function &callback); + void preadv(int fd, const std::vector &vecs, off_t offset, + const std::function &callback); + + // Submit all operations queued since the last Submit(). They must all be in + // a valid state when this is called. + void Submit(); + + // Wait for one operation to complete and synchronously call its callback. + void Wait(); + + // If an operation is complete, synchronously call its callback and return + // true. + bool Try(); + + private: + io_uring ring_; + int ring_fd_; + + struct Entry { + Entry *free_next; + std::vector vecs; + std::function callback; + }; + + std::vector entries_; + Entry *free_head_ = nullptr; + + Entry *GetEntry(const std::function &callback); + void PutEntry(Entry *entry, int32_t res); + + io_uring_sqe *GetSQE(); + + void ProcessCQE(io_uring_cqe *cqe); +}; + +} // namespace uring