Typos

.clang-format

@@ -0,0 +1,151 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories: 
+  - Regex:           '^<ext/.*\.h>'
+    Priority:        2
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+RawStringFormats: 
+  - Language:        Cpp
+    Delimiters:      
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+  - Language:        TextProto
+    Delimiters:      
+      - pb
+      - PB
+      - proto
+      - PROTO
+    EnclosingFunctions: 
+      - EqualsProto
+      - EquivToProto
+      - PARSE_PARTIAL_TEXT_PROTO
+      - PARSE_TEST_PROTO
+      - PARSE_TEXT_PROTO
+      - ParseTextOrDie
+      - ParseTextProtoOrDie
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+StatementMacros: 
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseTab:          Never
+...
+

.gitignore

@@ -1,5 +1,6 @@
 *.o
 *.a
+*.so
 example_simple
 connection_afl
 findings

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "firebuf"]
 	path = firebuf
 	url = ../firebuf.git
+[submodule "fireusage"]
+	path = fireusage
+	url = ../fireusage.git

Makefile

@@ -1,35 +1,46 @@
 FIRE_CXX ?= clang++
-FIRE_CXXFLAGS ?= -O3 -std=gnu++2a -Wall -Werror
+FIRE_CXXFLAGS ?= -O3 -std=gnu++2a -Wall -Werror -Wextra -fPIE -fPIC -fstack-protector-strong -fsanitize=safe-stack -fsanitize=safe-stack
+FIRE_LDFLAGS ?= -fuse-ld=gold -flto -Wl,-z,relro -Wl,-z,now
 FIRE_LDLIBS ?= -lgflags -lglog -lpthread
 
-all: firecgi.a example_simple
+all: firecgi.a firecgi.o firecgi.so example_simple
 
-objects = firecgi.o connection.o request.o parse.o
+objects = server.o connection.o request.o parse.o
+
+firebuf/firebuf.o:
+	$(MAKE) --directory=firebuf firebuf.o
+
+fireusage/fireusage.o:
+	$(MAKE) --directory=fireusage fireusage.o
 
 firecgi.a: $(objects)
-	$(MAKE) --directory=firebuf
 	ar rcs $@ $^
 
-example_simple: example_simple.o $(objects)
-	$(MAKE) --directory=firebuf
-	$(FIRE_CXX) $(FIRE_CXXFLAGS) -o $@ $+ firebuf/firebuf.a $(FIRE_LDLIBS)
+firecgi.o: $(objects) firebuf/firebuf.o fireusage/fireusage.o
+	gold -z relro -z now -r --output=$@ $+
+
+firecgi.so: $(objects) firebuf/firebuf.o fireusage/fireusage.o
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -shared -o $@ $+ $(FIRE_LDLIBS)
+
+example_simple: example_simple.o firecgi.o
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -pie -o $@ $+ $(FIRE_LDLIBS)
 
 %.o: %.cc *.h Makefile
 	$(FIRE_CXX) $(FIRE_CXXFLAGS) -c -o $@ $<
 
 clean:
 	$(MAKE) --directory=firebuf clean
-	rm --force example_simple connection_afl *.o *.a
+	$(MAKE) --directory=fireusage clean
+	rm --force example_simple connection_afl *.so *.o *.a
 
 afl:
 	$(MAKE) clean
-	FIRE_CXX=afl-g++ $(MAKE) afl_int
+	FIRE_CXX=afl-g++ FIRE_CXXFLAGS="-O3 -std=gnu++2a -fPIC -fPIE" $(MAKE) afl_int
 
 afl_int: connection_afl
 
-connection_afl: connection_afl.o $(objects)
-	$(MAKE) --directory=firebuf
-	$(FIRE_CXX) $(FIRE_CXXFLAGS) -o $@ $+ firebuf/firebuf.a $(FIRE_LDLIBS)
+connection_afl: connection_afl.o firecgi.o
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -pie -o $@ $+ $(FIRE_LDLIBS)
 
 test: test_connection
 
@@ -37,3 +48,15 @@ test_connection: connection_afl
 	@echo "Running $$(ls testcases | wc -l) tests"
 	for FILE in testcases/*; do ./connection_afl < $$FILE; done
 	@printf '\033[0;32mALL TESTS PASSED\033[0m\n'
+
+asan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=address -fno-omit-frame-pointer -std=gnu++2a -fPIE -fPIC" $(MAKE) all
+
+tsan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=thread -std=gnu++2a -fPIE -fPIC" $(MAKE) all
+
+ubsan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=undefined -std=gnu++2a -fPIE -fPIC" $(MAKE) all

connection.cc

@@ -9,15 +9,19 @@
 
 namespace firecgi {
 
-Connection::Connection(int sock, const sockaddr_in6& client_addr, const std::function<void(std::unique_ptr<Request>)>& callback, const std::unordered_set<std::string_view>& headers)
+Connection::Connection(int sock, const sockaddr_in6 &client_addr,
+                       const std::function<void(Request *)> &callback,
+                       int max_request_len)
     : sock_(sock),
       callback_(callback),
-		  headers_(headers),
-		  buf_(sock, max_record_len) {
+      buf_(sock, max_request_len),
+      request_(this) {
   char client_addr_str[INET6_ADDRSTRLEN];
-	PCHECK(inet_ntop(AF_INET6, &client_addr.sin6_addr, client_addr_str, sizeof(client_addr_str)));
+  PCHECK(inet_ntop(AF_INET6, &client_addr.sin6_addr, client_addr_str,
+                   sizeof(client_addr_str)));
 
-	LOG(INFO) << "new connection: [" << client_addr_str << "]:" << ntohs(client_addr.sin6_port);
+  LOG(INFO) << "new connection: [" << client_addr_str
+            << "]:" << ntohs(client_addr.sin6_port);
 }
 
 Connection::~Connection() {
@@ -25,9 +29,9 @@ Connection::~Connection() {
   LOG(INFO) << "connection closed (handled " << requests_ << " requests)";
 }
 
-bool Connection::Write(const std::vector<iovec>& vecs) {
+bool Connection::Write(const std::vector<iovec> &vecs) {
   ssize_t total_size = 0;
-	for (const auto& vec : vecs) {
+  for (const auto &vec : vecs) {
     total_size += vec.iov_len;
   }
   return writev(sock_, vecs.data(), vecs.size()) == total_size;
@@ -56,10 +60,10 @@ int Connection::Read() {
     }
 
     switch (header->type) {
-		  case 1:
-		  	{
+      case 1: {
         if (header->ContentLength() != sizeof(BeginRequest)) {
-					LOG(ERROR) << "FCGI_BeginRequestBody is the wrong length: " << header->ContentLength();
+          LOG(ERROR) << "FCGI_BeginRequestBody is the wrong length: "
+                     << header->ContentLength();
           return sock_;
         }
 
@@ -70,18 +74,18 @@ int Connection::Read() {
           return sock_;
         }
 
-				request_.reset(new Request(header->RequestId(), this));
-			}
-			break;
+        request_.NewRequest(header->RequestId());
+      } break;
 
-		  case 4:
-		    {
-				if (request_ == nullptr || header->RequestId() != request_->RequestId()) {
-					LOG(ERROR) << "out of order FCGI_PARAMS record, or client is multiplexing requests (which we don't support)";
+      case 4: {
+        if (header->RequestId() != request_.RequestId()) {
+          LOG(ERROR) << "out of order FCGI_PARAMS record, or client is "
+                        "multiplexing requests (which we don't support)";
           return sock_;
         }
 
-				firebuf::ConstBuffer param_buf(buf_.Read(header->ContentLength()), header->ContentLength());
+        firebuf::ConstBuffer param_buf(buf_.Read(header->ContentLength()),
+                                       header->ContentLength());
         while (param_buf.ReadMaxLen() > 0) {
           const auto *param_header = param_buf.ReadObj<ParamHeader>();
           if (!param_header) {
@@ -103,30 +107,31 @@ int Connection::Read() {
           }
           std::string_view value(value_buf, param_header->value_length);
 
-					if (headers_.find(key) != headers_.end()) {
-						request_->AddParam(key, value);
+          request_.AddParam(key, value);
         }
-				}
-			}
-			break;
+      } break;
 
-		  case 5:
-		  	{
-				if (request_ == nullptr || header->RequestId() != request_->RequestId()) {
-					LOG(ERROR) << "out of order FCGI_STDIN record, or client is multiplexing requests (which we don't support)";
+      case 5: {
+        if (header->RequestId() != request_.RequestId()) {
+          LOG(ERROR) << "out of order FCGI_STDIN record, or client is "
+                        "multiplexing requests (which we don't support)";
           return sock_;
         }
 
         if (header->ContentLength() == 0) {
           // Magic signal for completed request (mirrors the HTTP/1.1 protocol)
           requests_++;
-					callback_(std::move(request_));
+          callback_(&request_);
+          buf_.Consume();  // discard data and invalidate pointers
         } else {
-					std::string_view in(buf_.Read(header->ContentLength()), header->ContentLength());
-					request_->AddIn(in);
+          if (!request_.GetBody().empty()) {
+            LOG(ERROR) << "received multiple stdin records. have you set "
+                          "\"fastcgi_request_buffering on\"?";
           }
+          request_.SetBody(
+              {buf_.Read(header->ContentLength()), header->ContentLength()});
         }
-			break;
+      } break;
 
       default:
         LOG(ERROR) << "unknown record type: " << header->type;
@@ -140,8 +145,9 @@ int Connection::Read() {
     buf_.Commit();  // we've acted on the bytes read so far
   }
 
-	buf_.Consume();
   return -1;
 }
 
+uint64_t Connection::Requests() const { return requests_; }
+
 }  // namespace firecgi

connection.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <functional>
 #include <sys/uio.h>
+#include <functional>
 #include <unordered_map>
 #include <unordered_set>
 
@@ -13,22 +13,23 @@ namespace firecgi {
 
 class Connection {
  public:
-	Connection(int sock, const sockaddr_in6& client_addr, const std::function<void(std::unique_ptr<Request>)>& callback, const std::unordered_set<std::string_view>& headers);
+  Connection(int sock, const sockaddr_in6& client_addr,
+             const std::function<void(Request*)>& callback,
+             int max_request_len);
   ~Connection();
 
   [[nodiscard]] int Read();
   [[nodiscard]] bool Write(const std::vector<iovec>& vecs);
 
+  [[nodiscard]] uint64_t Requests() const;
+
  private:
   const int sock_;
-	const std::function<void(std::unique_ptr<Request>)>& callback_;
-	const std::unordered_set<std::string_view>& headers_;
+  const std::function<void(Request*)>& callback_;
+  firebuf::StreamBuffer buf_;
+  Request request_;
 
   uint64_t requests_ = 0;
-
-	firebuf::StreamBuffer buf_;
-
-	std::unique_ptr<Request> request_;
 };
 
 }  // namespace firecgi

connection_afl.cc

@@ -7,7 +7,9 @@ int main(int argc, char* argv[]) {
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
   {
-		firecgi::Connection conn(STDIN_FILENO, {}, [](std::unique_ptr<firecgi::Request> req) { req->End(); }, {});
+    firecgi::Connection conn(
+        STDIN_FILENO, {}, [](firecgi::Request* req) { req->End(); }, {},
+        16 * 1024);
     static_cast<void>(conn.Read());
   }
 

example_simple.cc

@@ -1,19 +1,26 @@
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
-#include "firecgi.h"
+#include "server.h"
 
 DEFINE_int32(port, 9000, "TCP port to bind");
 DEFINE_int32(threads, 1, "Number of server threads");
 
-int main(int argc, char *argv[]) {
+int main(int argc, char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
-	firecgi::Server server(FLAGS_port, [](std::unique_ptr<firecgi::Request> request) {
+  firecgi::Server server(
+      FLAGS_port,
+      [](firecgi::Request* request) {
         request->WriteHeader("Content-Type", "text/plain");
         request->WriteBody("Hello world");
         request->End();
-	}, FLAGS_threads);
+      },
+      FLAGS_threads);
+  server.RegisterSignalHandlers();
   server.Serve();
+
+  gflags::ShutDownCommandLineFlags();
+  google::ShutdownGoogleLogging();
 }

firecgi.cc

@@ -1,117 +0,0 @@
-#include <arpa/inet.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <signal.h>
-#include <sys/epoll.h>
-#include <sys/socket.h>
-#include <thread>
-
-#include "firecgi.h"
-#include "connection.h"
-
-namespace firecgi {
-
-Server::Server(int port, const std::function<void(std::unique_ptr<Request>)>& callback, int threads, const std::unordered_set<std::string_view>& headers)
-		: port_(port),
-		  callback_(callback),
-		  threads_(threads),
-		  headers_(headers) {
-	LOG(INFO) << "listening on [::1]:" << port_;
-
-	signal(SIGPIPE, SIG_IGN);
-}
-
-void Server::Serve() {
-	std::vector<std::thread> threads;
-	for (int i = 0; i < threads_ - 1; ++i) {
-		threads.emplace_back([this]() { ServeInt(); });
-	}
-	ServeInt();
-}
-
-void Server::ServeInt() {
-	auto epoll_fd = epoll_create1(0);
-	PCHECK(epoll_fd >= 0) << "epoll_create()";
-
-	auto listen_sock = NewListenSock();
-
-	{
-		struct epoll_event ev{
-			.events = EPOLLIN,
-			.data = {
-				.ptr = nullptr,
-			},
-		};
-		PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, listen_sock, &ev) == 0);
-	}
-
-	while (true) {
-		constexpr auto max_events = 256;
-		struct epoll_event events[max_events];
-		auto num_fd = epoll_wait(epoll_fd, events, max_events, -1);
-		if (num_fd == -1 && errno == EINTR) {
-			continue;
-		}
-		PCHECK(num_fd > 0) << "epoll_wait()";
-
-		for (auto i = 0; i < num_fd; ++i) {
-			if (events[i].data.ptr == nullptr) {
-				NewConn(listen_sock, epoll_fd);
-			} else {
-				auto conn = static_cast<Connection*>(events[i].data.ptr);
-				auto fd = conn->Read();
-				if (fd != -1) {
-					PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, nullptr) == 0);
-					delete conn;
-				}
-			}
-		}
-	}
-}
-
-void Server::NewConn(int listen_sock, int epoll_fd) {
-	sockaddr_in6 client_addr;
-	socklen_t client_addr_len = sizeof(client_addr);
-
-	auto client_sock = accept(listen_sock, (sockaddr*) &client_addr, &client_addr_len);
-	PCHECK(client_sock >= 0) << "accept()";
-	CHECK_EQ(client_addr.sin6_family, AF_INET6);
-
-	int flags = 1;
-	PCHECK(setsockopt(client_sock, SOL_TCP, TCP_NODELAY, &flags, sizeof(flags)) == 0);
-
-	{
-		auto *conn = new Connection(client_sock, client_addr, callback_, headers_);
-		struct epoll_event ev{
-			.events = EPOLLIN,
-			.data = {
-				.ptr = conn,
-			},
-		};
-		PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_sock, &ev) == 0);
-	}
-}
-
-int Server::NewListenSock() {
-	auto sock = socket(AF_INET6, SOCK_STREAM, 0);
-	PCHECK(sock >= 0) << "socket()";
-
-	{
-		int optval = 1;
-		PCHECK(setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &optval, sizeof(optval)) == 0);
-	}
-
-	{
-		sockaddr_in6 bind_addr = {
-			.sin6_family = AF_INET6,
-			.sin6_port = htons(port_),
-			.sin6_addr = IN6ADDR_LOOPBACK_INIT,
-		};
-		PCHECK(bind(sock, (sockaddr*) &bind_addr, sizeof(bind_addr)) == 0);
-	}
-
-	PCHECK(listen(sock, 128) == 0);
-	return sock;
-}
-
-} // namespace firecgi

firecgi.h

@@ -1,27 +0,0 @@
-#pragma once
-
-#include <functional>
-#include <memory>
-#include <unordered_set>
-
-#include "request.h"
-
-namespace firecgi {
-
-class Server {
-  public:
-	Server(int port, const std::function<void(std::unique_ptr<Request>)>& callback, int threads=1, const std::unordered_set<std::string_view>& headers={});
-	void Serve();
-
-  private:
-	void NewConn(int listen_sock, int epoll_fd);
-	int NewListenSock();
-	void ServeInt();
-
-	const int port_;
-	const std::function<void(std::unique_ptr<Request>)> callback_;
-	const int threads_;
-	const std::unordered_set<std::string_view> headers_;
-};
-
-} // firecgi

parse.h

@@ -9,6 +9,7 @@ struct Header {
 
   uint8_t version = 1;
   uint8_t type;
+
  private:
   uint16_t request_id_;      // network byte order
   uint16_t content_length_;  // network byte order
@@ -20,7 +21,9 @@ struct Header {
   uint16_t ContentLength() const { return ntohs(content_length_); }
 
   void SetRequestId(uint16_t request_id) { request_id_ = htons(request_id); }
-	void SetContentLength(uint16_t content_length) { content_length_ = htons(content_length); }
+  void SetContentLength(uint16_t content_length) {
+    content_length_ = htons(content_length);
+  }
 };
 
 struct BeginRequest {
@@ -44,8 +47,4 @@ struct ParamHeader {
   uint8_t value_length;
 };
 
-constexpr auto max_content_len = 65535;
-constexpr auto max_padding_len = 255;
-constexpr auto max_record_len = sizeof(Header) + max_content_len + max_padding_len;
-
 }  // namespace firecgi

request.cc

@@ -7,7 +7,8 @@
 namespace firecgi {
 namespace {
 
-template<class T> void AppendVec(const T& obj, std::vector<iovec>* vec) {
+template <class T>
+void AppendVec(const T& obj, std::vector<iovec>* vec) {
   vec->push_back(iovec{
       .iov_base = (void*)(&obj),
       .iov_len = sizeof(obj),
@@ -16,33 +17,55 @@ template<class T> void AppendVec(const T& obj, std::vector<iovec>* vec) {
 
 }  // namespace
 
-Request::Request(uint16_t request_id, Connection* conn)
-		: request_id_(request_id),
-		  conn_(conn),
-		  out_buf_(max_record_len) {}
+Request::Request(Connection* conn) : conn_(conn), out_buf_(64 * 1024) {}
 
-uint16_t Request::RequestId() {
-	return request_id_;
+Request::~Request() {
+  if (on_close_) {
+    on_close_();
+  }
 }
 
-void Request::AddParam(const std::string_view& key, const std::string_view& value) {
-	params_.try_emplace(std::string(key), std::string(value));
+void Request::NewRequest(uint16_t request_id) {
+  if (on_close_) {
+    on_close_();
+  }
+
+  request_id_ = request_id;
+  params_.clear();
+  body_ = {};
+  on_close_ = nullptr;
+  out_buf_.Reset();
+  body_written_ = false;
 }
 
-void Request::AddIn(const std::string_view& in) {
-	in_.append(in);
+uint16_t Request::RequestId() const { return request_id_; }
+
+void Request::AddParam(const std::string_view& key,
+                       const std::string_view& value) {
+  params_.try_emplace(key, value);
 }
 
-const std::string& Request::GetParam(const std::string& key) {
+void Request::SetBody(const std::string_view& body) { body_ = body; }
+
+const std::string_view& Request::GetParam(const std::string_view& key) const {
   auto iter = params_.find(key);
   if (iter == params_.end()) {
-		static const std::string none;
+    static const std::string_view none;
     return none;
   }
   return iter->second;
 }
 
-void Request::WriteHeader(const std::string_view& name, const std::string_view& value) {
+const std::string_view& Request::GetBody() const { return body_; }
+
+void Request::OnClose(const std::function<void()>& on_close) {
+  on_close_ = on_close;
+}
+
+void Request::WriteHeader(const std::string_view& name,
+                          const std::string_view& value) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+
   CHECK(!body_written_);
   CHECK(out_buf_.Write(name));
   CHECK(out_buf_.Write(": "));
@@ -51,6 +74,7 @@ void Request::WriteHeader(const std::string_view& name, const std::string_view&
 }
 
 void Request::WriteBody(const std::string_view& body) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
   if (!body_written_) {
     CHECK(out_buf_.Write("\n"));
     body_written_ = true;
@@ -60,6 +84,8 @@ void Request::WriteBody(const std::string_view& body) {
 }
 
 bool Request::Flush() {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+
   std::vector<iovec> vecs;
 
   auto header = OutputHeader();
@@ -71,11 +97,13 @@ bool Request::Flush() {
     return false;
   }
   out_buf_.Commit();
+  out_buf_.Consume();
   return true;
 }
 
 bool Request::End() {
-	// Fully empty response not allowed
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+
   WriteBody("");
 
   std::vector<iovec> vecs;
@@ -98,7 +126,7 @@ bool Request::End() {
 iovec Request::OutputVec() {
   const auto output_len = out_buf_.ReadMaxLen();
   return iovec{
-		.iov_base = (void *)(CHECK_NOTNULL(out_buf_.Read(output_len))),
+      .iov_base = (void*)(CHECK_NOTNULL(out_buf_.Read(output_len))),
       .iov_len = output_len,
   };
 }

request.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <functional>
+#include <mutex>
 #include <unordered_map>
 
 #include "firebuf/buffer.h"
@@ -12,32 +14,60 @@ class Connection;
 
 class Request {
  public:
-    Request(uint16_t request_id, Connection *conn);
+  Request(Connection* conn);
+  ~Request();
 
-	uint16_t RequestId();
+  void NewRequest(uint16_t request_id);
+
+  uint16_t RequestId() const;
 
   void AddParam(const std::string_view& key, const std::string_view& value);
-	void AddIn(const std::string_view& in);
+  void SetBody(const std::string_view& in);
 
-	const std::string& GetParam(const std::string& key);
+  const std::string_view& GetParam(const std::string_view& key) const;
+  const std::string_view& GetBody() const;
+
+  void OnClose(const std::function<void()>& callback);
 
   void WriteHeader(const std::string_view& name, const std::string_view& value);
   void WriteBody(const std::string_view& body);
   [[nodiscard]] bool Flush();
   bool End();
 
+  template <typename... Args>
+  void WriteBody(const std::string_view& first, Args... more);
+
+  template <typename T>
+  T InTransaction(const std::function<T()>& callback);
+
  private:
   Header OutputHeader();
   iovec OutputVec();
 
-	const uint16_t request_id_;
-	Connection *conn_;
+  Connection* conn_;
+  uint16_t request_id_ = 0;
 
-	std::unordered_map<std::string, std::string> params_;
-	std::string in_;
+  std::unordered_map<std::string_view, std::string_view> params_;
+  std::string_view body_;
+
+  std::function<void()> on_close_;
 
   firebuf::Buffer out_buf_;
-	bool body_written_ = false;
+  bool body_written_;
+  std::recursive_mutex output_mu_;
 };
 
+template <typename... Args>
+void Request::WriteBody(const std::string_view& first, Args... more) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+  WriteBody(first);
+  WriteBody(more...);
+}
+
+template <typename T>
+T Request::InTransaction(const std::function<T()>& callback) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+  return callback();
+}
+
 }  // namespace firecgi

server.cc

@@ -0,0 +1,185 @@
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <signal.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <iomanip>
+#include <thread>
+
+#include "fireusage/usage.h"
+
+#include "connection.h"
+#include "server.h"
+
+namespace firecgi {
+
+Server::Server(int port, const std::function<void(Request*)>& callback,
+               int threads, int max_request_len)
+    : port_(port),
+      callback_(callback),
+      threads_(threads),
+      max_request_len_(max_request_len),
+      close_fd_(eventfd(0, 0)) {
+  CHECK_GE(close_fd_, 0);
+
+  LOG(INFO) << "listening on [::1]:" << port_;
+  signal(SIGPIPE, SIG_IGN);
+}
+
+Server::~Server() { PCHECK(close(close_fd_) == 0); }
+
+void Server::Serve() {
+  std::vector<std::thread> threads;
+  for (int i = 0; i < threads_ - 1; ++i) {
+    threads.emplace_back([this]() { ServeInt(); });
+  }
+  ServeInt();
+  for (auto& thread : threads) {
+    thread.join();
+  }
+  LOG(INFO) << "all threads shut down";
+}
+
+void Server::Shutdown() {
+  uint64_t shutdown = 1;
+  PCHECK(write(close_fd_, &shutdown, sizeof(shutdown)) == sizeof(shutdown));
+}
+
+namespace {
+Server* shutdown_server = nullptr;
+}  // namespace
+
+void Server::RegisterSignalHandlers() {
+  shutdown_server = this;
+  for (auto sig : {SIGINT, SIGTERM}) {
+    signal(sig, [](int signum) {
+      LOG(INFO) << "received " << strsignal(signum);
+      shutdown_server->Shutdown();
+    });
+  }
+}
+
+void Server::ServeInt() {
+  auto epoll_fd = epoll_create1(0);
+  PCHECK(epoll_fd >= 0) << "epoll_create()";
+
+  auto listen_sock = NewListenSock();
+
+  char new_conn;
+  {
+    struct epoll_event ev {
+      .events = EPOLLIN,
+      .data = {
+          .ptr = &new_conn,
+      },
+    };
+    PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, listen_sock, &ev) == 0);
+  }
+
+  char shutdown;
+  {
+    struct epoll_event ev {
+      .events = EPOLLIN,
+      .data = {
+          .ptr = &shutdown,
+      },
+    };
+    PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, close_fd_, &ev) == 0);
+  }
+
+  std::unordered_set<Connection*> connections;
+
+  fireusage::UsageTracker usage_tracker;
+  usage_tracker.Start();
+
+  while (true) {
+    constexpr auto max_events = 256;
+    struct epoll_event events[max_events];
+    auto num_fd = epoll_wait(epoll_fd, events, max_events, -1);
+    if (num_fd == -1 && errno == EINTR) {
+      continue;
+    }
+    PCHECK(num_fd > 0) << "epoll_wait()";
+
+    for (auto i = 0; i < num_fd; ++i) {
+      if (events[i].data.ptr == &new_conn) {
+        connections.insert(CHECK_NOTNULL(NewConn(listen_sock, epoll_fd)));
+      } else if (events[i].data.ptr == &shutdown) {
+        for (auto& conn : connections) {
+          usage_tracker.AddEvents(conn->Requests());
+          delete conn;
+        }
+        usage_tracker.Stop();
+        PCHECK(close(listen_sock) == 0);
+        PCHECK(close(epoll_fd) == 0);
+        usage_tracker.Log("requests");
+        return;
+      } else {
+        auto conn = static_cast<Connection*>(events[i].data.ptr);
+        auto fd = conn->Read();
+        if (fd != -1) {
+          PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, nullptr) == 0);
+          usage_tracker.AddEvents(conn->Requests());
+          connections.erase(conn);
+          delete conn;
+        }
+      }
+    }
+  }
+}
+
+Connection* Server::NewConn(int listen_sock, int epoll_fd) {
+  sockaddr_in6 client_addr;
+  socklen_t client_addr_len = sizeof(client_addr);
+
+  auto client_sock =
+      accept(listen_sock, (sockaddr*)&client_addr, &client_addr_len);
+  PCHECK(client_sock >= 0) << "accept()";
+  CHECK_EQ(client_addr.sin6_family, AF_INET6);
+
+  int flags = 1;
+  PCHECK(setsockopt(client_sock, SOL_TCP, TCP_NODELAY, &flags, sizeof(flags)) ==
+         0);
+
+  auto* conn =
+      new Connection(client_sock, client_addr, callback_, max_request_len_);
+  {
+    struct epoll_event ev {
+      .events = EPOLLIN,
+      .data = {
+          .ptr = conn,
+      },
+    };
+    PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_sock, &ev) == 0);
+  }
+
+  return conn;
+}
+
+int Server::NewListenSock() {
+  auto sock = socket(AF_INET6, SOCK_STREAM, 0);
+  PCHECK(sock >= 0) << "socket()";
+
+  {
+    int optval = 1;
+    PCHECK(setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &optval,
+                      sizeof(optval)) == 0);
+  }
+
+  {
+    sockaddr_in6 bind_addr = {
+        .sin6_family = AF_INET6,
+        .sin6_port = htons(port_),
+        .sin6_addr = IN6ADDR_LOOPBACK_INIT,
+    };
+    PCHECK(bind(sock, (sockaddr*)&bind_addr, sizeof(bind_addr)) == 0);
+  }
+
+  PCHECK(listen(sock, 128) == 0);
+  return sock;
+}
+
+}  // namespace firecgi

server.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <unordered_set>
+
+#include "request.h"
+
+namespace firecgi {
+
+class Server {
+ public:
+  Server(int port, const std::function<void(Request*)>& callback,
+         int threads = 1, int max_request_len = (16 * 1024));
+  ~Server();
+
+  void Serve();
+  void Shutdown();
+  void RegisterSignalHandlers();
+
+ private:
+  Connection* NewConn(int listen_sock, int epoll_fd);
+  int NewListenSock();
+  void ServeInt();
+
+  const int port_;
+  const std::function<void(Request*)> callback_;
+  const int threads_;
+  const int max_request_len_;
+
+  int close_fd_;
+};
+
+}  // namespace firecgi