Typos

.clang-format

@@ -0,0 +1,151 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories: 
+  - Regex:           '^<ext/.*\.h>'
+    Priority:        2
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+RawStringFormats: 
+  - Language:        Cpp
+    Delimiters:      
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+  - Language:        TextProto
+    Delimiters:      
+      - pb
+      - PB
+      - proto
+      - PROTO
+    EnclosingFunctions: 
+      - EqualsProto
+      - EquivToProto
+      - PARSE_PARTIAL_TEXT_PROTO
+      - PARSE_TEST_PROTO
+      - PARSE_TEXT_PROTO
+      - ParseTextOrDie
+      - ParseTextProtoOrDie
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+StatementMacros: 
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseTab:          Never
+...
+

.gitignore

@@ -1,5 +1,6 @@
 *.o
 *.a
+*.so
 example_simple
 connection_afl
 findings

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "firebuf"]
 	path = firebuf
 	url = ../firebuf.git
+[submodule "fireusage"]
+	path = fireusage
+	url = ../fireusage.git

Makefile

@@ -1,37 +1,46 @@
 FIRE_CXX ?= clang++
-FIRE_CXXFLAGS ?= -O3 -std=gnu++2a -Wall -Werror
+FIRE_CXXFLAGS ?= -O3 -std=gnu++2a -Wall -Werror -Wextra -fPIE -fPIC -fstack-protector-strong -fsanitize=safe-stack -fsanitize=safe-stack
+FIRE_LDFLAGS ?= -fuse-ld=gold -flto -Wl,-z,relro -Wl,-z,now
 FIRE_LDLIBS ?= -lgflags -lglog -lpthread
 
-all: firecgi.a example_simple
+all: firecgi.a firecgi.o firecgi.so example_simple
 
-objects = firecgi.o connection.o request.o parse.o
+objects = server.o connection.o request.o parse.o
 
-_firebuf:
+firebuf/firebuf.o:
-	$(MAKE) --directory=firebuf
+	$(MAKE) --directory=firebuf firebuf.o
 
-firecgi.a: $(objects) _firebuf
+fireusage/fireusage.o:
-	ar x firebuf/firebuf.a
+	$(MAKE) --directory=fireusage fireusage.o
-	ar rcs $@ $(objects) $(shell ar t firebuf/firebuf.a)
 
-example_simple: example_simple.o firecgi.a
+firecgi.a: $(objects)
-	$(FIRE_CXX) $(FIRE_CXXFLAGS) -o $@ $+ $(FIRE_LDLIBS)
+	ar rcs $@ $^
+
+firecgi.o: $(objects) firebuf/firebuf.o fireusage/fireusage.o
+	gold -z relro -z now -r --output=$@ $+
+
+firecgi.so: $(objects) firebuf/firebuf.o fireusage/fireusage.o
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -shared -o $@ $+ $(FIRE_LDLIBS)
+
+example_simple: example_simple.o firecgi.o
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -pie -o $@ $+ $(FIRE_LDLIBS)
 
 %.o: %.cc *.h Makefile
 	$(FIRE_CXX) $(FIRE_CXXFLAGS) -c -o $@ $<
 
 clean:
 	$(MAKE) --directory=firebuf clean
-	rm --force example_simple connection_afl *.o *.a
+	$(MAKE) --directory=fireusage clean
+	rm --force example_simple connection_afl *.so *.o *.a
 
 afl:
 	$(MAKE) clean
-	FIRE_CXX=afl-g++ $(MAKE) afl_int
+	FIRE_CXX=afl-g++ FIRE_CXXFLAGS="-O3 -std=gnu++2a -fPIC -fPIE" $(MAKE) afl_int
 
 afl_int: connection_afl
 
-connection_afl: connection_afl.o firecgi.a
+connection_afl: connection_afl.o firecgi.o
-	$(MAKE) --directory=firebuf
+	$(FIRE_CXX) $(FIRE_CXXFLAGS) $(FIRE_LDFLAGS) -pie -o $@ $+ $(FIRE_LDLIBS)
-	$(FIRE_CXX) $(FIRE_CXXFLAGS) -o $@ $+ $(FIRE_LDLIBS)
 
 test: test_connection
 
@@ -42,4 +51,12 @@ test_connection: connection_afl
 
 asan:
 	$(MAKE) clean
-	FIRE_CXXFLAGS="-O1 -g -fsanitize=address -fno-omit-frame-pointer -std=gnu++2a -Wall -Werror" $(MAKE) all
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=address -fno-omit-frame-pointer -std=gnu++2a -fPIE -fPIC" $(MAKE) all
+
+tsan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=thread -std=gnu++2a -fPIE -fPIC" $(MAKE) all
+
+ubsan:
+	$(MAKE) clean
+	FIRE_CXXFLAGS="-O1 -g -fsanitize=undefined -std=gnu++2a -fPIE -fPIC" $(MAKE) all

connection.cc

@@ -9,140 +9,145 @@
 
 namespace firecgi {
 
-Connection::Connection(int sock, const sockaddr_in6& client_addr, const std::function<void(Request*)>& callback, const std::unordered_set<std::string_view>& headers)
+Connection::Connection(int sock, const sockaddr_in6 &client_addr,
-		: sock_(sock),
+                       const std::function<void(Request *)> &callback,
-		  callback_(callback),
+                       int max_request_len)
-		  headers_(headers),
+    : sock_(sock),
-		  buf_(sock, max_record_len),
+      callback_(callback),
-		  request_(this) {
+      buf_(sock, max_request_len),
-	char client_addr_str[INET6_ADDRSTRLEN];
+      request_(this) {
-	PCHECK(inet_ntop(AF_INET6, &client_addr.sin6_addr, client_addr_str, sizeof(client_addr_str)));
+  char client_addr_str[INET6_ADDRSTRLEN];
+  PCHECK(inet_ntop(AF_INET6, &client_addr.sin6_addr, client_addr_str,
+                   sizeof(client_addr_str)));
 
-	LOG(INFO) << "new connection: [" << client_addr_str << "]:" << ntohs(client_addr.sin6_port);
+  LOG(INFO) << "new connection: [" << client_addr_str
+            << "]:" << ntohs(client_addr.sin6_port);
 }
 
 Connection::~Connection() {
-	PCHECK(close(sock_) == 0);
+  PCHECK(close(sock_) == 0);
-	LOG(INFO) << "connection closed (handled " << requests_ << " requests)";
+  LOG(INFO) << "connection closed (handled " << requests_ << " requests)";
 }
 
-bool Connection::Write(const std::vector<iovec>& vecs) {
+bool Connection::Write(const std::vector<iovec> &vecs) {
-	ssize_t total_size = 0;
+  ssize_t total_size = 0;
-	for (const auto& vec : vecs) {
+  for (const auto &vec : vecs) {
-		total_size += vec.iov_len;
+    total_size += vec.iov_len;
-	}
+  }
-	return writev(sock_, vecs.data(), vecs.size()) == total_size;
+  return writev(sock_, vecs.data(), vecs.size()) == total_size;
 }
 
 int Connection::Read() {
-	if (!buf_.Refill()) {
+  if (!buf_.Refill()) {
-		return sock_;
+    return sock_;
-	}
+  }
 
-	while (true) {
+  while (true) {
-		buf_.ResetRead();
+    buf_.ResetRead();
 
-		const auto *header = buf_.ReadObj<Header>();
+    const auto *header = buf_.ReadObj<Header>();
-		if (!header) {
+    if (!header) {
-			break;
+      break;
-		}
+    }
 
-		if (header->version != 1) {
+    if (header->version != 1) {
-			LOG(ERROR) << "invalid FastCGI protocol version: " << header->version;
+      LOG(ERROR) << "invalid FastCGI protocol version: " << header->version;
-			return sock_;
+      return sock_;
-		}
+    }
 
-		if (buf_.ReadMaxLen() < header->ContentLength()) {
+    if (buf_.ReadMaxLen() < header->ContentLength()) {
-			break;
+      break;
-		}
+    }
 
-		switch (header->type) {
+    switch (header->type) {
-		  case 1:
+      case 1: {
-		  	{
+        if (header->ContentLength() != sizeof(BeginRequest)) {
-				if (header->ContentLength() != sizeof(BeginRequest)) {
+          LOG(ERROR) << "FCGI_BeginRequestBody is the wrong length: "
-					LOG(ERROR) << "FCGI_BeginRequestBody is the wrong length: " << header->ContentLength();
+                     << header->ContentLength();
-					return sock_;
+          return sock_;
-				}
+        }
 
-				const auto *begin_request = CHECK_NOTNULL(buf_.ReadObj<BeginRequest>());
+        const auto *begin_request = CHECK_NOTNULL(buf_.ReadObj<BeginRequest>());
 
-				if (begin_request->Role() != 1) {
+        if (begin_request->Role() != 1) {
-					LOG(ERROR) << "unsupported FastCGI role: " << begin_request->Role();
+          LOG(ERROR) << "unsupported FastCGI role: " << begin_request->Role();
-					return sock_;
+          return sock_;
-				}
+        }
 
-				request_.NewRequest(header->RequestId());
+        request_.NewRequest(header->RequestId());
-			}
+      } break;
-			break;
 
-		  case 4:
+      case 4: {
-		    {
+        if (header->RequestId() != request_.RequestId()) {
-				if (header->RequestId() != request_.RequestId()) {
+          LOG(ERROR) << "out of order FCGI_PARAMS record, or client is "
-					LOG(ERROR) << "out of order FCGI_PARAMS record, or client is multiplexing requests (which we don't support)";
+                        "multiplexing requests (which we don't support)";
-					return sock_;
+          return sock_;
-				}
+        }
 
-				firebuf::ConstBuffer param_buf(buf_.Read(header->ContentLength()), header->ContentLength());
+        firebuf::ConstBuffer param_buf(buf_.Read(header->ContentLength()),
-				while (param_buf.ReadMaxLen() > 0) {
+                                       header->ContentLength());
-					const auto *param_header = param_buf.ReadObj<ParamHeader>();
+        while (param_buf.ReadMaxLen() > 0) {
-					if (!param_header) {
+          const auto *param_header = param_buf.ReadObj<ParamHeader>();
-						LOG(ERROR) << "FCGI_PARAMS missing header";
+          if (!param_header) {
-						return sock_;
+            LOG(ERROR) << "FCGI_PARAMS missing header";
-					}
+            return sock_;
+          }
 
-					const auto *key_buf = param_buf.Read(param_header->key_length);
+          const auto *key_buf = param_buf.Read(param_header->key_length);
-					if (!key_buf) {
+          if (!key_buf) {
-						LOG(ERROR) << "FCGI_PARAMS missing key";
+            LOG(ERROR) << "FCGI_PARAMS missing key";
-						return sock_;
+            return sock_;
-					}
+          }
-					std::string_view key(key_buf, param_header->key_length);
+          std::string_view key(key_buf, param_header->key_length);
 
-					const auto *value_buf = param_buf.Read(param_header->value_length);
+          const auto *value_buf = param_buf.Read(param_header->value_length);
-					if (!value_buf) {
+          if (!value_buf) {
-						LOG(ERROR) << "FCGI_PARAMS missing value";
+            LOG(ERROR) << "FCGI_PARAMS missing value";
-						return sock_;
+            return sock_;
-					}
+          }
-					std::string_view value(value_buf, param_header->value_length);
+          std::string_view value(value_buf, param_header->value_length);
 
-					if (headers_.find(key) != headers_.end()) {
+          request_.AddParam(key, value);
-						request_.AddParam(key, value);
+        }
-					}
+      } break;
-				}
-			}
-			break;
 
-		  case 5:
+      case 5: {
-		  	{
+        if (header->RequestId() != request_.RequestId()) {
-				if (header->RequestId() != request_.RequestId()) {
+          LOG(ERROR) << "out of order FCGI_STDIN record, or client is "
-					LOG(ERROR) << "out of order FCGI_STDIN record, or client is multiplexing requests (which we don't support)";
+                        "multiplexing requests (which we don't support)";
-					return sock_;
+          return sock_;
-				}
+        }
 
-				if (header->ContentLength() == 0) {
+        if (header->ContentLength() == 0) {
-					// Magic signal for completed request (mirrors the HTTP/1.1 protocol)
+          // Magic signal for completed request (mirrors the HTTP/1.1 protocol)
-					requests_++;
+          requests_++;
-					callback_(&request_);
+          callback_(&request_);
-				} else {
+          buf_.Consume();  // discard data and invalidate pointers
-					std::string_view in(buf_.Read(header->ContentLength()), header->ContentLength());
+        } else {
-					request_.AddIn(in);
+          if (!request_.GetBody().empty()) {
-				}
+            LOG(ERROR) << "received multiple stdin records. have you set "
-			}
+                          "\"fastcgi_request_buffering on\"?";
-			break;
+          }
+          request_.SetBody(
+              {buf_.Read(header->ContentLength()), header->ContentLength()});
+        }
+      } break;
 
-		  default:
+      default:
-			LOG(ERROR) << "unknown record type: " << header->type;
+        LOG(ERROR) << "unknown record type: " << header->type;
-			return sock_;
+        return sock_;
-		}
+    }
 
-		if (!buf_.Discard(header->padding_length)) {
+    if (!buf_.Discard(header->padding_length)) {
-			break;
+      break;
-		}
+    }
 
-		buf_.Commit(); // we've acted on the bytes read so far
+    buf_.Commit();  // we've acted on the bytes read so far
-	}
+  }
 
-	buf_.Consume();
+  return -1;
-	return -1;
 }
 
-} // namespace firecgi
+uint64_t Connection::Requests() const { return requests_; }
+
+}  // namespace firecgi

connection.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <functional>
 #include <sys/uio.h>
+#include <functional>
 #include <unordered_map>
 #include <unordered_set>
 
@@ -12,22 +12,24 @@
 namespace firecgi {
 
 class Connection {
-  public:
+ public:
-	Connection(int sock, const sockaddr_in6& client_addr, const std::function<void(Request*)>& callback, const std::unordered_set<std::string_view>& headers);
+  Connection(int sock, const sockaddr_in6& client_addr,
-	~Connection();
+             const std::function<void(Request*)>& callback,
+             int max_request_len);
+  ~Connection();
 
-	[[nodiscard]] int Read();
+  [[nodiscard]] int Read();
-	[[nodiscard]] bool Write(const std::vector<iovec>& vecs);
+  [[nodiscard]] bool Write(const std::vector<iovec>& vecs);
 
-  private:
+  [[nodiscard]] uint64_t Requests() const;
-  	const int sock_;
-	const std::function<void(Request*)>& callback_;
-	const std::unordered_set<std::string_view>& headers_;
-	firebuf::StreamBuffer buf_;
-	Request request_;
 
-	uint64_t requests_ = 0;
+ private:
+  const int sock_;
+  const std::function<void(Request*)>& callback_;
+  firebuf::StreamBuffer buf_;
+  Request request_;
 
+  uint64_t requests_ = 0;
 };
 
-} // namespace firecgi
+}  // namespace firecgi

connection_afl.cc

@@ -1,18 +1,20 @@
 #include "connection.h"
 
 int main(int argc, char* argv[]) {
-	FLAGS_logtostderr = 1;
+  FLAGS_logtostderr = 1;
-	FLAGS_minloglevel = 3;
+  FLAGS_minloglevel = 3;
-	google::InitGoogleLogging(argv[0]);
+  google::InitGoogleLogging(argv[0]);
-	gflags::ParseCommandLineFlags(&argc, &argv, true);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
 
-	{
+  {
-		firecgi::Connection conn(STDIN_FILENO, {}, [](std::unique_ptr<firecgi::Request> req) { req->End(); }, {});
+    firecgi::Connection conn(
-		static_cast<void>(conn.Read());
+        STDIN_FILENO, {}, [](firecgi::Request* req) { req->End(); }, {},
-	}
+        16 * 1024);
+    static_cast<void>(conn.Read());
+  }
 
-	gflags::ShutDownCommandLineFlags();
+  gflags::ShutDownCommandLineFlags();
-	google::ShutdownGoogleLogging();
+  google::ShutdownGoogleLogging();
 
-	return 0;
+  return 0;
 }

example_simple.cc

@@ -1,19 +1,26 @@
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
-#include "firecgi.h"
+#include "server.h"
 
 DEFINE_int32(port, 9000, "TCP port to bind");
 DEFINE_int32(threads, 1, "Number of server threads");
 
-int main(int argc, char *argv[]) {
+int main(int argc, char* argv[]) {
-	google::InitGoogleLogging(argv[0]);
+  google::InitGoogleLogging(argv[0]);
-	gflags::ParseCommandLineFlags(&argc, &argv, true);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
 
-	firecgi::Server server(FLAGS_port, [](firecgi::Request* request) {
+  firecgi::Server server(
-		request->WriteHeader("Content-Type", "text/plain");
+      FLAGS_port,
-		request->WriteBody("Hello world");
+      [](firecgi::Request* request) {
-		request->End();
+        request->WriteHeader("Content-Type", "text/plain");
-	}, FLAGS_threads);
+        request->WriteBody("Hello world");
-	server.Serve();
+        request->End();
+      },
+      FLAGS_threads);
+  server.RegisterSignalHandlers();
+  server.Serve();
+
+  gflags::ShutDownCommandLineFlags();
+  google::ShutdownGoogleLogging();
 }

firecgi.cc

@@ -1,117 +0,0 @@
-#include <arpa/inet.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <signal.h>
-#include <sys/epoll.h>
-#include <sys/socket.h>
-#include <thread>
-
-#include "firecgi.h"
-#include "connection.h"
-
-namespace firecgi {
-
-Server::Server(int port, const std::function<void(Request*)>& callback, int threads, const std::unordered_set<std::string_view>& headers)
-		: port_(port),
-		  callback_(callback),
-		  threads_(threads),
-		  headers_(headers) {
-	LOG(INFO) << "listening on [::1]:" << port_;
-
-	signal(SIGPIPE, SIG_IGN);
-}
-
-void Server::Serve() {
-	std::vector<std::thread> threads;
-	for (int i = 0; i < threads_ - 1; ++i) {
-		threads.emplace_back([this]() { ServeInt(); });
-	}
-	ServeInt();
-}
-
-void Server::ServeInt() {
-	auto epoll_fd = epoll_create1(0);
-	PCHECK(epoll_fd >= 0) << "epoll_create()";
-
-	auto listen_sock = NewListenSock();
-
-	{
-		struct epoll_event ev{
-			.events = EPOLLIN,
-			.data = {
-				.ptr = nullptr,
-			},
-		};
-		PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, listen_sock, &ev) == 0);
-	}
-
-	while (true) {
-		constexpr auto max_events = 256;
-		struct epoll_event events[max_events];
-		auto num_fd = epoll_wait(epoll_fd, events, max_events, -1);
-		if (num_fd == -1 && errno == EINTR) {
-			continue;
-		}
-		PCHECK(num_fd > 0) << "epoll_wait()";
-
-		for (auto i = 0; i < num_fd; ++i) {
-			if (events[i].data.ptr == nullptr) {
-				NewConn(listen_sock, epoll_fd);
-			} else {
-				auto conn = static_cast<Connection*>(events[i].data.ptr);
-				auto fd = conn->Read();
-				if (fd != -1) {
-					PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, nullptr) == 0);
-					delete conn;
-				}
-			}
-		}
-	}
-}
-
-void Server::NewConn(int listen_sock, int epoll_fd) {
-	sockaddr_in6 client_addr;
-	socklen_t client_addr_len = sizeof(client_addr);
-
-	auto client_sock = accept(listen_sock, (sockaddr*) &client_addr, &client_addr_len);
-	PCHECK(client_sock >= 0) << "accept()";
-	CHECK_EQ(client_addr.sin6_family, AF_INET6);
-
-	int flags = 1;
-	PCHECK(setsockopt(client_sock, SOL_TCP, TCP_NODELAY, &flags, sizeof(flags)) == 0);
-
-	{
-		auto *conn = new Connection(client_sock, client_addr, callback_, headers_);
-		struct epoll_event ev{
-			.events = EPOLLIN,
-			.data = {
-				.ptr = conn,
-			},
-		};
-		PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_sock, &ev) == 0);
-	}
-}
-
-int Server::NewListenSock() {
-	auto sock = socket(AF_INET6, SOCK_STREAM, 0);
-	PCHECK(sock >= 0) << "socket()";
-
-	{
-		int optval = 1;
-		PCHECK(setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &optval, sizeof(optval)) == 0);
-	}
-
-	{
-		sockaddr_in6 bind_addr = {
-			.sin6_family = AF_INET6,
-			.sin6_port = htons(port_),
-			.sin6_addr = IN6ADDR_LOOPBACK_INIT,
-		};
-		PCHECK(bind(sock, (sockaddr*) &bind_addr, sizeof(bind_addr)) == 0);
-	}
-
-	PCHECK(listen(sock, 128) == 0);
-	return sock;
-}
-
-} // namespace firecgi

firecgi.h

@@ -1,27 +0,0 @@
-#pragma once
-
-#include <functional>
-#include <memory>
-#include <unordered_set>
-
-#include "request.h"
-
-namespace firecgi {
-
-class Server {
-  public:
-	Server(int port, const std::function<void(Request*)>& callback, int threads=1, const std::unordered_set<std::string_view>& headers={});
-	void Serve();
-
-  private:
-	void NewConn(int listen_sock, int epoll_fd);
-	int NewListenSock();
-	void ServeInt();
-
-	const int port_;
-	const std::function<void(Request*)> callback_;
-	const int threads_;
-	const std::unordered_set<std::string_view> headers_;
-};
-
-} // firecgi

parse.cc

@@ -3,9 +3,9 @@
 namespace firecgi {
 
 Header::Header(uint8_t type_in, uint16_t request_id, uint16_t content_length)
-		: type(type_in) {
+    : type(type_in) {
-	SetRequestId(request_id);
+  SetRequestId(request_id);
-	SetContentLength(content_length);
+  SetContentLength(content_length);
 }
 
-} // namespace firecgi
+}  // namespace firecgi

parse.h

@@ -5,47 +5,46 @@
 namespace firecgi {
 
 struct Header {
-	Header(uint8_t type_in, uint16_t request_id, uint16_t content_length);
+  Header(uint8_t type_in, uint16_t request_id, uint16_t content_length);
 
-	uint8_t version = 1;
+  uint8_t version = 1;
-	uint8_t type;
+  uint8_t type;
-  private:
-	uint16_t request_id_; // network byte order
-	uint16_t content_length_; // network byte order
-  public:
-	uint8_t padding_length = 0;
-	uint8_t reserved = 0;
 
-	uint16_t RequestId() const { return ntohs(request_id_); }
+ private:
-	uint16_t ContentLength() const { return ntohs(content_length_); }
+  uint16_t request_id_;      // network byte order
+  uint16_t content_length_;  // network byte order
+ public:
+  uint8_t padding_length = 0;
+  uint8_t reserved = 0;
 
-	void SetRequestId(uint16_t request_id) { request_id_ = htons(request_id); }
+  uint16_t RequestId() const { return ntohs(request_id_); }
-	void SetContentLength(uint16_t content_length) { content_length_ = htons(content_length); }
+  uint16_t ContentLength() const { return ntohs(content_length_); }
+
+  void SetRequestId(uint16_t request_id) { request_id_ = htons(request_id); }
+  void SetContentLength(uint16_t content_length) {
+    content_length_ = htons(content_length);
+  }
 };
 
 struct BeginRequest {
-  private:
+ private:
-	uint16_t role_; // network byte order
+  uint16_t role_;  // network byte order
-  public:
+ public:
-	uint8_t flags;
+  uint8_t flags;
-	uint8_t reserved[5];
+  uint8_t reserved[5];
 
-	uint16_t Role() const { return ntohs(role_); }
+  uint16_t Role() const { return ntohs(role_); }
 };
 
 struct EndRequest {
-	uint32_t app_status = htonl(0); // network byte order
+  uint32_t app_status = htonl(0);  // network byte order
-	uint8_t protocol_status;
+  uint8_t protocol_status;
-	uint8_t reserved[3] = {};
+  uint8_t reserved[3] = {};
 };
 
 struct ParamHeader {
-	uint8_t key_length;
+  uint8_t key_length;
-	uint8_t value_length;
+  uint8_t value_length;
 };
 
-constexpr auto max_content_len = 65535;
+}  // namespace firecgi
-constexpr auto max_padding_len = 255;
-constexpr auto max_record_len = sizeof(Header) + max_content_len + max_padding_len;
-
-} // namespace firecgi

request.cc

@@ -7,111 +7,132 @@
 namespace firecgi {
 namespace {
 
-template<class T> void AppendVec(const T& obj, std::vector<iovec>* vec) {
+template <class T>
-	vec->push_back(iovec{
+void AppendVec(const T& obj, std::vector<iovec>* vec) {
-		.iov_base = (void*)(&obj),
+  vec->push_back(iovec{
-		.iov_len = sizeof(obj),
+      .iov_base = (void*)(&obj),
-	});
+      .iov_len = sizeof(obj),
+  });
 }
 
-} // namespace
+}  // namespace
 
-Request::Request(Connection* conn)
+Request::Request(Connection* conn) : conn_(conn), out_buf_(64 * 1024) {}
-		: conn_(conn),
+
-		  out_buf_(max_record_len) {}
+Request::~Request() {
+  if (on_close_) {
+    on_close_();
+  }
+}
 
 void Request::NewRequest(uint16_t request_id) {
-	request_id_ = request_id;
+  if (on_close_) {
-	params_.clear();
+    on_close_();
-	in_.clear();
+  }
-	out_buf_.Reset();
+
-	body_written_ = false;
+  request_id_ = request_id;
+  params_.clear();
+  body_ = {};
+  on_close_ = nullptr;
+  out_buf_.Reset();
+  body_written_ = false;
 }
 
-uint16_t Request::RequestId() {
+uint16_t Request::RequestId() const { return request_id_; }
-	return request_id_;
+
+void Request::AddParam(const std::string_view& key,
+                       const std::string_view& value) {
+  params_.try_emplace(key, value);
 }
 
-void Request::AddParam(const std::string_view& key, const std::string_view& value) {
+void Request::SetBody(const std::string_view& body) { body_ = body; }
-	params_.try_emplace(std::string(key), std::string(value));
+
+const std::string_view& Request::GetParam(const std::string_view& key) const {
+  auto iter = params_.find(key);
+  if (iter == params_.end()) {
+    static const std::string_view none;
+    return none;
+  }
+  return iter->second;
 }
 
-void Request::AddIn(const std::string_view& in) {
+const std::string_view& Request::GetBody() const { return body_; }
-	in_.append(in);
+
+void Request::OnClose(const std::function<void()>& on_close) {
+  on_close_ = on_close;
 }
 
-const std::string& Request::GetParam(const std::string& key) {
+void Request::WriteHeader(const std::string_view& name,
-	auto iter = params_.find(key);
+                          const std::string_view& value) {
-	if (iter == params_.end()) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
-		static const std::string none;
-		return none;
-	}
-	return iter->second;
-}
 
-void Request::WriteHeader(const std::string_view& name, const std::string_view& value) {
+  CHECK(!body_written_);
-	CHECK(!body_written_);
+  CHECK(out_buf_.Write(name));
-	CHECK(out_buf_.Write(name));
+  CHECK(out_buf_.Write(": "));
-	CHECK(out_buf_.Write(": "));
+  CHECK(out_buf_.Write(value));
-	CHECK(out_buf_.Write(value));
+  CHECK(out_buf_.Write("\n"));
-	CHECK(out_buf_.Write("\n"));
 }
 
 void Request::WriteBody(const std::string_view& body) {
-	if (!body_written_) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
-		CHECK(out_buf_.Write("\n"));
+  if (!body_written_) {
-		body_written_ = true;
+    CHECK(out_buf_.Write("\n"));
-	}
+    body_written_ = true;
-	// TODO: make this able to span multiple packets
+  }
-	CHECK(out_buf_.Write(body));
+  // TODO: make this able to span multiple packets
+  CHECK(out_buf_.Write(body));
 }
 
 bool Request::Flush() {
-	std::vector<iovec> vecs;
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
 
-	auto header = OutputHeader();
+  std::vector<iovec> vecs;
-	AppendVec(header, &vecs);
 
-	vecs.push_back(OutputVec());
+  auto header = OutputHeader();
+  AppendVec(header, &vecs);
 
-	if (!conn_->Write(vecs)) {
+  vecs.push_back(OutputVec());
-		return false;
+
-	}
+  if (!conn_->Write(vecs)) {
-	out_buf_.Commit();
+    return false;
-	return true;
+  }
+  out_buf_.Commit();
+  out_buf_.Consume();
+  return true;
 }
 
 bool Request::End() {
-	// Fully empty response not allowed
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
-	WriteBody("");
 
-	std::vector<iovec> vecs;
+  WriteBody("");
 
-	// Must be outside if block, so it lives through Write() below
+  std::vector<iovec> vecs;
-	auto output_header = OutputHeader();
-	if (output_header.ContentLength()) {
-		AppendVec(output_header, &vecs);
-		vecs.push_back(OutputVec());
-	}
 
-	EndRequest end;
+  // Must be outside if block, so it lives through Write() below
-	Header end_header(3, request_id_, sizeof(end));
+  auto output_header = OutputHeader();
-	AppendVec(end_header, &vecs);
+  if (output_header.ContentLength()) {
-	AppendVec(end, &vecs);
+    AppendVec(output_header, &vecs);
+    vecs.push_back(OutputVec());
+  }
 
-	return conn_->Write(vecs);
+  EndRequest end;
+  Header end_header(3, request_id_, sizeof(end));
+  AppendVec(end_header, &vecs);
+  AppendVec(end, &vecs);
+
+  return conn_->Write(vecs);
 }
 
 iovec Request::OutputVec() {
-	const auto output_len = out_buf_.ReadMaxLen();
+  const auto output_len = out_buf_.ReadMaxLen();
-	return iovec{
+  return iovec{
-		.iov_base = (void *)(CHECK_NOTNULL(out_buf_.Read(output_len))),
+      .iov_base = (void*)(CHECK_NOTNULL(out_buf_.Read(output_len))),
-		.iov_len = output_len,
+      .iov_len = output_len,
-	};
+  };
 }
 
 Header Request::OutputHeader() {
-	return Header(6, request_id_, out_buf_.ReadMaxLen());
+  return Header(6, request_id_, out_buf_.ReadMaxLen());
 }
 
-} // namespace firecgi
+}  // namespace firecgi

request.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <functional>
+#include <mutex>
 #include <unordered_map>
 
 #include "firebuf/buffer.h"
@@ -11,35 +13,61 @@ namespace firecgi {
 class Connection;
 
 class Request {
-  public:
+ public:
-	Request(Connection *conn);
+  Request(Connection* conn);
+  ~Request();
 
-	void NewRequest(uint16_t request_id);
+  void NewRequest(uint16_t request_id);
 
-	uint16_t RequestId();
+  uint16_t RequestId() const;
 
-	void AddParam(const std::string_view& key, const std::string_view& value);
+  void AddParam(const std::string_view& key, const std::string_view& value);
-	void AddIn(const std::string_view& in);
+  void SetBody(const std::string_view& in);
 
-	const std::string& GetParam(const std::string& key);
+  const std::string_view& GetParam(const std::string_view& key) const;
+  const std::string_view& GetBody() const;
 
-	void WriteHeader(const std::string_view& name, const std::string_view& value);
+  void OnClose(const std::function<void()>& callback);
-	void WriteBody(const std::string_view& body);
-	[[nodiscard]] bool Flush();
-	bool End();
 
-  private:
+  void WriteHeader(const std::string_view& name, const std::string_view& value);
-	Header OutputHeader();
+  void WriteBody(const std::string_view& body);
-	iovec OutputVec();
+  [[nodiscard]] bool Flush();
+  bool End();
 
-	Connection *conn_;
+  template <typename... Args>
-	uint16_t request_id_ = 0;
+  void WriteBody(const std::string_view& first, Args... more);
 
-	std::unordered_map<std::string, std::string> params_;
+  template <typename T>
-	std::string in_;
+  T InTransaction(const std::function<T()>& callback);
 
-	firebuf::Buffer out_buf_;
+ private:
-	bool body_written_;
+  Header OutputHeader();
+  iovec OutputVec();
+
+  Connection* conn_;
+  uint16_t request_id_ = 0;
+
+  std::unordered_map<std::string_view, std::string_view> params_;
+  std::string_view body_;
+
+  std::function<void()> on_close_;
+
+  firebuf::Buffer out_buf_;
+  bool body_written_;
+  std::recursive_mutex output_mu_;
 };
 
-} // namespace firecgi
+template <typename... Args>
+void Request::WriteBody(const std::string_view& first, Args... more) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+  WriteBody(first);
+  WriteBody(more...);
+}
+
+template <typename T>
+T Request::InTransaction(const std::function<T()>& callback) {
+  std::lock_guard<std::recursive_mutex> l(output_mu_);
+  return callback();
+}
+
+}  // namespace firecgi

server.cc

@@ -0,0 +1,185 @@
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <signal.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <iomanip>
+#include <thread>
+
+#include "fireusage/usage.h"
+
+#include "connection.h"
+#include "server.h"
+
+namespace firecgi {
+
+Server::Server(int port, const std::function<void(Request*)>& callback,
+               int threads, int max_request_len)
+    : port_(port),
+      callback_(callback),
+      threads_(threads),
+      max_request_len_(max_request_len),
+      close_fd_(eventfd(0, 0)) {
+  CHECK_GE(close_fd_, 0);
+
+  LOG(INFO) << "listening on [::1]:" << port_;
+  signal(SIGPIPE, SIG_IGN);
+}
+
+Server::~Server() { PCHECK(close(close_fd_) == 0); }
+
+void Server::Serve() {
+  std::vector<std::thread> threads;
+  for (int i = 0; i < threads_ - 1; ++i) {
+    threads.emplace_back([this]() { ServeInt(); });
+  }
+  ServeInt();
+  for (auto& thread : threads) {
+    thread.join();
+  }
+  LOG(INFO) << "all threads shut down";
+}
+
+void Server::Shutdown() {
+  uint64_t shutdown = 1;
+  PCHECK(write(close_fd_, &shutdown, sizeof(shutdown)) == sizeof(shutdown));
+}
+
+namespace {
+Server* shutdown_server = nullptr;
+}  // namespace
+
+void Server::RegisterSignalHandlers() {
+  shutdown_server = this;
+  for (auto sig : {SIGINT, SIGTERM}) {
+    signal(sig, [](int signum) {
+      LOG(INFO) << "received " << strsignal(signum);
+      shutdown_server->Shutdown();
+    });
+  }
+}
+
+void Server::ServeInt() {
+  auto epoll_fd = epoll_create1(0);
+  PCHECK(epoll_fd >= 0) << "epoll_create()";
+
+  auto listen_sock = NewListenSock();
+
+  char new_conn;
+  {
+    struct epoll_event ev {
+      .events = EPOLLIN,
+      .data = {
+          .ptr = &new_conn,
+      },
+    };
+    PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, listen_sock, &ev) == 0);
+  }
+
+  char shutdown;
+  {
+    struct epoll_event ev {
+      .events = EPOLLIN,
+      .data = {
+          .ptr = &shutdown,
+      },
+    };
+    PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, close_fd_, &ev) == 0);
+  }
+
+  std::unordered_set<Connection*> connections;
+
+  fireusage::UsageTracker usage_tracker;
+  usage_tracker.Start();
+
+  while (true) {
+    constexpr auto max_events = 256;
+    struct epoll_event events[max_events];
+    auto num_fd = epoll_wait(epoll_fd, events, max_events, -1);
+    if (num_fd == -1 && errno == EINTR) {
+      continue;
+    }
+    PCHECK(num_fd > 0) << "epoll_wait()";
+
+    for (auto i = 0; i < num_fd; ++i) {
+      if (events[i].data.ptr == &new_conn) {
+        connections.insert(CHECK_NOTNULL(NewConn(listen_sock, epoll_fd)));
+      } else if (events[i].data.ptr == &shutdown) {
+        for (auto& conn : connections) {
+          usage_tracker.AddEvents(conn->Requests());
+          delete conn;
+        }
+        usage_tracker.Stop();
+        PCHECK(close(listen_sock) == 0);
+        PCHECK(close(epoll_fd) == 0);
+        usage_tracker.Log("requests");
+        return;
+      } else {
+        auto conn = static_cast<Connection*>(events[i].data.ptr);
+        auto fd = conn->Read();
+        if (fd != -1) {
+          PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, nullptr) == 0);
+          usage_tracker.AddEvents(conn->Requests());
+          connections.erase(conn);
+          delete conn;
+        }
+      }
+    }
+  }
+}
+
+Connection* Server::NewConn(int listen_sock, int epoll_fd) {
+  sockaddr_in6 client_addr;
+  socklen_t client_addr_len = sizeof(client_addr);
+
+  auto client_sock =
+      accept(listen_sock, (sockaddr*)&client_addr, &client_addr_len);
+  PCHECK(client_sock >= 0) << "accept()";
+  CHECK_EQ(client_addr.sin6_family, AF_INET6);
+
+  int flags = 1;
+  PCHECK(setsockopt(client_sock, SOL_TCP, TCP_NODELAY, &flags, sizeof(flags)) ==
+         0);
+
+  auto* conn =
+      new Connection(client_sock, client_addr, callback_, max_request_len_);
+  {
+    struct epoll_event ev {
+      .events = EPOLLIN,
+      .data = {
+          .ptr = conn,
+      },
+    };
+    PCHECK(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_sock, &ev) == 0);
+  }
+
+  return conn;
+}
+
+int Server::NewListenSock() {
+  auto sock = socket(AF_INET6, SOCK_STREAM, 0);
+  PCHECK(sock >= 0) << "socket()";
+
+  {
+    int optval = 1;
+    PCHECK(setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &optval,
+                      sizeof(optval)) == 0);
+  }
+
+  {
+    sockaddr_in6 bind_addr = {
+        .sin6_family = AF_INET6,
+        .sin6_port = htons(port_),
+        .sin6_addr = IN6ADDR_LOOPBACK_INIT,
+    };
+    PCHECK(bind(sock, (sockaddr*)&bind_addr, sizeof(bind_addr)) == 0);
+  }
+
+  PCHECK(listen(sock, 128) == 0);
+  return sock;
+}
+
+}  // namespace firecgi

server.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <unordered_set>
+
+#include "request.h"
+
+namespace firecgi {
+
+class Server {
+ public:
+  Server(int port, const std::function<void(Request*)>& callback,
+         int threads = 1, int max_request_len = (16 * 1024));
+  ~Server();
+
+  void Serve();
+  void Shutdown();
+  void RegisterSignalHandlers();
+
+ private:
+  Connection* NewConn(int listen_sock, int epoll_fd);
+  int NewListenSock();
+  void ServeInt();
+
+  const int port_;
+  const std::function<void(Request*)> callback_;
+  const int threads_;
+  const int max_request_len_;
+
+  int close_fd_;
+};
+
+}  // namespace firecgi