Response compression (#9300)

* First draft of response compression.

* Cleanup.

* Removed compression from /_api/version.

* Added ruby test for response compression.

arangod/Agency/RestAgencyHandler.cpp

@@ -583,6 +583,7 @@ RestStatus RestAgencyHandler::reportMethodNotAllowed() {
 }
 
 RestStatus RestAgencyHandler::execute() {
+  response()->setAllowCompression(true);
   try {
     auto const& suffixes = _request->suffixes();
     if (suffixes.empty()) {  // Empty request

arangod/GeneralServer/RestHandler.cpp

@@ -295,6 +295,8 @@ void RestHandler::runHandlerStateMachine() {
 
       case HandlerState::FINALIZE:
         RequestStatistics::SET_REQUEST_END(_statistics);
+        // compress response if required
+        compressResponse();
         // Callback may stealStatistics!
         _callback(this);
         // Schedule callback BEFORE! finalize
@@ -485,6 +487,22 @@ void RestHandler::generateError(rest::ResponseCode code, int errorNumber,
   }
 }
 
+void RestHandler::compressResponse() {
+  if (_response->isCompressionAllowed()) {
+
+    switch (_request->acceptEncoding()) {
+      case rest::EncodingType::DEFLATE:
+        _response->deflate();
+        _response->setHeader(StaticStrings::ContentEncoding, StaticStrings::EncodingDeflate);
+        break;
+
+      default:
+        break;
+    }
+
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief generates an error
 ////////////////////////////////////////////////////////////////////////////////

arangod/GeneralServer/RestHandler.h

@@ -147,6 +147,7 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
   ///        otherwise execute() will be called
   void executeEngine(bool isContinue);
   void shutdownEngine();
+  void compressResponse();
 
  protected:
   enum class HandlerState {

arangod/RestHandler/RestVersionHandler.cpp

@@ -88,6 +88,8 @@ RestStatus RestVersionHandler::execute() {
     }  // found
   }    // allowInfo
   result.close();
+  response()->setAllowCompression(true);
+
   generateResult(rest::ResponseCode::OK, result.slice());
   return RestStatus::DONE;
 }

lib/Basics/StaticStrings.cpp

@@ -189,6 +189,9 @@ std::string const StaticStrings::MimeTypeText("text/plain; charset=utf-8");
 std::string const StaticStrings::MimeTypeVPack("application/x-velocypack");
 std::string const StaticStrings::MultiPartContentType("multipart/form-data");
 
+// accept-encodings
+std::string const StaticStrings::EncodingDeflate("deflate");
+
 // collection attributes
 std::string const StaticStrings::DistributeShardsLike("distributeShardsLike");
 std::string const StaticStrings::IsSmart("isSmart");

lib/Basics/StaticStrings.h

@@ -174,6 +174,9 @@ class StaticStrings {
   static std::string const MimeTypeVPack;
   static std::string const MultiPartContentType;
 
+  // encodings
+  static std::string const EncodingDeflate;
+
   // collection attributes
   static std::string const NumberOfShards;
   static std::string const IsSmart;

lib/Rest/CommonDefines.h

@@ -91,6 +91,11 @@ enum class ContentType {
   UNSET
 };
 
+enum class EncodingType {
+  DEFLATE,
+  UNSET
+};
+
 enum class ProtocolVersion { HTTP_1_0, HTTP_1_1, VST_1_0, VST_1_1, UNKNOWN };
 
 enum class ConnectionType { C_NONE, C_KEEP_ALIVE, C_CLOSE };

lib/Rest/GeneralRequest.h

@@ -48,6 +48,7 @@ class StringBuffer;
 }
 
 using rest::ContentType;
+using rest::EncodingType;
 using rest::ProtocolVersion;
 using rest::RequestType;
 
@@ -86,7 +87,8 @@ class GeneralRequest {
         _authenticated(false),
         _type(RequestType::ILLEGAL),
         _contentType(ContentType::UNSET),
-        _contentTypeResponse(ContentType::UNSET) {}
+        _contentTypeResponse(ContentType::UNSET),
+        _acceptEncoding(EncodingType::UNSET) {}
 
   virtual ~GeneralRequest();
 
@@ -217,6 +219,8 @@ class GeneralRequest {
   ContentType contentType() const { return _contentType; }
   /// @brief should generally reflect the Accept header
   ContentType contentTypeResponse() const { return _contentTypeResponse; }
+  /// @brief should generally reflect the Accept-Encoding header
+  EncodingType acceptEncoding() const { return _acceptEncoding; }
 
   rest::AuthenticationMethod authenticationMethod() const {
     return _authenticationMethod;
@@ -251,6 +255,7 @@ class GeneralRequest {
   std::vector<std::string> _suffixes;
   ContentType _contentType;  // UNSET, VPACK, JSON
   ContentType _contentTypeResponse;
+  EncodingType _acceptEncoding;
 
   std::unordered_map<std::string, std::string> _headers;
   std::unordered_map<std::string, std::string> _values;

lib/Rest/GeneralResponse.cpp

@@ -437,4 +437,5 @@ GeneralResponse::GeneralResponse(ResponseCode responseCode)
       _contentType(ContentType::UNSET),
       _connectionType(ConnectionType::C_NONE),
       _generateBody(false),
+      _allowCompression(false),
       _contentTypeRequested(ContentType::UNSET) {}

lib/Rest/GeneralResponse.h

@@ -41,6 +41,7 @@ class Slice;
 
 using rest::ConnectionType;
 using rest::ContentType;
+using rest::EncodingType;
 using rest::ResponseCode;
 
 class GeneralRequest;
@@ -77,6 +78,10 @@ class GeneralResponse {
     _contentType = ContentType::CUSTOM;
   }
 
+
+  void setAllowCompression(bool allowed) { _allowCompression = allowed; }
+  virtual bool isCompressionAllowed() { return _allowCompression; }
+
   void setConnectionType(ConnectionType type) { _connectionType = type; }
   void setContentTypeRequested(ContentType type) {
     _contentTypeRequested = type;
@@ -158,6 +163,8 @@ class GeneralResponse {
   /// used for head
   virtual bool setGenerateBody(bool) { return _generateBody; };
 
+  virtual int deflate(size_t size = 16384) = 0;
+
  protected:
   ResponseCode _responseCode;                             // http response code
   std::unordered_map<std::string, std::string> _headers;  // headers/metadata map
@@ -165,6 +172,7 @@ class GeneralResponse {
   ContentType _contentType;
   ConnectionType _connectionType;
   bool _generateBody;
+  bool _allowCompression;
   ContentType _contentTypeRequested;
 };
 }  // namespace arangodb

lib/Rest/HttpRequest.cpp

@@ -552,6 +552,13 @@ void HttpRequest::setHeader(char const* key, size_t keyLength,
       memcmp(key, StaticStrings::Accept.c_str(), keyLength) == 0 &&
       memcmp(value, StaticStrings::MimeTypeVPack.c_str(), valueLength) == 0) {
     _contentTypeResponse = ContentType::VPACK;
+  } else if (keyLength == StaticStrings::AcceptEncoding.size() &&
+      valueLength == StaticStrings::EncodingDeflate.size() &&
+      memcmp(key, StaticStrings::AcceptEncoding.c_str(), keyLength) == 0 &&
+      memcmp(value, StaticStrings::EncodingDeflate.c_str(), valueLength) == 0) {
+    // This can be much more elaborated as the can specify weights on encodings
+    // However, for now just toggle on deflate if deflate is requested
+    _acceptEncoding = EncodingType::DEFLATE;
   } else if (keyLength == StaticStrings::ContentTypeHeader.size() &&
              valueLength == StaticStrings::MimeTypeVPack.size() &&
              memcmp(key, StaticStrings::ContentTypeHeader.c_str(), keyLength) == 0 &&

lib/Rest/HttpResponse.h

@@ -96,7 +96,9 @@ class HttpResponse : public GeneralResponse {
 
  private:
   // the body must already be set. deflate is then run on the existing body
-  int deflate(size_t = 16384);
+  int deflate(size_t size = 16384) override {
+    return _body->deflate(size);
+  }
 
   std::unique_ptr<basics::StringBuffer> stealBody() {
     std::unique_ptr<basics::StringBuffer> bb(_body);

lib/Rest/VstResponse.h

@@ -61,6 +61,9 @@ class VstResponse : public GeneralResponse {
   void addPayload(VPackBuffer<uint8_t>&&, arangodb::velocypack::Options const* = nullptr,
                   bool resolveExternals = true) override;
 
+  bool isCompressionAllowed() override { return false; }
+  int deflate(size_t size = 16384) override { return 0; };
+
  private:
   //_responseCode   - from Base
   //_headers        - from Base

tests/IResearch/RestHandlerMock.h

@@ -55,6 +55,8 @@ struct GeneralResponseMock: public arangodb::GeneralResponse {
   virtual void addPayload(arangodb::velocypack::Slice const& slice, arangodb::velocypack::Options const* options = nullptr, bool resolveExternals = true) override;
   virtual void reset(arangodb::ResponseCode code) override;
   virtual arangodb::Endpoint::TransportType transportType() override;
+  int deflate(size_t size = 16384) override { return 0; }
+  bool isCompressionAllowed() override { return false; }
 };
 
 #endif

tests/rb/HttpInterface/api-http-spec.rb

@@ -271,6 +271,29 @@ describe ArangoDB do
       end
     end
 
+################################################################################
+## checking GZIP requests
+################################################################################
+
+    context "deflate requests" do
+      it "checks handling of a request, with deflate support" do
+        require 'uri'
+        require 'net/http'
+        require 'zlib'
+
+        cmd = "/_api/version"
+        deflatedVersion = ArangoDB.log_get("version-deflate-get", cmd, :headers => { "Accept-Encoding" => "deflate" }, :format => :plain)
+        version = ArangoDB.log_get("version-get", cmd, :headers => { "Accept-Encoding" => "" }, :format => :plain)
+
+        # check content encoding
+        deflatedVersion.headers['Content-Encoding'].should eq('deflate')
+
+        # compare both responses
+        inflatedVersionStr = Zlib::inflate deflatedVersion.body
+        version.body.should eq(inflatedVersionStr)
+      end
+    end
+
 ################################################################################
 ## checking CORS requests
 ################################################################################