Repair traffic accounting in statistics. (#10387)

* First stab at porting superuser traffic accounting to devel.
* Remove token forwarding again.
* Fix descriptions of metrics.
* Repair statistics.
* Fix a crash.
* Fix forwarding and permissions.

CHANGELOG

@@ -1,6 +1,9 @@
 devel
 -----
 
+* Separately account for superuser and user request traffic. This is
+  needed for Oasis.
+
 * No longer put system services in _app on single server. On cluster, this
   has never worked. This was unnecessary.
 

arangod/GeneralServer/CommTask.cpp

@@ -337,9 +337,10 @@ void CommTask::executeRequest(std::unique_ptr<GeneralRequest> request,
   bool forwarded;
   auto res = handler->forwardRequest(forwarded);
   if (forwarded) {
-    std::move(res).thenFinal([self = shared_from_this(), handler = std::move(handler)](
+    RequestStatistics::SET_SUPERUSER(statistics(messageId));
+    std::move(res).thenFinal([self = shared_from_this(), handler = std::move(handler), messageId](
                                  futures::Try<Result> && /*ignored*/) -> void {
-      self->sendResponse(handler->stealResponse(), handler->stealStatistics());
+      self->sendResponse(handler->stealResponse(), self->stealStatistics(messageId));
     });
     return;
   }

arangod/GeneralServer/HttpCommTask.cpp

@@ -202,7 +202,6 @@ int HttpCommTask<T>::on_message_complete(llhttp_t* p) {
 
   RequestStatistics* stat = self->statistics(1UL);
   RequestStatistics::SET_READ_END(stat);
-  RequestStatistics::ADD_RECEIVED_BYTES(stat, self->_request->body().size());
   self->_messageDone = true;
 
   return HPE_PAUSED;
@@ -293,6 +292,9 @@ bool HttpCommTask<T>::readCallback(asio_ns::error_code ec) {
     TRI_ASSERT(parsedBytes < std::numeric_limits<size_t>::max());
     // Remove consumed data from receive buffer.
     this->_protocol->buffer.consume(parsedBytes);
+    // And count it in the statistics:
+    RequestStatistics* stat = this->statistics(1UL);
+    RequestStatistics::ADD_RECEIVED_BYTES(stat, parsedBytes);
 
     if (err == HPE_PAUSED_UPGRADE) {
       this->addSimpleResponse(rest::ResponseCode::NOT_IMPLEMENTED,
@@ -422,6 +424,11 @@ void HttpCommTask<T>::processRequest() {
     return;
   }
 
+  // We want to separate superuser token traffic:
+  if (_request->authenticated() && _request->user().empty()) {
+    RequestStatistics::SET_SUPERUSER(this->statistics(1UL));
+  }
+
   // first check whether we allow the request to continue
   CommTask::Flow cont = this->prepareExecution(*_request);
   if (cont != CommTask::Flow::Continue) {
@@ -795,14 +802,16 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
     buffers[1] = asio_ns::buffer(body->data(), body->size());
     TRI_ASSERT(len == body->size());
   }
+  RequestStatistics::SET_WRITE_START(stat);
 
   // FIXME measure performance w/o sync write
   auto cb = [self = CommTask::shared_from_this(),
              h = std::move(header),
-             b = std::move(body)](asio_ns::error_code ec,
+             b = std::move(body),
-                                  size_t nwrite) {
+             stat](asio_ns::error_code ec, size_t nwrite) {
     auto* thisPtr = static_cast<HttpCommTask<T>*>(self.get());
-
+    RequestStatistics::SET_WRITE_END(stat);
+    RequestStatistics::ADD_SENT_BYTES(stat, h->size() + b->size());
     llhttp_errno_t err = llhttp_get_errno(&thisPtr->_parser);
     if (ec || !thisPtr->_shouldKeepAlive || err != HPE_PAUSED) {
       if (ec) {
@@ -815,6 +824,9 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
       llhttp_resume(&thisPtr->_parser);
       thisPtr->asyncReadSome();
     }
+    if (stat != nullptr) {
+      stat->release();
+    }
   };
   asio_ns::async_write(this->_protocol->socket, buffers, std::move(cb));
 }

arangod/GeneralServer/RestHandler.cpp

@@ -141,6 +141,29 @@ futures::Future<Result> RestHandler::forwardRequest(bool& forwarded) {
   std::map<std::string, std::string> headers{_request->headers().begin(),
                                              _request->headers().end()};
 
+  if (headers.find(StaticStrings::Authorization) == headers.end()) {
+    // No authorization header is set, this is in particular the case if this
+    // request is coming in with VelocyStream, where the authentication happens
+    // once at the beginning of the connection and not with every request.
+    // In this case, we have to produce a proper JWT token as authorization:
+      auto auth = AuthenticationFeature::instance();
+    if (auth != nullptr && auth->isActive()) {
+      // when in superuser mode, username is empty
+      // in this case ClusterComm will add the default superuser token
+      std::string const& username = _request->user();
+      if (!username.empty()) {
+        VPackBuilder builder;
+        {
+          VPackObjectBuilder payload{&builder};
+          payload->add("preferred_username", VPackValue(username));
+        }
+        VPackSlice slice = builder.slice();
+        headers.emplace(StaticStrings::Authorization,
+                        "bearer " + auth->tokenCache().generateJwt(slice));
+      }
+    }
+  }
+
   network::RequestOptions options;
   options.database = dbname;
   options.timeout = network::Timeout(300);

arangod/GeneralServer/VstCommTask.cpp

@@ -219,11 +219,20 @@ bool VstCommTask<T>::processMessage(velocypack::Buffer<uint8_t> buffer,
     // error is handled below
   }
 
-  RequestStatistics::SET_READ_END(this->statistics(messageId));
+  RequestStatistics* stat = this->statistics(messageId);
+  RequestStatistics::SET_READ_END(stat);
+  RequestStatistics::ADD_RECEIVED_BYTES(stat, buffer.size());
   
   // handle request types
   if (mt == MessageType::Authentication) {  // auth
     handleAuthHeader(VPackSlice(buffer.data()), messageId);
+    // Separate superuser traffic:
+    // Note that currently, velocystream traffic will never come from
+    // a forwarding, since we always forward with HTTP.
+    if (_authMethod != AuthenticationMethod::NONE && _authorized &&
+        this->_authToken._username.empty()) {
+      RequestStatistics::SET_SUPERUSER(stat);
+    }
   } else if (mt == MessageType::Request) {  // request
     
     VPackSlice header(buffer.data());
@@ -240,6 +249,14 @@ bool VstCommTask<T>::processMessage(velocypack::Buffer<uint8_t> buffer,
       this->_auth->userManager()->refreshUser(this->_authToken._username);
     }
     
+    // Separate superuser traffic:
+    // Note that currently, velocystream traffic will never come from
+    // a forwarding, since we always forward with HTTP.
+    if (_authMethod != AuthenticationMethod::NONE && _authorized &&
+        this->_authToken._username.empty()) {
+      RequestStatistics::SET_SUPERUSER(stat);
+    }
+
     LOG_TOPIC("92fd6", DEBUG, Logger::REQUESTS)
     << "\"vst-request-begin\",\"" << (void*)this << "\",\""
     << this->_connectionInfo.clientAddress << "\",\""
@@ -280,6 +297,8 @@ void VstCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes, Requ
   auto resItem = std::make_unique<ResponseItem>();
   response.writeMessageHeader(resItem->metadata);
   resItem->response = std::move(baseRes);
+  RequestStatistics::SET_WRITE_START(stat);
+  resItem->stat = stat;
   
   asio_ns::const_buffer payload;
   if (response.generateBody()) {
@@ -353,6 +372,8 @@ void VstCommTask<T>::doWrite() {
                          (asio_ns::error_code ec, size_t transferred) {
       
       auto* thisPtr = static_cast<VstCommTask<T>*>(self.get());
+      RequestStatistics::SET_WRITE_END(rsp->stat);
+      RequestStatistics::ADD_SENT_BYTES(rsp->stat, rsp->buffers[0].size() + rsp->buffers[1].size());
       if (ec) {
         LOG_TOPIC("5c6b4", INFO, arangodb::Logger::REQUESTS)
         << "asio write error: '" << ec.message() << "'";
@@ -360,6 +381,7 @@ void VstCommTask<T>::doWrite() {
       } else {
         thisPtr->doWrite(); // write next one
       }
+      rsp->stat->release();
     });
     
     break; // done

arangod/GeneralServer/VstCommTask.h

@@ -100,6 +100,7 @@ class VstCommTask final : public GeneralCommTask<T> {
   };
   
   struct ResponseItem {
+    RequestStatistics* stat;
     velocypack::Buffer<uint8_t> metadata;
     std::unique_ptr<GeneralResponse> response;
     std::vector<asio_ns::const_buffer> buffers;

arangod/RestHandler/RestAdminStatisticsHandler.cpp

@@ -27,6 +27,7 @@
 #include "RestAdminStatisticsHandler.h"
 #include "GeneralServer/ServerSecurityFeature.h"
 #include "Statistics/Descriptions.h"
+#include "Statistics/RequestStatistics.h"
 #include "Statistics/StatisticsFeature.h"
 
 using namespace arangodb;
@@ -85,9 +86,13 @@ void RestAdminStatisticsHandler::getStatistics() {
   tmp.close();  // system
 
   tmp.add("client", VPackValue(VPackValueType::Object, true));
-  desc->clientStatistics(tmp);
+  desc->clientStatistics(tmp, stats::RequestStatisticsSource::ALL);
   tmp.close();  // client
 
+  tmp.add("clientUser", VPackValue(VPackValueType::Object, true));
+  desc->clientStatistics(tmp, stats::RequestStatisticsSource::USER);
+  tmp.close();  // clientUser
+
   tmp.add("http", VPackValue(VPackValueType::Object, true));
   desc->httpStatistics(tmp);
   tmp.close();  // http

arangod/Statistics/Descriptions.cpp

@@ -41,6 +41,8 @@ std::string stats::fromGroupType(stats::GroupType gt) {
       return "system";
     case stats::GroupType::Client:
       return "client";
+    case stats::GroupType::ClientUser:
+      return "clientUser";
     case stats::GroupType::Http:
       return "http";
     case stats::GroupType::Vst:
@@ -113,6 +115,9 @@ stats::Descriptions::Descriptions()
   _groups.emplace_back(Group{stats::GroupType::Client,
                              "Client Connection Statistics",
                              "Statistics about the connections."});
+  _groups.emplace_back(Group{stats::GroupType::ClientUser,
+                             "Client User Connection Statistics",
+                             "Statistics about the connections, only user traffic (ignoring superuser JWT traffic)."});
   _groups.emplace_back(Group{stats::GroupType::Http, "HTTP Request Statistics",
                              "Statistics about the HTTP requests."});
   _groups.emplace_back(Group{stats::GroupType::Server, "Server Statistics",
@@ -248,6 +253,60 @@ stats::Descriptions::Descriptions()
              // cuts: internal.connectionTimeDistribution,
              stats::Unit::Seconds, _connectionTimeCuts});
 
+  // Only user traffic:
+
+  _figures.emplace_back(
+      Figure{stats::GroupType::ClientUser,
+             "httpConnections",
+             "Client Connections",
+             "The number of connections that are currently open (only user traffic).",
+             stats::FigureType::Current,
+             stats::Unit::Number,
+             {}});
+
+  _figures.emplace_back(
+      Figure{stats::GroupType::ClientUser, "totalTime", "Total Time",
+             "Total time needed to answer a request (only user traffic).",
+             stats::FigureType::Distribution,
+             // cuts: internal.requestTimeDistribution,
+             stats::Unit::Seconds, _requestTimeCuts});
+
+  _figures.emplace_back(
+      Figure{stats::GroupType::ClientUser, "requestTime", "Request Time",
+             "Request time needed to answer a request (only user traffic).",
+             stats::FigureType::Distribution,
+             // cuts: internal.requestTimeDistribution,
+             stats::Unit::Seconds, _requestTimeCuts});
+
+  _figures.emplace_back(
+      Figure{stats::GroupType::ClientUser, "queueTime", "Queue Time",
+             "Queue time needed to answer a request (only user traffic).",
+             stats::FigureType::Distribution,
+             // cuts: internal.requestTimeDistribution,
+             stats::Unit::Seconds, _requestTimeCuts});
+
+  _figures.emplace_back(Figure{stats::GroupType::ClientUser, "bytesSent",
+                               "Bytes Sent",
+                               "Bytes sents for a request (only user traffic).",
+                               stats::FigureType::Distribution,
+                               // cuts: internal.bytesSentDistribution,
+                               stats::Unit::Bytes, _bytesSendCuts});
+
+  _figures.emplace_back(Figure{stats::GroupType::ClientUser, "bytesReceived",
+                               "Bytes Received",
+                               "Bytes received for a request (only user traffic).",
+                               stats::FigureType::Distribution,
+                               // cuts: internal.bytesReceivedDistribution,
+                               stats::Unit::Bytes, _bytesReceivedCuts});
+
+  _figures.emplace_back(
+      Figure{stats::GroupType::ClientUser, "connectionTime",
+             "Connection Time",
+             "Total connection time of a client (only user traffic).",
+             stats::FigureType::Distribution,
+             // cuts: internal.connectionTimeDistribution,
+             stats::Unit::Seconds, _connectionTimeCuts});
+
   _figures.emplace_back(Figure{stats::GroupType::Http,
                                "requestsTotal",
                                "Total requests",
@@ -411,7 +470,7 @@ static void FillDistribution(VPackBuilder& b, std::string const& name,
   b.close();
 }
 
-void stats::Descriptions::clientStatistics(velocypack::Builder& b) const {
+void stats::Descriptions::clientStatistics(velocypack::Builder& b, RequestStatisticsSource source) const {
   basics::StatisticsCounter httpConnections;
   basics::StatisticsCounter totalRequests;
   std::array<basics::StatisticsCounter, basics::MethodRequestsStatisticsSize> methodRequests;
@@ -432,7 +491,7 @@ void stats::Descriptions::clientStatistics(velocypack::Builder& b) const {
   basics::StatisticsDistribution bytesSent;
   basics::StatisticsDistribution bytesReceived;
 
-  RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived);
+  RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived, source);
 
   FillDistribution(b, "totalTime", totalTime);
   FillDistribution(b, "requestTime", requestTime);

arangod/Statistics/Descriptions.h

@@ -31,7 +31,13 @@
 namespace arangodb {
 namespace stats {
 
-enum class GroupType { System, Client, Http, Vst, Server };
+enum RequestStatisticsSource {
+  USER,
+  SUPERUSER,
+  ALL
+};
+
+enum class GroupType { System, Client, ClientUser, Http, Vst, Server };
 
 std::string fromGroupType(stats::GroupType);
 
@@ -75,7 +81,7 @@ class Descriptions final {
   std::vector<stats::Figure> const& figures() const { return _figures; }
 
   void serverStatistics(velocypack::Builder&) const;
-  void clientStatistics(velocypack::Builder&) const;
+  void clientStatistics(velocypack::Builder&, RequestStatisticsSource source) const;
   void httpStatistics(velocypack::Builder&) const;
   void processStatistics(velocypack::Builder&) const;
 

arangod/Statistics/RequestStatistics.cpp

@@ -126,26 +126,50 @@ void RequestStatistics::process(RequestStatistics* statistics) {
         totalTime = statistics->_writeEnd - statistics->_readStart;
       }
 
-      TRI_TotalTimeDistributionStatistics.addFigure(totalTime);
+      if (statistics->_superuser) {
+        TRI_TotalTimeDistributionStatistics.addFigure(totalTime);
 
-      double requestTime = statistics->_requestEnd - statistics->_requestStart;
+        double requestTime = statistics->_requestEnd - statistics->_requestStart;
-      TRI_RequestTimeDistributionStatistics.addFigure(requestTime);
+        TRI_RequestTimeDistributionStatistics.addFigure(requestTime);
 
-      double queueTime = 0.0;
+        double queueTime = 0.0;
+
+        if (statistics->_queueStart != 0.0 && statistics->_queueEnd != 0.0) {
+          queueTime = statistics->_queueEnd - statistics->_queueStart;
+          TRI_QueueTimeDistributionStatistics.addFigure(queueTime);
+        }
+
+        double ioTime = totalTime - requestTime - queueTime;
+
+        if (ioTime >= 0.0) {
+          TRI_IoTimeDistributionStatistics.addFigure(ioTime);
+        }
+
+        TRI_BytesSentDistributionStatistics.addFigure(statistics->_sentBytes);
+        TRI_BytesReceivedDistributionStatistics.addFigure(statistics->_receivedBytes);
+      } else {
+        TRI_TotalTimeDistributionStatisticsUser.addFigure(totalTime);
+
+        double requestTime = statistics->_requestEnd - statistics->_requestStart;
+        TRI_RequestTimeDistributionStatisticsUser.addFigure(requestTime);
+
+        double queueTime = 0.0;
+
+        if (statistics->_queueStart != 0.0 && statistics->_queueEnd != 0.0) {
+          queueTime = statistics->_queueEnd - statistics->_queueStart;
+          TRI_QueueTimeDistributionStatisticsUser.addFigure(queueTime);
+        }
+
+        double ioTime = totalTime - requestTime - queueTime;
+
+        if (ioTime >= 0.0) {
+          TRI_IoTimeDistributionStatisticsUser.addFigure(ioTime);
+        }
+
+        TRI_BytesSentDistributionStatisticsUser.addFigure(statistics->_sentBytes);
+        TRI_BytesReceivedDistributionStatisticsUser.addFigure(statistics->_receivedBytes);
 
-      if (statistics->_queueStart != 0.0 && statistics->_queueEnd != 0.0) {
-        queueTime = statistics->_queueEnd - statistics->_queueStart;
-        TRI_QueueTimeDistributionStatistics.addFigure(queueTime);
       }
-
-      double ioTime = totalTime - requestTime - queueTime;
-
-      if (ioTime >= 0.0) {
-        TRI_IoTimeDistributionStatistics.addFigure(ioTime);
-      }
-
-      TRI_BytesSentDistributionStatistics.addFigure(statistics->_sentBytes);
-      TRI_BytesReceivedDistributionStatistics.addFigure(statistics->_receivedBytes);
     }
   }
 
@@ -190,18 +214,36 @@ void RequestStatistics::fill(StatisticsDistribution& totalTime,
                              StatisticsDistribution& requestTime,
                              StatisticsDistribution& queueTime,
                              StatisticsDistribution& ioTime, StatisticsDistribution& bytesSent,
-                             StatisticsDistribution& bytesReceived) {
+                             StatisticsDistribution& bytesReceived,
+                             stats::RequestStatisticsSource source) {
   if (!StatisticsFeature::enabled()) {
     // all the below objects may be deleted if we don't have statistics enabled
     return;
   }
 
-  totalTime = TRI_TotalTimeDistributionStatistics;
+  if (source == stats::RequestStatisticsSource::USER) {
-  requestTime = TRI_RequestTimeDistributionStatistics;
+    totalTime = TRI_TotalTimeDistributionStatisticsUser;
-  queueTime = TRI_QueueTimeDistributionStatistics;
+    requestTime = TRI_RequestTimeDistributionStatisticsUser;
-  ioTime = TRI_IoTimeDistributionStatistics;
+    queueTime = TRI_QueueTimeDistributionStatisticsUser;
-  bytesSent = TRI_BytesSentDistributionStatistics;
+    ioTime = TRI_IoTimeDistributionStatisticsUser;
-  bytesReceived = TRI_BytesReceivedDistributionStatistics;
+    bytesSent = TRI_BytesSentDistributionStatisticsUser;
+    bytesReceived = TRI_BytesReceivedDistributionStatisticsUser;
+  } else {
+    totalTime = TRI_TotalTimeDistributionStatistics;
+    requestTime = TRI_RequestTimeDistributionStatistics;
+    queueTime = TRI_QueueTimeDistributionStatistics;
+    ioTime = TRI_IoTimeDistributionStatistics;
+    bytesSent = TRI_BytesSentDistributionStatistics;
+    bytesReceived = TRI_BytesReceivedDistributionStatistics;
+  }
+  if (source == stats::RequestStatisticsSource::ALL) {
+    totalTime.add(TRI_TotalTimeDistributionStatisticsUser);
+    requestTime.add(TRI_RequestTimeDistributionStatisticsUser);
+    queueTime.add(TRI_QueueTimeDistributionStatisticsUser);
+    ioTime.add(TRI_IoTimeDistributionStatisticsUser);
+    bytesSent.add(TRI_BytesSentDistributionStatisticsUser);
+    bytesReceived.add(TRI_BytesReceivedDistributionStatisticsUser);
+  }
 }
 
 std::string RequestStatistics::timingsCsv() {
@@ -233,7 +275,8 @@ std::string RequestStatistics::to_string() {
      << "_async          " << _async << std::endl
      << "_tooLarge       " << _tooLarge << std::endl
      << "_executeError   " << _executeError << std::endl
-     << "_ignore         " << _ignore << std::endl;
+     << "_ignore         " << _ignore << std::endl
+     << "_superuser      " << _superuser << std::endl;
 
   return ss.str();
 }
@@ -280,4 +323,7 @@ void RequestStatistics::trace_log() {
 
   LOG_TOPIC("31657", TRACE, Logger::REQUESTS) << std::boolalpha << std::setprecision(20)
                                      << "_ignore         " << _ignore;
+
+  LOG_TOPIC("31658", TRACE, Logger::REQUESTS) << std::boolalpha << std::setprecision(20)
+                                     << "_superuser      " << _superuser;
 }

arangod/Statistics/RequestStatistics.h

@@ -144,6 +144,12 @@ class RequestStatistics {
     }
   }
 
+  static void SET_SUPERUSER(RequestStatistics* stat) {
+    if (stat != nullptr) {
+      stat->_superuser = true;
+    }
+  }
+
   double requestStart() const { return _requestStart; }
 
   static void fill(basics::StatisticsDistribution& totalTime,
@@ -151,7 +157,8 @@ class RequestStatistics {
                    basics::StatisticsDistribution& queueTime,
                    basics::StatisticsDistribution& ioTime,
                    basics::StatisticsDistribution& bytesSent,
-                   basics::StatisticsDistribution& bytesReceived);
+                   basics::StatisticsDistribution& bytesReceived,
+                   stats::RequestStatisticsSource source);
 
   std::string timingsCsv();
   std::string to_string();
@@ -189,6 +196,7 @@ class RequestStatistics {
     _ignore = false;
     _released = true;
     _inQueue = false;
+    _superuser = false;
   }
 
   double _readStart;   // CommTask::processRead - read first byte of message
@@ -213,6 +221,7 @@ class RequestStatistics {
   bool _ignore;
   bool _released;
   bool _inQueue;
+  bool _superuser;
 };
 }  // namespace arangodb
 

arangod/Statistics/StatisticsFeature.cpp

@@ -74,6 +74,12 @@ StatisticsDistribution TRI_IoTimeDistributionStatistics(TRI_RequestTimeDistribut
 StatisticsDistribution TRI_QueueTimeDistributionStatistics(TRI_RequestTimeDistributionVectorStatistics);
 StatisticsDistribution TRI_RequestTimeDistributionStatistics(TRI_RequestTimeDistributionVectorStatistics);
 StatisticsDistribution TRI_TotalTimeDistributionStatistics(TRI_RequestTimeDistributionVectorStatistics);
+StatisticsDistribution TRI_BytesReceivedDistributionStatisticsUser(TRI_BytesReceivedDistributionVectorStatistics);
+StatisticsDistribution TRI_BytesSentDistributionStatisticsUser(TRI_BytesSentDistributionVectorStatistics);
+StatisticsDistribution TRI_IoTimeDistributionStatisticsUser(TRI_RequestTimeDistributionVectorStatistics);
+StatisticsDistribution TRI_QueueTimeDistributionStatisticsUser(TRI_RequestTimeDistributionVectorStatistics);
+StatisticsDistribution TRI_RequestTimeDistributionStatisticsUser(TRI_RequestTimeDistributionVectorStatistics);
+StatisticsDistribution TRI_TotalTimeDistributionStatisticsUser(TRI_RequestTimeDistributionVectorStatistics);
 
 }  // namespace basics
 }  // namespace arangodb

arangod/Statistics/StatisticsFeature.h

@@ -59,6 +59,12 @@ extern StatisticsDistribution TRI_IoTimeDistributionStatistics;
 extern StatisticsDistribution TRI_QueueTimeDistributionStatistics;
 extern StatisticsDistribution TRI_RequestTimeDistributionStatistics;
 extern StatisticsDistribution TRI_TotalTimeDistributionStatistics;
+extern StatisticsDistribution TRI_BytesReceivedDistributionStatisticsUser;
+extern StatisticsDistribution TRI_BytesSentDistributionStatisticsUser;
+extern StatisticsDistribution TRI_IoTimeDistributionStatisticsUser;
+extern StatisticsDistribution TRI_QueueTimeDistributionStatisticsUser;
+extern StatisticsDistribution TRI_RequestTimeDistributionStatisticsUser;
+extern StatisticsDistribution TRI_TotalTimeDistributionStatisticsUser;
 }  // namespace basics
 
 class StatisticsFeature final : public application_features::ApplicationFeature {

arangod/Statistics/StatisticsWorker.cpp

@@ -834,7 +834,7 @@ void StatisticsWorker::generateRawStatistics(VPackBuilder& builder, double const
   StatisticsDistribution bytesSent;
   StatisticsDistribution bytesReceived;
 
-  RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived);
+  RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived, stats::RequestStatisticsSource::ALL);
 
   ServerStatistics const& serverInfo = ServerStatistics::statistics();
 

arangod/Statistics/figures.h

@@ -95,6 +95,19 @@ struct StatisticsDistribution {
     ++(*j);
   }
 
+  void add(StatisticsDistribution& other) {
+    MUTEX_LOCKER(lock, _mutex);
+    MUTEX_LOCKER(lock2, other._mutex);
+    TRI_ASSERT(_counts.size() == other._counts.size() &&
+               _cuts.size() == _cuts.size());
+    _count += other._count;
+    _total += other._total;
+    for (size_t i = 0; i < _counts.size(); ++i) {
+      TRI_ASSERT(i < _cuts.size() ? _cuts[i] == other._cuts[i] : true);
+      _counts[i] += other._counts[i];
+    }
+  }
+
   uint64_t _count;
   double _total;
   std::vector<double> _cuts;

arangod/V8Server/v8-statistics.cpp

@@ -224,7 +224,7 @@ static void JS_ClientStatistics(v8::FunctionCallbackInfo<v8::Value> const& args)
   StatisticsDistribution bytesSent;
   StatisticsDistribution bytesReceived;
 
-  RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived);
+  RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived, stats::RequestStatisticsSource::ALL);
 
   FillDistribution(isolate, result, TRI_V8_ASCII_STRING(isolate, "totalTime"), totalTime);
   FillDistribution(isolate, result, TRI_V8_ASCII_STRING(isolate, "requestTime"), requestTime);