Merge branch 'spdvpk' of ssh://github.com/ArangoDB/ArangoDB into spdvpk

.gitignore

@@ -16,6 +16,7 @@
 *.diff
 *.patch
 *.lnk
+Thumbs.db
 
 testresult.json
 

3rdParty/V8/CMakeLists.txt

@@ -83,7 +83,7 @@ set(V8_INCLUDE_DIR
 
 
 if (${CMAKE_GENERATOR} MATCHES "Ninja")
-  message("NINJA!")
+  message(STATUS "using generator NINJA!")
 
   option(USE_DEBUG_V8 "compile V8 in DEBUG mode" OFF)
 

CMakeLists.txt

@@ -393,13 +393,13 @@ if (CMAKE_COMPILER_IS_GNUCC)
 
   set(BASE_FLAGS "${BASE_FLAGS} -Wall -Wextra -Wno-unused-parameter")
 
-  set(CMAKE_C_FLAGS                ""                               CACHE INTERNAL "default C compiler flags")
+  set(CMAKE_C_FLAGS                "-g"                             CACHE INTERNAL "default C compiler flags")
   set(CMAKE_C_FLAGS_DEBUG          "-O0 -g"                         CACHE INTERNAL "C debug flags")
   set(CMAKE_C_FLAGS_MINSIZEREL     "-Os"                            CACHE INTERNAL "C minimal size flags")
   set(CMAKE_C_FLAGS_RELEASE        "-O3 -fomit-frame-pointer"       CACHE INTERNAL "C release flags")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -fno-omit-frame-pointer" CACHE INTERNAL "C release with debug info flags")
 
-  set(CMAKE_CXX_FLAGS                ""                               CACHE INTERNAL "default C++ compiler flags")
+  set(CMAKE_CXX_FLAGS                "-g"                             CACHE INTERNAL "default C++ compiler flags")
   set(CMAKE_CXX_FLAGS_DEBUG          "-O0 -g"                         CACHE INTERNAL "C++ debug flags")
   set(CMAKE_CXX_FLAGS_MINSIZEREL     "-Os"                            CACHE INTERNAL "C++ minimal size flags")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O3 -fomit-frame-pointer"       CACHE INTERNAL "C++ release flags")
@@ -412,13 +412,13 @@ elseif (CMAKE_COMPILER_IS_CLANG)
 
   set(BASE_FLAGS "${BASE_FLAGS} -Wall -Wextra -Wno-unused-parameter")
 
-  set(CMAKE_C_FLAGS                ""                               CACHE INTERNAL "default C compiler flags")
+  set(CMAKE_C_FLAGS                "-g"                             CACHE INTERNAL "default C compiler flags")
   set(CMAKE_C_FLAGS_DEBUG          "-O0 -g"                         CACHE INTERNAL "C debug flags")
   set(CMAKE_C_FLAGS_MINSIZEREL     "-Os"                            CACHE INTERNAL "C minimal size flags")
   set(CMAKE_C_FLAGS_RELEASE        "-O3 -fomit-frame-pointer"       CACHE INTERNAL "C release flags")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -fno-omit-frame-pointer" CACHE INTERNAL "C release with debug info flags")
 
-  set(CMAKE_CXX_FLAGS                ""                               CACHE INTERNAL "default C++ compiler flags")
+  set(CMAKE_CXX_FLAGS                "-g"                             CACHE INTERNAL "default C++ compiler flags")
   set(CMAKE_CXX_FLAGS_DEBUG          "-O0 -g"                         CACHE INTERNAL "C++ debug flags")
   set(CMAKE_CXX_FLAGS_MINSIZEREL     "-Os"                            CACHE INTERNAL "C++ minimal size flags")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O3 -fomit-frame-pointer"       CACHE INTERNAL "C++ release flags")
@@ -429,13 +429,13 @@ elseif (MSVC)
     message(STATUS "Compiler type MSVC: ${CMAKE_CXX_COMPILER}")
   endif ()
 
-  set(CMAKE_C_FLAGS                ""                                  CACHE INTERNAL "default C++ compiler flags")
+  set(CMAKE_C_FLAGS                "-g"                                CACHE INTERNAL "default C++ compiler flags")
   set(CMAKE_C_FLAGS_DEBUG          "/D _DEBUG /MTd /Zi /Ob0 /Od /RTC1" CACHE INTERNAL "C++ debug flags")
   set(CMAKE_C_FLAGS_MINSIZEREL     "/MT /O1 /Ob1 /D NDEBUG"            CACHE INTERNAL "C++ minimal size flags")
   set(CMAKE_C_FLAGS_RELEASE        "/MT /O2 /Ob2 /D NDEBUG"            CACHE INTERNAL "C++ release flags")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "/MT /Zi /O2 /Ob1 /D NDEBUG"        CACHE INTERNAL "C++ release with debug info flags")
 
-  set(CMAKE_CXX_FLAGS                ""                                  CACHE INTERNAL "default C++ compiler flags")
+  set(CMAKE_CXX_FLAGS                "-g"                                CACHE INTERNAL "default C++ compiler flags")
   set(CMAKE_CXX_FLAGS_DEBUG          "/D _DEBUG /MTd /Zi /Ob0 /Od /RTC1" CACHE INTERNAL "C++ debug flags")
   set(CMAKE_CXX_FLAGS_MINSIZEREL     "/MT /O1 /Ob1 /D NDEBUG"            CACHE INTERNAL "C++ minimal size flags")
   set(CMAKE_CXX_FLAGS_RELEASE        "/MT /O2 /Ob2 /D NDEBUG"            CACHE INTERNAL "C++ release flags")
@@ -447,13 +447,13 @@ else ()
 
   set(BASE_FLAGS "${BASE_FLAGS} -Wall")
 
-  set(CMAKE_C_FLAGS                ""                               CACHE INTERNAL "default C compiler flags")
+  set(CMAKE_C_FLAGS                "-g"                             CACHE INTERNAL "default C compiler flags")
   set(CMAKE_C_FLAGS_DEBUG          "-O0 -g"                         CACHE INTERNAL "C debug flags")
   set(CMAKE_C_FLAGS_MINSIZEREL     "-Os"                            CACHE INTERNAL "C minimal size flags")
   set(CMAKE_C_FLAGS_RELEASE        "-O3 -fomit-frame-pointer"       CACHE INTERNAL "C release flags")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -fno-omit-frame-pointer" CACHE INTERNAL "C release with debug info flags")
 
-  set(CMAKE_CXX_FLAGS                ""                               CACHE INTERNAL "default C++ compiler flags")
+  set(CMAKE_CXX_FLAGS                "-g"                             CACHE INTERNAL "default C++ compiler flags")
   set(CMAKE_CXX_FLAGS_DEBUG          "-O0 -g"                         CACHE INTERNAL "C++ debug flags")
   set(CMAKE_CXX_FLAGS_MINSIZEREL     "-Os"                            CACHE INTERNAL "C++ minimal size flags")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O3 -fomit-frame-pointer"       CACHE INTERNAL "C++ release flags")

LICENSES-OTHER-COMPONENTS.md

@@ -18,11 +18,6 @@
 * Project Home: https://code.google.com/p/fast-hash/
 * License: [MIT License](https://code.google.com/p/fast-hash/)
 
-### levenshtein distance
-
-* Project Home: https://github.com/bennybp/stringmatch/
-* License: [MIT License](https://github.com/bennybp/stringmatch/blob/master/stringmatch.h)
-
 ### Google V8 4.3.61
 
 * Project Home: https://code.google.com/p/v8/

README_maintainers.md

@@ -639,3 +639,34 @@ If there is a third argument and it is "D", then all servers are started
 up in the GNU debugger in separate windows (using `xterm`s). In that
 case one has to hit ENTER in the original terminal where the script runs
 to continue, once all processes have been start up in the debugger.
+
+Front-End (WebUI)
+=========
+
+To see the changes of possible modifications in javascript files, templates
+and scss files please use grunt to generate the bundled version. 
+
+To install grunt (with all related dependencies), just go to the frontend app
+folder (/js/apps/system/_admin/aardvark/APP) and run: 
+
+`npm install`
+
+On Mac OS you also have to install grunt-cli:
+
+`(sudo) npm install -g grunt-cli`
+
+Then you can choose between three choices:
+
+1. Build all arangodb related files:
+
+  * `grunt`
+
+2. Build all arangodb related files, including libraries. This should always
+be used when we offer a new major release of arangodb.
+
+  * `grunt deploy`
+
+3. Live build arangodb related files, when a file has been changed. This task
+does not include the minifying process.
+
+  * `grunt watch`

arangod/Actions/actions.cpp

@@ -27,7 +27,7 @@
 #include "Basics/ReadWriteLock.h"
 #include "Basics/StringUtils.h"
 #include "Basics/WriteLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Rest/HttpRequest.h"
 
 using namespace arangodb::basics;

arangod/ApplicationServer/ApplicationServer.cpp

@@ -35,7 +35,7 @@
 #include "Basics/conversions.h"
 #include "Basics/files.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/RandomGenerator.h"
 #include "Basics/StringUtils.h"
 #include "Basics/tri-strings.h"

arangod/Aql/ExecutionEngine.cpp

@@ -21,13 +21,13 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "Aql/ExecutionEngine.h"
+#include "ExecutionEngine.h"
-#include "Aql/CollectOptions.h"
 #include "Aql/BasicBlocks.h"
 #include "Aql/CalculationBlock.h"
 #include "Aql/ClusterBlocks.h"
 #include "Aql/CollectBlock.h"
 #include "Aql/CollectNode.h"
+#include "Aql/CollectOptions.h"
 #include "Aql/EnumerateCollectionBlock.h"
 #include "Aql/EnumerateListBlock.h"
 #include "Aql/ExecutionBlock.h"
@@ -41,9 +41,9 @@
 #include "Aql/TraversalBlock.h"
 #include "Aql/WalkerWorker.h"
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/ClusterComm.h"
+#include "Logger/Logger.h"
 #include "VocBase/server.h"
 
 using namespace arangodb::aql;

arangod/Aql/QueryList.cpp

@@ -23,7 +23,7 @@
 
 #include "Aql/QueryList.h"
 #include "Aql/Query.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/WriteLocker.h"
 #include "Basics/Exceptions.h"

arangod/Aql/QueryRegistry.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "Aql/QueryRegistry.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/WriteLocker.h"
 #include "Aql/ExecutionEngine.h"

arangod/Aql/RestAqlHandler.cpp

@@ -25,7 +25,7 @@
 #include "Aql/ClusterBlocks.h"
 #include "Aql/ExecutionEngine.h"
 #include "Aql/ExecutionBlock.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringUtils.h"
 #include "Basics/tri-strings.h"
 #include "Basics/VelocyPackHelper.h"

arangod/Aql/SortCondition.cpp

@@ -23,7 +23,7 @@
 
 #include "SortCondition.h"
 #include "Aql/AstNode.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 
 using namespace arangodb::aql;
 

arangod/Cluster/AgencyComm.cpp

@@ -21,15 +21,16 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "Cluster/AgencyComm.h"
+#include "AgencyComm.h"
-#include "Basics/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
+#include "Basics/json.h"
 #include "Basics/random.h"
 #include "Cluster/ServerState.h"
+#include "Logger/Logger.h"
 #include "Rest/Endpoint.h"
 #include "Rest/HttpRequest.h"
 #include "SimpleHttpClient/GeneralClientConnection.h"

arangod/Cluster/ApplicationCluster.cpp

@@ -25,7 +25,7 @@
 #include "Rest/Endpoint.h"
 #include "Basics/files.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/HeartbeatThread.h"
 #include "Cluster/ServerState.h"

arangod/Cluster/ClusterComm.cpp

@@ -23,7 +23,7 @@
 
 #include "Cluster/ClusterComm.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/StringUtils.h"
 #include "Cluster/ClusterInfo.h"

arangod/Cluster/ClusterComm.h

@@ -26,7 +26,7 @@
 
 #include "Basics/Common.h"
 #include "Basics/ConditionVariable.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ReadWriteLock.h"
 #include "Basics/Thread.h"
 #include "Cluster/AgencyComm.h"

arangod/Cluster/ClusterInfo.cpp

@@ -24,7 +24,7 @@
 
 #include "ClusterInfo.h"
 #include "Basics/json.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/json-utilities.h"
 #include "Basics/JsonHelper.h"
 #include "Basics/MutexLocker.h"

arangod/Cluster/HeartbeatThread.cpp

@@ -24,7 +24,7 @@
 #include "HeartbeatThread.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/JsonHelper.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/ClusterComm.h"

arangod/Cluster/HeartbeatThread.h

@@ -28,7 +28,7 @@
 #include "Basics/ConditionVariable.h"
 #include "Basics/Mutex.h"
 #include "Basics/Thread.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Cluster/AgencyComm.h"
 
 struct TRI_server_t;

arangod/Cluster/ServerJob.cpp

@@ -24,7 +24,7 @@
 #include "ServerJob.h"
 
 #include "Basics/MutexLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Cluster/HeartbeatThread.h"
 #include "Cluster/ClusterInfo.h"
 #include "Dispatcher/DispatcherQueue.h"

arangod/Cluster/ServerState.cpp

@@ -25,7 +25,7 @@
 #include "Basics/ReadLocker.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Cluster/AgencyComm.h"
 #include "Cluster/ClusterInfo.h"
 

arangod/Dispatcher/ApplicationDispatcher.cpp

@@ -27,7 +27,7 @@
 
 #include "ApplicationDispatcher.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Dispatcher/Dispatcher.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/PeriodicTask.h"

arangod/Dispatcher/Dispatcher.cpp

@@ -24,7 +24,7 @@
 
 #include "Dispatcher.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Dispatcher/DispatcherQueue.h"
 #include "Dispatcher/DispatcherThread.h"
 #include "Dispatcher/Job.h"

arangod/Dispatcher/DispatcherQueue.cpp

@@ -24,7 +24,7 @@
 
 #include "DispatcherQueue.h"
 #include "Basics/ConditionLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Dispatcher/DispatcherThread.h"
 #include "Dispatcher/Job.h"

arangod/Dispatcher/DispatcherThread.cpp

@@ -25,7 +25,7 @@
 #include "DispatcherThread.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Dispatcher/Dispatcher.h"
 #include "Dispatcher/DispatcherQueue.h"
 #include "Dispatcher/Job.h"

arangod/FulltextIndex/fulltext-index.cpp

@@ -24,7 +24,7 @@
 #include "fulltext-index.h"
 
 #include "Basics/locks.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 
 #include "fulltext-handles.h"
 #include "fulltext-list.h"

arangod/HttpServer/ApplicationEndpointServer.cpp

@@ -28,14 +28,14 @@
 #include "Basics/FileUtils.h"
 #include "Basics/RandomGenerator.h"
 #include "Basics/ReadLocker.h"
-#include "Basics/WriteLocker.h"
-#include "Basics/Logger.h"
-#include "Basics/ssl-helper.h"
 #include "Basics/VelocyPackHelper.h"
+#include "Basics/WriteLocker.h"
+#include "Basics/ssl-helper.h"
 #include "Dispatcher/ApplicationDispatcher.h"
 #include "HttpServer/HttpHandlerFactory.h"
 #include "HttpServer/HttpServer.h"
 #include "HttpServer/HttpsServer.h"
+#include "Logger/Logger.h"
 #include "Rest/Version.h"
 #include "Scheduler/ApplicationScheduler.h"
 

arangod/HttpServer/AsyncJobManager.cpp

@@ -24,7 +24,7 @@
 #include "AsyncJobManager.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/WriteLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "HttpServer/HttpHandler.h"
 #include "HttpServer/HttpServerJob.h"
 

arangod/HttpServer/HttpCommTask.cpp

@@ -26,7 +26,7 @@
 
 #include "Basics/MutexLocker.h"
 #include "Basics/StringBuffer.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "HttpServer/HttpHandler.h"
 #include "HttpServer/HttpHandlerFactory.h"
 #include "HttpServer/HttpServer.h"
@@ -75,7 +75,12 @@ HttpCommTask::HttpCommTask(HttpServer* server, TRI_socket_t socket,
       _sinceCompactification(0),
       _originalBodyLength(0),
       _setupDone(false) {
-  LOG(TRACE) << "connection established, client " << TRI_get_fd_or_handle_of_socket(socket) << ", server ip " << _connectionInfo.serverAddress << ", server port " << _connectionInfo.serverPort << ", client ip " << _connectionInfo.clientAddress << ", client port " << _connectionInfo.clientPort;
+  LOG(TRACE) << "connection established, client "
+             << TRI_get_fd_or_handle_of_socket(socket) << ", server ip "
+             << _connectionInfo.serverAddress << ", server port "
+             << _connectionInfo.serverPort << ", client ip "
+             << _connectionInfo.clientAddress << ", client port "
+             << _connectionInfo.clientPort;
 
   // acquire a statistics entry and set the type to HTTP
   ConnectionStatisticsAgent::acquire();
@@ -88,7 +93,8 @@ HttpCommTask::HttpCommTask(HttpServer* server, TRI_socket_t socket,
 ////////////////////////////////////////////////////////////////////////////////
 
 HttpCommTask::~HttpCommTask() {
-  LOG(TRACE) << "connection closed, client " << TRI_get_fd_or_handle_of_socket(_commSocket);
+  LOG(TRACE) << "connection closed, client "
+             << TRI_get_fd_or_handle_of_socket(_commSocket);
 
   // free write buffers and statistics
   for (auto& i : _writeBuffers) {
@@ -171,7 +177,8 @@ bool HttpCommTask::processRead() {
     size_t headerLength = ptr - (_readBuffer->c_str() + _startPosition);
 
     if (headerLength > MaximalHeaderSize) {
-      LOG(WARN) << "maximal header size is " << MaximalHeaderSize << ", request header size is " << headerLength;
+      LOG(WARN) << "maximal header size is " << MaximalHeaderSize
+                << ", request header size is " << headerLength;
 
       // header is too large
       HttpResponse response(HttpResponse::REQUEST_HEADER_FIELDS_TOO_LARGE,
@@ -189,8 +196,9 @@ bool HttpCommTask::processRead() {
     if (ptr < end) {
       _readPosition = ptr - _readBuffer->c_str() + 4;
 
-      LOG(TRACE) << "HTTP READ FOR " << (void*)this << ": " << std::string(_readBuffer->c_str() + _startPosition,
+      LOG(TRACE) << "HTTP READ FOR " << (void*)this << ": "
-                            _readPosition - _startPosition);
+                 << std::string(_readBuffer->c_str() + _startPosition,
+                                _readPosition - _startPosition);
 
       // check that we know, how to serve this request and update the connection
       // information, i. e. client and server addresses and ports and create a
@@ -250,7 +258,8 @@ bool HttpCommTask::processRead() {
       // and ports
       _request->setProtocol(_server->protocol());
 
-      LOG(TRACE) << "server port " << _connectionInfo.serverPort << ", client port " << _connectionInfo.clientPort;
+      LOG(TRACE) << "server port " << _connectionInfo.serverPort
+                 << ", client port " << _connectionInfo.clientPort;
 
       // set body start to current position
       _bodyPosition = _readPosition;
@@ -314,7 +323,9 @@ bool HttpCommTask::processRead() {
             l = 6;
           }
 
-          LOG(WARN) << "got corrupted HTTP request '" << std::string(_readBuffer->c_str() + _startPosition, l) << "'";
+          LOG(WARN) << "got corrupted HTTP request '"
+                    << std::string(_readBuffer->c_str() + _startPosition, l)
+                    << "'";
 
           // bad request, method not allowed
           HttpResponse response(HttpResponse::METHOD_NOT_ALLOWED,
@@ -399,7 +410,7 @@ bool HttpCommTask::processRead() {
     _request->setBody(_readBuffer->c_str() + _bodyPosition, _bodyLength);
 
     LOG(TRACE) << "" << std::string(_readBuffer->c_str() + _bodyPosition,
-                                _bodyLength);
+                                    _bodyLength);
 
     // remove body from read buffer and reset read position
     _readRequestBody = false;
@@ -643,7 +654,12 @@ void HttpCommTask::addResponse(HttpResponse* response) {
   _writeBuffers.push_back(buffer.get());
   auto b = buffer.release();
 
-  LOG(TRACE) << "HTTP WRITE FOR " << (void*)this << ": " << b->c_str();
+  if (!b->empty()) {
+    LOG_TOPIC(TRACE, Logger::REQUESTS)
+        << "\"http-request-response\",\"" << (void*)this << "\",\""
+        << (StringUtils::escapeUnicode(std::string(b->c_str(), b->length())))
+        << "\"";
+  }
 
   // clear body
   response->body().clear();
@@ -652,15 +668,14 @@ void HttpCommTask::addResponse(HttpResponse* response) {
 
   _writeBuffersStats.push_back(RequestStatisticsAgent::transfer());
 
-  LOG_TOPIC(INFO, Logger::REQUESTS) 
+  LOG_TOPIC(INFO, Logger::REQUESTS)
-      << "\"http-request\",\"" << _connectionInfo.clientAddress 
+      << "\"http-request-end\",\"" << (void*)this << "\",\""
-      << "\",\"" << HttpRequest::translateMethod(_requestType) << "\",\""
+      << _connectionInfo.clientAddress << "\",\""
-      << HttpRequest::translateVersion(_httpVersion) << "\"," 
+      << HttpRequest::translateMethod(_requestType) << "\",\""
-      << static_cast<int>(response->responseCode()) << "," 
+      << HttpRequest::translateVersion(_httpVersion) << "\","
-      << _originalBodyLength << "," 
+      << static_cast<int>(response->responseCode()) << ","
-      << responseBodyLength << ",\"" 
+      << _originalBodyLength << "," << responseBodyLength << ",\"" << _fullUrl
-      << _fullUrl << "\","
+      << "\"," << Logger::DURATION(totalTime, 6);
-      << Logger::DURATION(totalTime, 6);
 
   // start output
   fillWriteBuffer();
@@ -687,11 +702,13 @@ bool HttpCommTask::checkContentLength(bool expectContentLength) {
     // content-length header was sent but the request method does not support
     // that
     // we'll warn but read the body anyway
-    LOG(WARN) << "received HTTP GET/HEAD request with content-length, this should not happen";
+    LOG(WARN) << "received HTTP GET/HEAD request with content-length, this "
+                 "should not happen";
   }
 
   if ((size_t)bodyLength > MaximalBodySize) {
-    LOG(WARN) << "maximal body size is " << MaximalBodySize << ", request body size is " << bodyLength;
+    LOG(WARN) << "maximal body size is " << MaximalBodySize
+              << ", request body size is " << bodyLength;
 
     // request entity too large
     HttpResponse response(HttpResponse::REQUEST_ENTITY_TOO_LARGE,
@@ -762,7 +779,8 @@ void HttpCommTask::processCorsOptions(uint32_t compatibility) {
       // server. that's a client problem.
       response.setHeader(TRI_CHAR_LENGTH_PAIR("access-control-allow-headers"),
                          allowHeaders);
-      LOG(TRACE) << "client requested validation of the following headers: " << allowHeaders;
+      LOG(TRACE) << "client requested validation of the following headers: "
+                 << allowHeaders;
     }
 
     // set caching time (hard-coded value)
@@ -780,8 +798,7 @@ void HttpCommTask::processCorsOptions(uint32_t compatibility) {
 void HttpCommTask::processRequest(uint32_t compatibility) {
   // check for deflate
   bool found;
-  std::string const acceptEncoding =
+  std::string const acceptEncoding = _request->header("accept-encoding", found);
-      _request->header("accept-encoding", found);
 
   if (found) {
     if (acceptEncoding.find("deflate") != std::string::npos) {
@@ -789,6 +806,13 @@ void HttpCommTask::processRequest(uint32_t compatibility) {
     }
   }
 
+  LOG_TOPIC(INFO, Logger::REQUESTS)
+      << "\"http-request-begin\",\"" << (void*)this << "\",\""
+      << _connectionInfo.clientAddress << "\",\""
+      << HttpRequest::translateMethod(_requestType) << "\",\""
+      << HttpRequest::translateVersion(_httpVersion) << "\"," << _fullUrl
+      << "\"";
+
   // check for an async request
   std::string const asyncExecution = _request->header("x-arango-async", found);
 
@@ -807,6 +831,25 @@ void HttpCommTask::processRequest(uint32_t compatibility) {
     return;
   }
 
+  if (_request != nullptr) {
+    char const* body = "";
+    size_t bodySize = 0;
+
+    if (_request != nullptr) {
+      bodySize = _request->bodySize();
+
+      if (bodySize != 0) {
+        body = _request->body();
+      }
+    }
+
+    if (bodySize != 0) {
+      LOG_TOPIC(DEBUG, Logger::REQUESTS)
+          << "\"http-request-body\",\"" << (void*)this << "\",\""
+          << (StringUtils::escapeUnicode(std::string(body, bodySize))) << "\"";
+    }
+  }
+
   handler->setTaskId(_taskId, _loop);
 
   // clear request object

arangod/HttpServer/HttpHandler.cpp

@@ -24,7 +24,7 @@
 #include "HttpHandler.h"
 
 #include "Basics/StringUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Dispatcher/Dispatcher.h"
 #include "Rest/HttpRequest.h"
 

arangod/HttpServer/HttpHandlerFactory.cpp

@@ -23,7 +23,7 @@
 
 #include "HttpHandlerFactory.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "HttpServer/HttpHandler.h"
 #include "Rest/HttpRequest.h"
 

arangod/HttpServer/HttpServer.cpp

@@ -26,7 +26,7 @@
 
 #include "Basics/MutexLocker.h"
 #include "Basics/WorkMonitor.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Dispatcher/Dispatcher.h"
 #include "HttpServer/AsyncJobManager.h"
 #include "HttpServer/HttpCommTask.h"

arangod/HttpServer/HttpServerJob.cpp

@@ -25,7 +25,7 @@
 #include "HttpServerJob.h"
 
 #include "Basics/WorkMonitor.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Dispatcher/DispatcherQueue.h"
 #include "HttpServer/AsyncJobManager.h"
 #include "HttpServer/HttpCommTask.h"

arangod/HttpServer/HttpsCommTask.cpp

@@ -26,7 +26,7 @@
 
 #include <openssl/err.h>
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/socket-utils.h"
 #include "Basics/ssl-helper.h"
 #include "Basics/StringBuffer.h"
@@ -85,7 +85,7 @@ bool HttpsCommTask::setup(Scheduler* scheduler, EventLoop loop) {
 
   if (_ssl == nullptr) {
     LOG(DEBUG) << "cannot build new SSL connection: "
-               << arangodb::basics::lastSSLError();
+               << lastSSLError();
 
     shutdownSsl(false);
     return false;  // terminate ourselves, ssl is nullptr
@@ -202,7 +202,7 @@ bool HttpsCommTask::trySSLAccept() {
   // shutdown of connection
   if (res == 0) {
     LOG(DEBUG) << "SSL_accept failed: "
-               << arangodb::basics::lastSSLError();
+               << lastSSLError();
 
     shutdownSsl(false);
     return false;
@@ -222,7 +222,7 @@ bool HttpsCommTask::trySSLAccept() {
   }
 
   LOG(TRACE) << "error in SSL handshake: "
-             << arangodb::basics::lastSSLError();
+             << lastSSLError();
 
   shutdownSsl(false);
   return false;
@@ -249,7 +249,7 @@ again:
       case SSL_ERROR_SSL:
         LOG(DEBUG) << "received SSL error (bytes read " << nr << ", socket "
                    << TRI_get_fd_or_handle_of_socket(_commSocket)
-                   << "): " << arangodb::basics::lastSSLError();
+                   << "): " << lastSSLError();
 
         shutdownSsl(false);
         return false;
@@ -278,7 +278,7 @@ again:
       case SSL_ERROR_SYSCALL:
         if (res != 0) {
           LOG(DEBUG) << "SSL_read returned syscall error with: "
-                     << arangodb::basics::lastSSLError();
+                     << lastSSLError();
         } else if (nr == 0) {
           LOG(DEBUG)
               << "SSL_read returned syscall error because an EOF was received";
@@ -292,7 +292,7 @@ again:
 
       default:
         LOG(DEBUG) << "received error with " << res << " and " << nr << ": "
-                   << arangodb::basics::lastSSLError();
+                   << lastSSLError();
 
         shutdownSsl(false);
         return false;
@@ -363,7 +363,7 @@ bool HttpsCommTask::trySSLWrite() {
         case SSL_ERROR_SYSCALL:
           if (res != 0) {
             LOG(DEBUG) << "SSL_write returned syscall error with: "
-                       << arangodb::basics::lastSSLError();
+                       << lastSSLError();
           } else if (nr == 0) {
             LOG(DEBUG) << "SSL_write returned syscall error because an EOF was "
                           "received";
@@ -377,7 +377,7 @@ bool HttpsCommTask::trySSLWrite() {
 
         default:
           LOG(DEBUG) << "received error with " << res << " and " << nr << ": "
-                     << arangodb::basics::lastSSLError();
+                     << lastSSLError();
 
           shutdownSsl(false);
           return false;
@@ -437,7 +437,7 @@ void HttpsCommTask::shutdownSsl(bool initShutdown) {
 
           if (err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) {
             LOG(DEBUG) << "received shutdown error with " << res << ", " << err
-                       << ": " << arangodb::basics::lastSSLError();
+                       << ": " << lastSSLError();
             break;
           }
         }

arangod/HttpServer/PathHandler.cpp

@@ -24,7 +24,7 @@
 #include "PathHandler.h"
 
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/mimetypes.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/StringUtils.h"

arangod/Indexes/FulltextIndex.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "FulltextIndex.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/Utf8Helper.h"
 #include "FulltextIndex/fulltext-index.h"
 #include "VocBase/document-collection.h"

arangod/Indexes/GeoIndex2.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "GeoIndex2.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "VocBase/document-collection.h"
 #include "VocBase/transaction.h"
 

arangod/Indexes/PathBasedIndex.cpp

@@ -23,7 +23,7 @@
 
 #include "PathBasedIndex.h"
 #include "Aql/AstNode.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 
 #include <velocypack/Iterator.h>
 #include <velocypack/velocypack-aliases.h>

arangod/Replication/ContinuousSyncer.cpp

@@ -22,12 +22,13 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "ContinuousSyncer.h"
-
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
+#include "Basics/JsonHelper.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
+#include "Logger/Logger.h"
 #include "Replication/InitialSyncer.h"
 #include "Rest/HttpRequest.h"
 #include "SimpleHttpClient/SimpleHttpClient.h"

arangod/Replication/InitialSyncer.cpp

@@ -23,12 +23,14 @@
 
 #include "InitialSyncer.h"
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
+#include "Basics/JsonHelper.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Indexes/Index.h"
 #include "Indexes/PrimaryIndex.h"
+#include "Logger/Logger.h"
 #include "SimpleHttpClient/SimpleHttpClient.h"
 #include "SimpleHttpClient/SimpleHttpResult.h"
 #include "Utils/CollectionGuard.h"

arangod/Replication/InitialSyncer.h

@@ -25,7 +25,7 @@
 #define ARANGOD_REPLICATION_INITIAL_SYNCER_H 1
 
 #include "Basics/Common.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Replication/Syncer.h"
 #include "Utils/SingleCollectionTransaction.h"
 

arangod/RestHandler/RestAdminLogHandler.cpp

@@ -26,7 +26,7 @@
 #include <velocypack/Builder.h>
 #include <velocypack/velocypack-aliases.h>
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringUtils.h"
 #include "Rest/HttpRequest.h"
 

arangod/RestHandler/RestBatchHandler.cpp

@@ -24,7 +24,7 @@
 #include "RestBatchHandler.h"
 
 #include "Basics/StringUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "HttpServer/HttpHandlerFactory.h"
 #include "HttpServer/HttpServer.h"
 #include "Rest/HttpRequest.h"

arangod/RestHandler/RestImportHandler.cpp

@@ -22,10 +22,11 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "RestImportHandler.h"
-#include "Basics/Logger.h"
+#include "Basics/json-utilities.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/ServerState.h"
+#include "Logger/Logger.h"
 #include "Rest/HttpRequest.h"
 #include "Utils/OperationOptions.h"
 #include "Utils/SingleCollectionTransaction.h"

arangod/RestHandler/RestReplicationHandler.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "RestReplicationHandler.h"
-#include "Basics/Logger.h"
+#include "Basics/JsonHelper.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/conversions.h"
@@ -33,6 +33,7 @@
 #include "Indexes/EdgeIndex.h"
 #include "Indexes/Index.h"
 #include "Indexes/PrimaryIndex.h"
+#include "Logger/Logger.h"
 #include "Replication/InitialSyncer.h"
 #include "Rest/HttpRequest.h"
 #include "Rest/Version.h"

arangod/RestHandler/RestUploadHandler.cpp

@@ -24,7 +24,7 @@
 #include "RestUploadHandler.h"
 #include "Basics/FileUtils.h"
 #include "Basics/files.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringUtils.h"
 #include "Basics/tri-strings.h"
 #include "HttpServer/HttpServer.h"

arangod/RestServer/ArangoServer.cpp

@@ -42,7 +42,7 @@
 #include "Aql/QueryCache.h"
 #include "Aql/RestAqlHandler.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/Nonce.h"
 #include "Basics/ProgramOptions.h"
 #include "Basics/ProgramOptionsDescription.h"
@@ -991,33 +991,12 @@ void ArangoServer::buildApplicationServer() {
 
   _applicationServer->addFeature(_applicationV8);
 
-  // .............................................................................
-  // MRuby engine (this has been removed from arangod in version 2.2)
-  // .............................................................................
-
-  std::string ignoreOpt;
-  std::map<std::string, ProgramOptionsDescription> additional;
-
-  additional["Hidden Options"](
-      "ruby.gc-interval", &ignoreOpt,
-      "Ruby garbage collection interval (each x requests)")(
-      "ruby.action-directory", &ignoreOpt, "path to the Ruby action directory")(
-      "ruby.modules-path", &ignoreOpt,
-      "one or more directories separated by (semi-) colons")(
-      "ruby.startup-directory", &ignoreOpt,
-      "path to the directory containing alternate Ruby startup scripts")(
-      "server.disable-replication-logger", &ignoreOpt,
-      "start with replication logger turned off")(
-      "database.force-sync-shapes", &ignoreOpt,
-      "force syncing of shape data to disk, will use waitForSync value of "
-      "collection when turned off (deprecated)")(
-      "database.remove-on-drop", &ignoreOpt,
-      "wipe a collection from disk after dropping");
-
   // .............................................................................
   // define server options
   // .............................................................................
 
+  std::map<std::string, ProgramOptionsDescription> additional;
+
   // command-line only options
   additional["General Options:help-default"](
       "console",

arangod/RestServer/ConsoleThread.cpp

@@ -26,9 +26,9 @@
 #include <iostream>
 
 #include "ApplicationServer/ApplicationServer.h"
-#include "Basics/Logger.h"
 #include "Basics/tri-strings.h"
 #include "Basics/MutexLocker.h"
+#include "Logger/Logger.h"
 #include "Rest/Version.h"
 #include "VocBase/vocbase.h"
 #include "V8/V8LineEditor.h"

arangod/RestServer/VocbaseContext.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "VocbaseContext.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/tri-strings.h"
 #include "Cluster/ServerState.h"

arangod/RestServer/WindowsServiceUtils.cpp

@@ -26,7 +26,7 @@
 #include "Basics/Common.h"
 #include "Basics/files.h"
 #include "Basics/messages.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/tri-strings.h"
 #include "Rest/InitializeRest.h"
 #include "RestServer/ArangoServer.h"

arangod/RestServer/arangod.cpp

@@ -93,7 +93,7 @@ int main(int argc, char* argv[]) {
 
   // initialize sub-systems
   TRI_GlobalEntryFunction();
-  TRIAGENS_REST_INITIALIZE(argc, argv);
+  TRIAGENS_REST_INITIALIZE();
 
   arangodb::basics::VelocyPackHelper::initialize();
 

arangod/Scheduler/ApplicationScheduler.cpp

@@ -26,8 +26,8 @@
 #include <iostream>
 
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
 #include "Basics/process-utils.h"
+#include "Logger/Logger.h"
 #include "Scheduler/PeriodicTask.h"
 #include "Scheduler/SchedulerLibev.h"
 #include "Scheduler/SignalTask.h"

arangod/Scheduler/ListenTask.cpp

@@ -42,7 +42,7 @@
 
 #include <sys/types.h>
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/socket-utils.h"
 #include "Basics/StringUtils.h"

arangod/Scheduler/Scheduler.cpp

@@ -31,7 +31,9 @@
 #include "Basics/MutexLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/Thread.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
+#include "Basics/JsonHelper.h"
+#include "Logger/Logger.h"
 #include "Scheduler/SchedulerThread.h"
 #include "Scheduler/Task.h"
 

arangod/Scheduler/SchedulerLibev.cpp

@@ -31,7 +31,7 @@
 #include <ev.h>
 
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Scheduler/SchedulerThread.h"
 #include "Scheduler/Task.h"
 

arangod/Scheduler/SchedulerThread.cpp

@@ -24,7 +24,7 @@
 
 #include "SchedulerThread.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 
 #ifdef _WIN32

arangod/Scheduler/SchedulerThread.h

@@ -139,12 +139,6 @@ class SchedulerThread : public Thread, private TaskManager {
 
   std::deque<Work> _queue;
 
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief open for business
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::atomic<bool> _open;
-
   //////////////////////////////////////////////////////////////////////////////
   /// @brief work indicator
   //////////////////////////////////////////////////////////////////////////////

arangod/Scheduler/SignalTask.cpp

@@ -25,7 +25,7 @@
 #include "SignalTask.h"
 
 #include "Basics/MutexLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 
 #include "Scheduler/Scheduler.h"
 

arangod/Scheduler/SocketTask.cpp

@@ -24,7 +24,7 @@
 
 #include "SocketTask.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/socket-utils.h"

arangod/Scheduler/TimerTask.cpp

@@ -23,7 +23,8 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "TimerTask.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
+#include "Logger/Logger.h"
 #include "Scheduler/Scheduler.h"
 
 #include <velocypack/Builder.h>

arangod/Statistics/statistics.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "statistics.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/Mutex.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/threads.h"

arangod/Utils/CollectionKeysRepository.cpp

@@ -23,7 +23,7 @@
 
 #include "CollectionKeysRepository.h"
 #include "Basics/json.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "VocBase/server.h"
 #include "VocBase/vocbase.h"

arangod/Utils/CursorRepository.cpp

@@ -22,8 +22,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "CursorRepository.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
 #include "Basics/MutexLocker.h"
+#include "Logger/Logger.h"
 #include "Utils/CollectionExport.h"
 #include "VocBase/server.h"
 #include "VocBase/vocbase.h"

arangod/Utils/WorkMonitorArangod.cpp

@@ -28,7 +28,7 @@
 #include <velocypack/velocypack-aliases.h>
 
 #include "Aql/QueryList.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringBuffer.h"
 #include "HttpServer/HttpHandler.h"
 #include "Scheduler/Scheduler.h"

arangod/V8Server/ApplicationV8.cpp

@@ -28,7 +28,6 @@
 #include "Aql/QueryRegistry.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
 #include "Basics/Mutex.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/random.h"
@@ -42,6 +41,7 @@
 #include "Cluster/v8-cluster.h"
 #include "Dispatcher/ApplicationDispatcher.h"
 #include "Dispatcher/DispatcherThread.h"
+#include "Logger/Logger.h"
 #include "Rest/HttpRequest.h"
 #include "Scheduler/ApplicationScheduler.h"
 #include "Scheduler/Scheduler.h"

arangod/V8Server/V8Job.cpp

@@ -22,8 +22,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "V8Job.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
 #include "Dispatcher/DispatcherQueue.h"
+#include "Logger/Logger.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
 #include "V8Server/ApplicationV8.h"

arangod/V8Server/V8QueueJob.cpp

@@ -22,8 +22,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "V8QueueJob.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
 #include "Dispatcher/DispatcherQueue.h"
+#include "Logger/Logger.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
 #include "V8Server/ApplicationV8.h"

arangod/V8Server/V8VPackWrapper.cpp

@@ -23,7 +23,7 @@
 
 #include "V8VPackWrapper.h"
 #include "Basics/conversions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/Transaction.h"
 #include "V8/v8-conv.h"

arangod/V8Server/v8-actions.cpp

@@ -29,10 +29,11 @@
 #include "Basics/WriteLocker.h"
 #include "Basics/conversions.h"
 #include "Basics/files.h"
-#include "Basics/Logger.h"
+#include "Basics/json.h"
 #include "Basics/tri-strings.h"
 #include "Cluster/ClusterComm.h"
 #include "Cluster/ServerState.h"
+#include "Logger/Logger.h"
 #include "HttpServer/HttpServer.h"
 #include "Rest/HttpRequest.h"
 #include "Rest/HttpResponse.h"

arangod/V8Server/v8-dispatcher.cpp

@@ -23,7 +23,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "v8-dispatcher.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/tri-strings.h"
 #include "Basics/StringUtils.h"
 #include "Dispatcher/ApplicationDispatcher.h"

arangod/V8Server/v8-shape-conv.cpp

@@ -24,7 +24,7 @@
 #include "v8-shape-conv.h"
 
 #include "Basics/conversions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/StringUtils.h"
 #include "Basics/tri-strings.h"

arangod/V8Server/v8-wrapshapedjson.cpp

@@ -23,7 +23,7 @@
 
 #include "v8-wrapshapedjson.h"
 #include "Basics/conversions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/SingleCollectionTransaction.h"
 #include "Utils/V8TransactionContext.h"

arangod/VocBase/DatafileStatistics.cpp

@@ -23,7 +23,7 @@
 
 #include "DatafileStatistics.h"
 #include "Basics/Exceptions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/WriteLocker.h"
 #include "VocBase/datafile.h"

arangod/VocBase/Ditch.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "Ditch.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "VocBase/document-collection.h"
 

arangod/VocBase/MasterPointers.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "MasterPointers.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "VocBase/MasterPointer.h"
 
 using namespace arangodb;

arangod/VocBase/VocShaper.cpp

@@ -32,7 +32,7 @@
 #include "Basics/WriteLocker.h"
 #include "Basics/associative.h"
 #include "Basics/hashes.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/tri-strings.h"
 #include "Basics/Utf8Helper.h"
 #include "VocBase/document-collection.h"

arangod/VocBase/auth.cpp

@@ -22,11 +22,11 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "auth.h"
-#include "Basics/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/tri-strings.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
+#include "Logger/Logger.h"
 #include "Rest/SslInterface.h"
 #include "Utils/SingleCollectionTransaction.h"
 #include "Utils/StandaloneTransactionContext.h"

arangod/VocBase/cleanup.cpp

@@ -23,8 +23,9 @@
 
 #include "cleanup.h"
 #include "Basics/files.h"
-#include "Basics/Logger.h"
 #include "Basics/ReadLocker.h"
+#include "Basics/tri-strings.h"
+#include "Logger/Logger.h"
 #include "Utils/CursorRepository.h"
 #include "VocBase/compactor.h"
 #include "VocBase/Ditch.h"

arangod/VocBase/collection.cpp

@@ -25,7 +25,7 @@
 
 #include "Basics/FileUtils.h"
 #include "Basics/JsonHelper.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/files.h"

arangod/VocBase/compactor.cpp

@@ -30,10 +30,10 @@
 #include "Basics/files.h"
 #include "Basics/FileUtils.h"
 #include "Basics/memory-map.h"
-#include "Basics/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/tri-strings.h"
 #include "Basics/WriteLocker.h"
+#include "Logger/Logger.h"
 #include "Indexes/PrimaryIndex.h"
 #include "Utils/SingleCollectionTransaction.h"
 #include "Utils/StandaloneTransactionContext.h"

arangod/VocBase/datafile.cpp

@@ -22,14 +22,13 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "datafile.h"
-
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
 #include "Basics/StringUtils.h"
 #include "Basics/files.h"
 #include "Basics/hashes.h"
 #include "Basics/memory-map.h"
 #include "Basics/tri-strings.h"
+#include "Logger/Logger.h"
 #include "VocBase/DatafileHelper.h"
 #include "VocBase/server.h"
 

arangod/VocBase/document-collection.cpp

@@ -28,7 +28,7 @@
 #include "Basics/Exceptions.h"
 #include "Basics/FileUtils.h"
 #include "Basics/files.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/tri-strings.h"
 #include "Basics/ThreadPool.h"
 #include "Basics/VelocyPackHelper.h"

arangod/VocBase/edge-collection.cpp

@@ -23,7 +23,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "edge-collection.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Indexes/EdgeIndex.h"
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/Transaction.h"

arangod/VocBase/replication-applier.cpp

@@ -23,7 +23,6 @@
 
 #include "replication-applier.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/ScopeGuard.h"
 #include "Basics/StringUtils.h"
@@ -34,6 +33,7 @@
 #include "Basics/files.h"
 #include "Basics/json.h"
 #include "Basics/tri-strings.h"
+#include "Logger/Logger.h"
 #include "Replication/ContinuousSyncer.h"
 #include "Rest/Version.h"
 #include "VocBase/collection.h"

arangod/VocBase/replication-dump.cpp

@@ -22,9 +22,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "replication-dump.h"
-#include "Basics/Logger.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/VPackStringBufferAdapter.h"
+#include "Logger/Logger.h"
 #include "VocBase/collection.h"
 #include "VocBase/DatafileHelper.h"
 #include "VocBase/datafile.h"

arangod/VocBase/server.cpp

@@ -31,7 +31,6 @@
 #include "Aql/QueryRegistry.h"
 #include "Basics/Exceptions.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/conversions.h"
@@ -42,6 +41,7 @@
 #include "Basics/tri-strings.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/ServerState.h"
+#include "Logger/Logger.h"
 #include "Utils/CursorRepository.h"
 #include "VocBase/auth.h"
 #include "VocBase/replication-applier.h"

arangod/VocBase/shape-accessor.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "shape-accessor.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/vector.h"
 #include "VocBase/shaped-json.h"
 #include "VocBase/VocShaper.h"

arangod/VocBase/shaped-json.cpp

@@ -24,7 +24,7 @@
 #include "shaped-json.h"
 
 #include "Basics/hashes.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/tri-strings.h"
 #include "Basics/vector.h"

arangod/VocBase/transaction.cpp

@@ -25,7 +25,7 @@
 
 #include "Aql/QueryCache.h"
 #include "Basics/conversions.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/tri-strings.h"
 #include "Basics/Exceptions.h"
 #include "VocBase/DatafileHelper.h"

arangod/VocBase/vocbase.cpp

@@ -28,7 +28,7 @@
 #include "Basics/files.h"
 #include "Basics/hashes.h"
 #include "Basics/locks.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/memory-map.h"
 #include "Basics/random.h"
 #include "Basics/tri-strings.h"

arangod/Wal/AllocatorThread.cpp

@@ -22,7 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "AllocatorThread.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/Exceptions.h"
 #include "Wal/LogfileManager.h"

arangod/Wal/CollectorThread.cpp

@@ -25,7 +25,7 @@
 #include "Basics/ConditionLocker.h"
 #include "Basics/Exceptions.h"
 #include "Basics/hashes.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/memory-map.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/ReadLocker.h"

arangod/Wal/Logfile.h

@@ -25,7 +25,7 @@
 #define ARANGOD_WAL_LOGFILE_H 1
 
 #include "Basics/Common.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "VocBase/datafile.h"
 #include "VocBase/DatafileHelper.h"
 #include "VocBase/voc-types.h"

arangod/Wal/LogfileManager.cpp

@@ -24,7 +24,7 @@
 #include "LogfileManager.h"
 #include "Basics/Exceptions.h"
 #include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/StringUtils.h"

arangod/Wal/RemoverThread.cpp

@@ -23,7 +23,7 @@
 
 #include "RemoverThread.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/Exceptions.h"
 #include "Wal/LogfileManager.h"

arangod/Wal/Slots.cpp

@@ -24,7 +24,7 @@
 #include "Slots.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/MutexLocker.h"
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "VocBase/DatafileHelper.h"
 #include "VocBase/datafile.h"
 #include "VocBase/server.h"

arangod/Wal/SynchronizerThread.cpp

@@ -23,7 +23,7 @@
 
 #include "SynchronizerThread.h"
 
-#include "Basics/Logger.h"
+#include "Logger/Logger.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/Exceptions.h"
 #include "VocBase/server.h"

arangosh/ArangoShell/ArangoClient.cpp

@@ -1,979 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-/// DISCLAIMER
-///
-/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
-/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
-///
-/// Licensed under the Apache License, Version 2.0 (the "License");
-/// you may not use this file except in compliance with the License.
-/// You may obtain a copy of the License at
-///
-///     http://www.apache.org/licenses/LICENSE-2.0
-///
-/// Unless required by applicable law or agreed to in writing, software
-/// distributed under the License is distributed on an "AS IS" BASIS,
-/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-/// See the License for the specific language governing permissions and
-/// limitations under the License.
-///
-/// Copyright holder is ArangoDB GmbH, Cologne, Germany
-///
-/// @author Dr. Frank Celler
-////////////////////////////////////////////////////////////////////////////////
-
-#include "ArangoClient.h"
-
-#include "Basics/FileUtils.h"
-#include "Basics/Logger.h"
-#include "Basics/ProgramOptions.h"
-#include "Basics/ProgramOptionsDescription.h"
-#include "Basics/StringUtils.h"
-#include "Basics/files.h"
-#include "Basics/messages.h"
-#include "Basics/terminal-utils.h"
-#include "Basics/tri-strings.h"
-#include "Rest/Endpoint.h"
-
-#include <iostream>
-
-using namespace arangodb;
-using namespace arangodb::basics;
-using namespace arangodb::rest;
-
-double const ArangoClient::DEFAULT_CONNECTION_TIMEOUT = 5.0;
-double const ArangoClient::DEFAULT_REQUEST_TIMEOUT = 1200.0;
-size_t const ArangoClient::DEFAULT_RETRIES = 2;
-
-double const ArangoClient::LONG_TIMEOUT = 86400.0;
-#ifdef _WIN32
-bool cygwinShell = false;
-#endif
-namespace {
-#ifdef _WIN32
-bool _newLine() {
-  COORD pos;
-  CONSOLE_SCREEN_BUFFER_INFO bufferInfo;
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &bufferInfo);
-  if (bufferInfo.dwCursorPosition.Y + 1 >= bufferInfo.dwSize.Y) {
-    // when we are at the last visible line of the console
-    // the first line of console is deleted (the content of the console
-    // is scrolled one line above
-    SMALL_RECT srctScrollRect;
-    srctScrollRect.Top = 0;
-    srctScrollRect.Bottom = bufferInfo.dwCursorPosition.Y + 1;
-    srctScrollRect.Left = 0;
-    srctScrollRect.Right = bufferInfo.dwSize.X;
-    COORD coordDest;
-    coordDest.X = 0;
-    coordDest.Y = -1;
-    CONSOLE_SCREEN_BUFFER_INFO consoleScreenBufferInfo;
-    CHAR_INFO chiFill;
-    chiFill.Char.AsciiChar = (char)' ';
-    if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE),
-                                   &consoleScreenBufferInfo)) {
-      chiFill.Attributes = consoleScreenBufferInfo.wAttributes;
-    } else {
-      // Fill the bottom row with green blanks.
-      chiFill.Attributes = BACKGROUND_GREEN | FOREGROUND_RED;
-    }
-    ScrollConsoleScreenBuffer(GetStdHandle(STD_OUTPUT_HANDLE), &srctScrollRect,
-                              nullptr, coordDest, &chiFill);
-    pos.Y = bufferInfo.dwCursorPosition.Y;
-    pos.X = 0;
-    SetConsoleCursorPosition(GetStdHandle(STD_OUTPUT_HANDLE), pos);
-    return true;
-  } else {
-    pos.Y = bufferInfo.dwCursorPosition.Y + 1;
-    pos.X = 0;
-    SetConsoleCursorPosition(GetStdHandle(STD_OUTPUT_HANDLE), pos);
-    return false;
-  }
-}
-
-#endif
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief ignore sequence used for prompt length calculation (starting point)
-///
-/// This sequence must be used before any non-visible characters in the prompt.
-////////////////////////////////////////////////////////////////////////////////
-
-char const* ArangoClient::PROMPT_IGNORE_START = "\001";
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief ignore sequence used for prompt length calculation (end point)
-///
-/// This sequence must be used behind any non-visible characters in the prompt.
-////////////////////////////////////////////////////////////////////////////////
-
-char const* ArangoClient::PROMPT_IGNORE_END = "\002";
-
-ArangoClient::ArangoClient(char const* appName)
-    : _configFile(),
-      _tempPath(),
-      _quiet(false),
-
-      _colorOptions(false),
-      _noColors(false),
-
-      _autoCompleteOptions(false),
-      _noAutoComplete(false),
-
-      _prettyPrintOptions(false),
-      _prettyPrint(false),
-
-      _pagerOptions(false),
-      _outputPager("less -X -R -F -L"),
-      _pager(stdout),
-      _usePager(false),
-
-      _auditLog(""),
-      _auditFile(nullptr),
-
-      _serverOptions(false),
-      _disableAuthentication(false),
-      _endpointString(),
-      _endpointServer(nullptr),
-      _databaseName("_system"),
-      _username("root"),
-      _password(""),
-      _hasPassword(false),
-      _connectTimeout(DEFAULT_CONNECTION_TIMEOUT),
-      _requestTimeout(DEFAULT_REQUEST_TIMEOUT),
-      _sslProtocol(4) {
-  TRI_SetApplicationName(appName);
-  std::string p = TRI_GetTempPath();
-
-  _logLevel.push_back("info");
-
-  if (p.empty()) {
-    _tempPath = p;
-  }
-}
-
-ArangoClient::~ArangoClient() { delete _endpointServer; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up a program-specific help message
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupSpecificHelp(std::string const& progname,
-                                     std::string const& message) {
-  _specificHelp.progname = progname;
-  _specificHelp.message = message;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the general and logging options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupGeneral(ProgramOptionsDescription& description) {
-  ProgramOptionsDescription loggingOptions("LOGGING options");
-
-  // clang-format off
-
-  loggingOptions
-    ("log.level,l", &_logLevel, "log level")
-    ("log.use-local-time", "log local dates and times in log messages")
-    ("log.output,o", &_logOutput, "log output")
-  ;
-
-  description
-    ("configuration,c", &_configFile, "read configuration file")
-    ("help,h", "help message")
-    ("temp-path", &_tempPath, "path for temporary files")
-    ("quiet,s", "no banner")
-    (loggingOptions, false)
-  ;
-
-  // clang-format on
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the color options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupColors(ProgramOptionsDescription& description) {
-  ProgramOptionsDescription hiddenOptions("HIDDEN options");
-
-  // clang-format off
-
-  hiddenOptions
-    ("colors", "activate color support")
-  ;
-
-  description
-    ("no-colors", "deactivate color support")
-    (hiddenOptions, true)
-  ;
-
-  // clang-format on
-
-  _colorOptions = true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the auto-complete options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupAutoComplete(ProgramOptionsDescription& description) {
-  ProgramOptionsDescription hiddenOptions("HIDDEN options");
-
-  // clang-format off
-
-  hiddenOptions
-    ("auto-complete", "enable auto completion, use no-auto-complete to disable")
-  ;
-
-  description
-    ("no-auto-complete", "disable auto completion")
-    (hiddenOptions, true)
-  ;
-
-  // clang-format on
-
-  _autoCompleteOptions = true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the pretty-printing options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupPrettyPrint(ProgramOptionsDescription& description) {
-  ProgramOptionsDescription hiddenOptions("HIDDEN options");
-
-  // clang-format off
-
-  hiddenOptions
-    ("no-pretty-print", "disable pretty printting")
-  ;
-
-  description
-    ("pretty-print", "pretty print values")
-    (hiddenOptions, true)
-  ;
-
-  // clang-format on
-
-  _prettyPrintOptions = true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the log options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupLog(ProgramOptionsDescription& description) {
-  // clang-format off
-
-  description
-    ("audit-log", &_auditLog, "audit log file to save commands and results to")
-  ;
-
-  // clang-format on
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the pager options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupPager(ProgramOptionsDescription& description) {
-  // clang-format off
-
-  description
-    ("pager", &_outputPager, "output pager")
-    ("use-pager", "use pager")
-  ;
-
-  // clang-format on
-
-  _pagerOptions = true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets up the server options
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setupServer(ProgramOptionsDescription& description) {
-  ProgramOptionsDescription clientOptions("CLIENT options");
-
-  // clang-format off
-
-  clientOptions
-    ("server.database", &_databaseName, "database name to use when connecting")
-    ("server.disable-authentication", &_disableAuthentication,
-       "disable the password prompt and authentication when connecting (note: "
-       "this doesn't control whether a server requires authentication)")
-    ("server.endpoint", &_endpointString,
-       "endpoint to connect to, use 'none' to start without a server")
-    ("server.username", &_username, "username to use when connecting")
-    ("server.password", &_password,
-       "password to use when connecting. Don't specify this option to be "
-       "prompted for the password (note: this requires "
-       "--server.disable-authentication to be 'false')")
-    ("server.connect-timeout", &_connectTimeout, "connect timeout in seconds")
-    ("server.request-timeout", &_requestTimeout, "request timeout in seconds")
-    ("server.ssl-protocol", &_sslProtocol,
-       "1 = SSLv2, 2 = SSLv23, 3 = SSLv3, 4 = TLSv1")
-  ;
-
-  description
-    (clientOptions, false)
-  ;
-
-  // clang-format on
-
-  _serverOptions = true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief parses command line and config file and prepares logging
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::parse(ProgramOptions& options,
-                         ProgramOptionsDescription& description,
-                         std::string const& example, int argc, char* argv[],
-                         std::string const& initFilename) {
-  // if options are invalid, exit directly
-  if (!options.parse(description, argc, argv)) {
-    LOG(FATAL) << options.lastError();
-    FATAL_ERROR_EXIT();
-  }
-
-  // setup the logging
-  Logger::setLogLevel(_logLevel);
-  Logger::setUseLocalTime(options.has("log.use-local-time"));
-  Logger::setShowLineNumber(false);
-  Logger::setShowThreadIdentifier(false);
-
-  std::unordered_set<std::string> existingAppenders;
-  if (_logOutput.empty()) {
-    Logger::addAppender("-", true, "", existingAppenders);
-  } else {
-    for (auto& definition : _logOutput) {
-      Logger::addAppender(definition, true, "", existingAppenders);
-    }
-  }
-
-  // parse config file
-  std::string configFile = "";
-  bool allowLocal = false;
-
-  if (!_configFile.empty()) {
-    if (StringUtils::tolower(_configFile) == std::string("none")) {
-      LOG(DEBUG) << "using no init file at all";
-    } else {
-      configFile = _configFile;
-    }
-  }
-
-  else {
-    char* d = TRI_LocateConfigDirectory();
-
-    if (d != nullptr) {
-      std::string sysDir = std::string(d) + initFilename;
-      TRI_FreeString(TRI_CORE_MEM_ZONE, d);
-
-      if (FileUtils::exists(sysDir)) {
-        configFile = sysDir;
-        allowLocal = true;
-      } else {
-        LOG(DEBUG) << "no system init file '" << sysDir << "'";
-      }
-    }
-  }
-
-  if (!configFile.empty()) {
-    if (allowLocal) {
-      std::string localConfigFile = configFile + ".local";
-
-      if (FileUtils::exists(localConfigFile)) {
-        LOG(DEBUG) << "using init override file '" << localConfigFile
-                   << "'";
-
-        if (!options.parse(description, localConfigFile)) {
-          LOG(FATAL) << "cannot parse config file '" << localConfigFile
-                     << "': " << options.lastError();
-          FATAL_ERROR_EXIT();
-        }
-      }
-    }
-
-    LOG(DEBUG) << "using init file '" << configFile << "'";
-
-    if (!options.parse(description, configFile)) {
-      LOG(FATAL) << "cannot parse config file '" << configFile
-                 << "': " << options.lastError();
-      FATAL_ERROR_EXIT();
-    }
-  }
-
-  // configuration is parsed and valid if we got to this point
-
-  // check for --help
-  std::set<std::string> help = options.needHelp("help");
-
-  if (!help.empty()) {
-    if (!example.empty()) {
-      std::cout << "USAGE:  " << argv[0] << " " << example << std::endl
-                << std::endl;
-    }
-    std::cout << description.usage(help) << std::endl;
-
-    // check for program-specific help
-    std::string const progname(argv[0]);
-    if (!_specificHelp.progname.empty() &&
-        progname.size() >= _specificHelp.progname.size() &&
-        progname.substr(progname.size() - _specificHelp.progname.size(),
-                        _specificHelp.progname.size()) ==
-            _specificHelp.progname) {
-      // found a program-specific help
-      std::cout << _specificHelp.message << std::endl;
-    }
-
-    // --help always returns success
-    TRI_EXIT_FUNCTION(EXIT_SUCCESS, nullptr);
-  }
-
-  // set temp path
-  if (options.has("temp-path")) {
-    TRI_SetUserTempPath((char*)_tempPath.c_str());
-  }
-
-  if (options.has("server.username")) {
-    // if a username is specified explicitly, assume authentication is desired
-    _disableAuthentication = false;
-  }
-
-  // check if have a password
-  _hasPassword = options.has("server.password") || _disableAuthentication ||
-                 options.has("jslint") || options.has("javascript.unit-tests");
-
-  // set colors
-  if (options.has("colors")) {
-    _noColors = false;
-  }
-
-  if (options.has("no-colors")) {
-    _noColors = true;
-  }
-
-  // set auto-completion
-  if (options.has("auto-complete")) {
-    _noAutoComplete = false;
-  }
-
-  if (options.has("no-auto-complete")) {
-    _noAutoComplete = true;
-  }
-
-  // set pretty print
-  if (options.has("pretty-print")) {
-    _prettyPrint = true;
-  }
-
-  if (options.has("no-pretty-print")) {
-    _prettyPrint = false;
-  }
-
-  // set pager
-  if (options.has("use-pager")) {
-    _usePager = true;
-  }
-
-  // set quiet
-  if (options.has("quiet")) {
-    _quiet = true;
-  }
-
-  // .............................................................................
-  // server options
-  // .............................................................................
-
-  if (_serverOptions) {
-    // check connection args
-    if (_connectTimeout < 0.0) {
-      LOG(FATAL) << "invalid value for --server.connect-timeout, must be >= 0";
-      FATAL_ERROR_EXIT();
-    } else if (_connectTimeout == 0.0) {
-      _connectTimeout = LONG_TIMEOUT;
-    }
-
-    if (_requestTimeout < 0.0) {
-      LOG(FATAL)
-          << "invalid value for --server.request-timeout, must be positive";
-      FATAL_ERROR_EXIT();
-    } else if (_requestTimeout == 0.0) {
-      _requestTimeout = LONG_TIMEOUT;
-    }
-
-    // must specify a user name
-    if (_username.size() == 0) {
-      LOG(FATAL) << "no value specified for --server.username";
-      FATAL_ERROR_EXIT();
-    }
-
-    // no password given on command-line
-    if (!_hasPassword) {
-      usleep(10 * 1000);
-      printContinuous("Please specify a password: ");
-
-// now prompt for it
-#ifdef TRI_HAVE_TERMIOS_H
-      TRI_SetStdinVisibility(false);
-      getline(std::cin, _password);
-
-      TRI_SetStdinVisibility(true);
-#else
-      getline(std::cin, _password);
-#endif
-      printLine("");
-    }
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief prints a string and a newline to stderr
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::printErrLine(std::string const& s) {
-#ifdef _WIN32
-  // no, we can use std::cerr as this doesn't support UTF-8 on Windows
-  printLine(s);
-#else
-  fprintf(stderr, "%s\n", s.c_str());
-#endif
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief prints a string and a newline to stdout
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::_printLine(std::string const& s) {
-#ifdef _WIN32
-  LPWSTR wBuf = (LPWSTR)TRI_Allocate(TRI_CORE_MEM_ZONE,
-                                     (sizeof WCHAR) * (s.size() + 1), true);
-  int wLen = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, wBuf,
-                                 (int)((sizeof WCHAR) * (s.size() + 1)));
-
-  if (wLen) {
-    DWORD n;
-    COORD pos;
-    CONSOLE_SCREEN_BUFFER_INFO bufferInfo;
-    GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &bufferInfo);
-    // save old cursor position
-    pos = bufferInfo.dwCursorPosition;
-
-    size_t newX = static_cast<size_t>(pos.X) + s.size();
-    // size_t oldY = static_cast<size_t>(pos.Y);
-    if (newX >= static_cast<size_t>(bufferInfo.dwSize.X)) {
-      for (size_t i = 0; i <= newX / bufferInfo.dwSize.X; ++i) {
-        // insert as many newlines as we need. this prevents running out of
-        // buffer space when printing lines
-        // at the end of the console output buffer
-        if (_newLine()) {
-          pos.Y = pos.Y - 1;
-        }
-      }
-    }
-
-    // save new cursor position
-    GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &bufferInfo);
-    auto newPos = bufferInfo.dwCursorPosition;
-
-    // print the actual string. note: printing does not advance the cursor
-    // position
-    SetConsoleCursorPosition(GetStdHandle(STD_OUTPUT_HANDLE), pos);
-    WriteConsoleOutputCharacterW(GetStdHandle(STD_OUTPUT_HANDLE), wBuf,
-                                 (DWORD)s.size(), pos, &n);
-
-    // finally set the cursor position to where the printing should have stopped
-    SetConsoleCursorPosition(GetStdHandle(STD_OUTPUT_HANDLE), newPos);
-  } else {
-    fprintf(stdout, "window error: '%d' \r\n", GetLastError());
-    fprintf(stdout, "%s\r\n", s.c_str());
-  }
-
-  if (wBuf) {
-    TRI_Free(TRI_CORE_MEM_ZONE, wBuf);
-  }
-#endif
-}
-
-void ArangoClient::printLine(std::string const& s, bool forceNewLine) {
-#if _WIN32
-  if (!cygwinShell) {
-    // no, we cannot use std::cout as this doesn't support UTF-8 on Windows
-    // fprintf(stdout, "%s\r\n", s.c_str());
-    TRI_vector_string_t subStrings = TRI_SplitString(s.c_str(), '\n');
-    bool hasNewLines = (s.find("\n") != std::string::npos) | forceNewLine;
-    if (hasNewLines) {
-      for (size_t i = 0; i < subStrings._length; i++) {
-        _printLine(subStrings._buffer[i]);
-        _newLine();
-      }
-    } else {
-      _printLine(s);
-    }
-    TRI_DestroyVectorString(&subStrings);
-  } else
-#endif
-    fprintf(stdout, "%s\n", s.c_str());
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief prints a string to stdout, without a newline (Non-Windows only)
-/// on Windows, we'll print the line and a newline
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::printContinuous(std::string const& s) {
-// no, we cannot use std::cout as this doesn't support UTF-8 on Windows
-#ifdef _WIN32
-  // On Windows, we just print the line followed by a newline
-  printLine(s, true);
-#else
-  fprintf(stdout, "%s", s.c_str());
-  fflush(stdout);
-#endif
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief starts pager
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::startPager() {
-#ifndef _WIN32
-  if (!_usePager || _outputPager == "" || _outputPager == "stdout" ||
-      _outputPager == "-") {
-    _pager = stdout;
-    return;
-  }
-
-  _pager = popen(_outputPager.c_str(), "w");
-
-  if (_pager == nullptr) {
-    printf("popen() failed! Using stdout instead!\n");
-    _pager = stdout;
-    _usePager = false;
-  }
-#endif
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief stops pager
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::stopPager() {
-#ifndef _WIN32
-  if (_pager != stdout) {
-    pclose(_pager);
-    _pager = stdout;
-  }
-#endif
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief strips binary data from string
-/// this is done before sending the string to a pager or writing it to the log
-////////////////////////////////////////////////////////////////////////////////
-
-static std::string StripBinary(char const* value) {
-  std::string result;
-
-  char const* p = value;
-  bool inBinary = false;
-
-  while (*p) {
-    if (inBinary) {
-      if (*p == 'm') {
-        inBinary = false;
-      }
-    } else {
-      if (*p == '\x1b') {
-        inBinary = true;
-      } else {
-        result.push_back(*p);
-      }
-    }
-    ++p;
-  }
-
-  return result;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief prints to pager
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::internalPrint(std::string const& str) {
-  if (_pager == stdout) {
-#ifdef _WIN32
-    // at moment the formating is ignored in windows
-    printLine(str);
-#else
-    fprintf(_pager, "%s", str.c_str());
-#endif
-    if (_auditFile) {
-      std::string sanitized = StripBinary(str.c_str());
-      log("%s", sanitized.c_str());
-    }
-  } else {
-    std::string sanitized = StripBinary(str.c_str());
-
-    if (!sanitized.empty()) {
-      fprintf(_pager, "%s", sanitized.c_str());
-      log("%s", sanitized.c_str());
-    }
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief opens the log file
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::openLog() {
-  if (!_auditLog.empty()) {
-    _auditFile = fopen(_auditLog.c_str(), "w");
-
-    std::ostringstream s;
-    if (_auditFile == nullptr) {
-      s << "Cannot open file '" << _auditLog << "' for logging.";
-      printErrLine(s.str());
-    } else {
-      s << "Logging input and output to '" << _auditLog << "'.";
-      printLine(s.str());
-    }
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief closes the log file
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::closeLog() {
-  if (!_auditLog.empty() && _auditFile != nullptr) {
-    fclose(_auditFile);
-    _auditFile = nullptr;
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief prints info message
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::printWelcomeInfo() {
-  if (_usePager) {
-    std::ostringstream s;
-    s << "Using pager '" << _outputPager << "' for output buffering.";
-
-    printLine(s.str());
-  }
-
-  if (_prettyPrint) {
-    printLine("Pretty printing values.");
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief prints bye-bye
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::printByeBye() {
-  if (!_quiet) {
-    printLine("<ctrl-D>");
-    printLine(TRI_BYE_MESSAGE);
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief logs output, without prompt
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::log(char const* format, char const* str) {
-  if (_auditFile) {
-    std::string sanitized = StripBinary(str);
-
-    if (!sanitized.empty()) {
-      // do not print terminal escape sequences into log
-      fprintf(_auditFile, format, sanitized.c_str());
-    }
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief logs output, with prompt
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::log(std::string const& format, std::string const& prompt,
-                       std::string const& str) {
-  if (_auditFile) {
-    std::string sanitized = StripBinary(str.c_str());
-
-    if (!sanitized.empty()) {
-      // do not print terminal escape sequences into log
-      fprintf(_auditFile, format.c_str(), prompt.c_str(), sanitized.c_str());
-    }
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief flushes log output
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::flushLog() {
-  if (_auditFile) {
-    fflush(_auditFile);
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief creates a new endpoint
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::createEndpoint() { createEndpoint(_endpointString); }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief creates a new endpoint
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::createEndpoint(std::string const& definition) {
-  // close previous endpoint
-  if (_endpointServer != nullptr) {
-    delete _endpointServer;
-    _endpointServer = nullptr;
-  }
-
-  _endpointServer = Endpoint::clientFactory(definition);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief quiet start
-////////////////////////////////////////////////////////////////////////////////
-
-bool ArangoClient::quiet() const { return _quiet; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief shuts up arangosh
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::shutup() { _quiet = true; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief deactivates colors
-////////////////////////////////////////////////////////////////////////////////
-
-bool ArangoClient::colors() const {
-  return (!_noColors && isatty(STDIN_FILENO));
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief gets the auto completion flag
-////////////////////////////////////////////////////////////////////////////////
-
-bool ArangoClient::autoComplete() const { return !_noAutoComplete; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief use pretty print
-////////////////////////////////////////////////////////////////////////////////
-
-bool ArangoClient::prettyPrint() const { return _prettyPrint; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief gets the output pager
-////////////////////////////////////////////////////////////////////////////////
-
-std::string const& ArangoClient::outputPager() const { return _outputPager; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief gets use pager
-////////////////////////////////////////////////////////////////////////////////
-
-bool ArangoClient::usePager() const { return _usePager; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets use pager
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setUsePager(bool value) { _usePager = value; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief gets endpoint to connect to as string
-////////////////////////////////////////////////////////////////////////////////
-
-std::string const& ArangoClient::endpointString() const {
-  return _endpointString;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets endpoint to connect to as string
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setEndpointString(std::string const& value) {
-  _endpointString = value;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief endpoint
-////////////////////////////////////////////////////////////////////////////////
-
-Endpoint* ArangoClient::endpointServer() const { return _endpointServer; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief database name
-////////////////////////////////////////////////////////////////////////////////
-
-std::string const& ArangoClient::databaseName() const { return _databaseName; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief user to send to endpoint
-////////////////////////////////////////////////////////////////////////////////
-
-std::string const& ArangoClient::username() const { return _username; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief password to send to endpoint
-////////////////////////////////////////////////////////////////////////////////
-
-std::string const& ArangoClient::password() const { return _password; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets database name
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setDatabaseName(std::string const& databaseName) {
-  _databaseName = databaseName;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets username
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setUsername(std::string const& username) {
-  _username = username;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sets password
-////////////////////////////////////////////////////////////////////////////////
-
-void ArangoClient::setPassword(std::string const& password) {
-  _password = password;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief connect timeout (in seconds)
-////////////////////////////////////////////////////////////////////////////////
-
-double ArangoClient::connectTimeout() const { return _connectTimeout; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief request timeout (in seconds)
-////////////////////////////////////////////////////////////////////////////////
-
-double ArangoClient::requestTimeout() const { return _requestTimeout; }
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief ssl protocol
-////////////////////////////////////////////////////////////////////////////////
-
-uint32_t ArangoClient::sslProtocol() const { return _sslProtocol; }

arangosh/ArangoShell/ArangoClient.h

@@ -1,561 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-/// DISCLAIMER
-///
-/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
-/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
-///
-/// Licensed under the Apache License, Version 2.0 (the "License");
-/// you may not use this file except in compliance with the License.
-/// You may obtain a copy of the License at
-///
-///     http://www.apache.org/licenses/LICENSE-2.0
-///
-/// Unless required by applicable law or agreed to in writing, software
-/// distributed under the License is distributed on an "AS IS" BASIS,
-/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-/// See the License for the specific language governing permissions and
-/// limitations under the License.
-///
-/// Copyright holder is ArangoDB GmbH, Cologne, Germany
-///
-/// @author Dr. Frank Celler
-////////////////////////////////////////////////////////////////////////////////
-
-#ifndef ARANGOSH_ARANGO_SHELL_ARANGO_CLIENT_H
-#define ARANGOSH_ARANGO_SHELL_ARANGO_CLIENT_H 1
-
-#include "Basics/Common.h"
-
-namespace arangodb {
-namespace basics {
-class ProgramOptionsDescription;
-class ProgramOptions;
-}
-
-namespace rest {
-class Endpoint;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief arango shell client base
-////////////////////////////////////////////////////////////////////////////////
-
-class ArangoClient {
- private:
-  ArangoClient(const ArangoClient&);
-  ArangoClient operator=(const ArangoClient&);
-
- public:
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief default request timeout
-  //////////////////////////////////////////////////////////////////////////////
-
-  static double const DEFAULT_REQUEST_TIMEOUT;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief default number of retries
-  //////////////////////////////////////////////////////////////////////////////
-
-  static size_t const DEFAULT_RETRIES;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief default connection timeout
-  //////////////////////////////////////////////////////////////////////////////
-
-  static double const DEFAULT_CONNECTION_TIMEOUT;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief default "long" timeout
-  //////////////////////////////////////////////////////////////////////////////
-
-  static double const LONG_TIMEOUT;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief ignore sequence used for prompt length calculation (starting point)
-  ///
-  /// This sequence must be used before any non-visible characters in the
-  /// prompt.
-  ///
-  /// \\001 (^A) start non-visible characters
-  /// \\002 (^B) end non-visible characters
-  ///
-  /// All characters except \\001 and \\002 (following a \\001) are copied to
-  /// the returned string; all characters except those between \001 and
-  /// \\002 are assumed to be `visible'.
-  //////////////////////////////////////////////////////////////////////////////
-
-  static char const* PROMPT_IGNORE_START;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief ignore sequence used for prompt length calculation (end point)
-  ///
-  /// This sequence must be used behind any non-visible characters in the
-  /// prompt.
-  //////////////////////////////////////////////////////////////////////////////
-
-  static char const* PROMPT_IGNORE_END;
-
- public:
-  explicit ArangoClient(char const* appName);
-
-  ~ArangoClient();
-
- public:
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up a program-specific help message
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupSpecificHelp(std::string const&, std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the general and logging options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupGeneral(arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the color options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupColors(arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the auto-complete options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupAutoComplete(
-      arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the pretty-printing options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupPrettyPrint(
-      arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the pager options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupPager(arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the log options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupLog(arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets up the server options
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setupServer(arangodb::basics::ProgramOptionsDescription& description);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief parses command line and config file and prepares logging
-  //////////////////////////////////////////////////////////////////////////////
-
-  void parse(arangodb::basics::ProgramOptions&,
-             arangodb::basics::ProgramOptionsDescription& description,
-             std::string const& example, int argc, char* argv[],
-             std::string const& initFilename);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief print a string and a newline to stderr, necessary for Windows
-  //////////////////////////////////////////////////////////////////////////////
-
-  void printErrLine(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief print a string and a newline to stdout, necessary for Windows
-  //////////////////////////////////////////////////////////////////////////////
-
-  void printLine(std::string const&, bool forceNewLine = false);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief print a string to stdout, without a newline, necessary for Windows
-  //////////////////////////////////////////////////////////////////////////////
-
-  void printContinuous(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief starts pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  void startPager();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief stops pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  void stopPager();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief print to pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  void internalPrint(std::string const& str);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief open log
-  //////////////////////////////////////////////////////////////////////////////
-
-  void openLog();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief close log
-  //////////////////////////////////////////////////////////////////////////////
-
-  void closeLog();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief print info message
-  //////////////////////////////////////////////////////////////////////////////
-
-  void printWelcomeInfo();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief print bye-bye
-  //////////////////////////////////////////////////////////////////////////////
-
-  void printByeBye();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief log output, without prompt
-  //////////////////////////////////////////////////////////////////////////////
-
-  void log(char const*, char const*);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief log output, with prompt
-  //////////////////////////////////////////////////////////////////////////////
-
-  void log(std::string const& format, std::string const& prompt,
-           std::string const& str);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief flush log
-  //////////////////////////////////////////////////////////////////////////////
-
-  void flushLog();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief creates a new endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  void createEndpoint();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief creates a new endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  void createEndpoint(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief quiet start
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool quiet() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief shut up arangosh
-  //////////////////////////////////////////////////////////////////////////////
-
-  void shutup();
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief deactivate colors
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool colors() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief gets the auto completion flag
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool autoComplete() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use pretty print
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool prettyPrint() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief gets the output pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string const& outputPager() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief gets use pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool usePager() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets use pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setUsePager(bool);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief gets endpoint to connect to as string
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string const& endpointString() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief sets endpoint to connect to as string
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setEndpointString(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  arangodb::rest::Endpoint* endpointServer() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief database name
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string const& databaseName() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief set database name
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setDatabaseName(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief user to send to endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string const& username() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief password to send to endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string const& password() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief set username
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setUsername(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief set password
-  //////////////////////////////////////////////////////////////////////////////
-
-  void setPassword(std::string const&);
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief connect timeout (in seconds)
-  //////////////////////////////////////////////////////////////////////////////
-
-  double connectTimeout() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief request timeout (in seconds)
-  //////////////////////////////////////////////////////////////////////////////
-
-  double requestTimeout() const;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief ssl protocol
-  //////////////////////////////////////////////////////////////////////////////
-
-  uint32_t sslProtocol() const;
-
- private:
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief prints a line
-  //////////////////////////////////////////////////////////////////////////////
-
-  void _printLine(std::string const&);
-
- private:
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief configuration file
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _configFile;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief temporary path
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _tempPath;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief log level
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::vector<std::string> _logLevel;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief log output
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::vector<std::string> _logOutput;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief quiet start
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _quiet;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use color options
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _colorOptions;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief deactivate colors
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _noColors;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use auto-complete options
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _autoCompleteOptions;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief disable auto completion
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _noAutoComplete;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use pretty-print options
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _prettyPrintOptions;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use pretty print
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _prettyPrint;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use pager options
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _pagerOptions;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief the output pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _outputPager;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief the pager FILE
-  //////////////////////////////////////////////////////////////////////////////
-
-  FILE* _pager;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use pager
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _usePager;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief log filename
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _auditLog;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief the log FILE
-  //////////////////////////////////////////////////////////////////////////////
-
-  FILE* _auditFile;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief use server options
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _serverOptions;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief whether or not to turn off authentication
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _disableAuthentication;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief endpoint to connect to as string
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _endpointString;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  arangodb::rest::Endpoint* _endpointServer;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief database name
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _databaseName;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief user to send to endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _username;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief password to send to endpoint
-  //////////////////////////////////////////////////////////////////////////////
-
-  std::string _password;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief whether or not a password was specified on the command line
-  //////////////////////////////////////////////////////////////////////////////
-
-  bool _hasPassword;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief connect timeout (in seconds)
-  //////////////////////////////////////////////////////////////////////////////
-
-  double _connectTimeout;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief request timeout (in seconds)
-  //////////////////////////////////////////////////////////////////////////////
-
-  double _requestTimeout;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief ssl protocol
-  //////////////////////////////////////////////////////////////////////////////
-
-  uint32_t _sslProtocol;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief program-specific help message
-  //////////////////////////////////////////////////////////////////////////////
-
-  struct {
-    std::string progname;
-    std::string message;
-  } _specificHelp;
-};
-}
-
-#endif

arangosh/Benchmark/ArangobFeature.cpp

@@ -0,0 +1,328 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#include "ArangobFeature.h"
+
+#include <iostream>
+
+#include "ApplicationFeatures/ClientFeature.h"
+#include "Basics/StringUtils.h"
+#include "Basics/random.h"
+#include "Benchmark/BenchmarkCounter.h"
+#include "Benchmark/BenchmarkOperation.h"
+#include "Benchmark/BenchmarkThread.h"
+#include "ProgramOptions2/ProgramOptions.h"
+#include "ProgramOptions2/Section.h"
+#include "SimpleHttpClient/SimpleHttpClient.h"
+#include "SimpleHttpClient/SimpleHttpResult.h"
+
+using namespace arangodb;
+using namespace arangodb::arangob;
+using namespace arangodb::basics;
+using namespace arangodb::httpclient;
+using namespace arangodb::options;
+using namespace arangodb::rest;
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief includes all the test cases
+///
+/// We use an evil global pointer here.
+////////////////////////////////////////////////////////////////////////////////
+
+ArangobFeature* ARANGOB;
+#include "Benchmark/test-cases.h"
+
+ArangobFeature::ArangobFeature(application_features::ApplicationServer* server,
+                               int* result)
+    : ApplicationFeature(server, "ArangobFeature"),
+      _async(false),
+      _concurreny(1),
+      _operations(1000),
+      _batchSize(0),
+      _keepAlive(true),
+      _collection("ArangoBenchmark"),
+      _testCase("version"),
+      _complexity(1),
+      _delay(false),
+      _progress(true),
+      _verbose(false),
+      _quiet(false),
+      _result(result) {
+  requiresElevatedPrivileges(false);
+  setOptional(false);
+  startsAfter("ClientFeature");
+  startsAfter("ConfigFeature");
+  startsAfter("LoggerFeature");
+}
+
+void ArangobFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
+  LOG_TOPIC(TRACE, Logger::STARTUP) << name() << "::collectOptions";
+
+  options->addSection(
+      Section("", "Global configuration", "global options", false, false));
+
+  options->addOption("--async", "send asynchronous requests",
+                     new BooleanParameter(&_async));
+
+  options->addOption("--concurrency", "number of parallel connections",
+                     new UInt64Parameter(&_concurreny));
+
+  options->addOption("--requests", "total number of operations",
+                     new UInt64Parameter(&_operations));
+
+  options->addOption("--batch-size",
+                     "number of operations in one batch (0 disables batching)",
+                     new UInt64Parameter(&_batchSize));
+
+  options->addOption("--keep-alive", "use HTTP keep-alive",
+                     new BooleanParameter(&_keepAlive));
+
+  options->addOption("--collection", "collection name to use in tests",
+                     new StringParameter(&_collection));
+
+  std::unordered_set<std::string> cases = {"version",
+                                           "document",
+                                           "collection",
+                                           "import-document",
+                                           "hash",
+                                           "skiplist",
+                                           "edge",
+                                           "shapes",
+                                           "shapes-append",
+                                           "random-shapes",
+                                           "crud",
+                                           "crud-append",
+                                           "crud-write-read",
+                                           "aqltrx",
+                                           "counttrx",
+                                           "multitrx",
+                                           "multi-collection",
+                                           "aqlinsert",
+                                           "aqlv8"};
+  std::vector<std::string> casesVector(cases.begin(), cases.end());
+  std::string casesJoined = StringUtils::join(casesVector, ", ");
+
+  options->addOption(
+      "--test-case", "test case to use (possible values: " + casesJoined + ")",
+      new DiscreteValuesParameter<StringParameter>(&_testCase, cases));
+
+  options->addOption("--complexity", "complexity parameter for the test",
+                     new UInt64Parameter(&_complexity));
+
+  options->addOption("--delay",
+                     "use a startup delay (necessary only when run in series)",
+                     new BooleanParameter(&_delay));
+
+  options->addOption("--progress", "show progress",
+                     new BooleanParameter(&_progress));
+
+  options->addOption("--verbose",
+                     "print out replies if the http-header indicates db-errors",
+                     new BooleanParameter(&_verbose, false));
+
+  options->addOption("--quiet", "supress status messages",
+                     new BooleanParameter(&_quiet, false));
+}
+
+void ArangobFeature::status(std::string const& value) {
+  if (!_quiet) {
+    std::cout << value << std::endl;
+  }
+}
+
+std::atomic<int> ArangobFeature::_started;
+
+void ArangobFeature::updateStartCounter() { ++_started; }
+
+int ArangobFeature::getStartCounter() { return _started; }
+
+void ArangobFeature::start() {
+  LOG_TOPIC(TRACE, Logger::STARTUP) << name() << "::start";
+
+  ClientFeature* client =
+      dynamic_cast<ClientFeature*>(server()->feature("ClientFeature"));
+  client->setRetries(3);
+  client->setWarn(true);
+
+  int ret = EXIT_SUCCESS;
+
+  *_result = ret;
+  ARANGOB = this;
+
+  std::unique_ptr<BenchmarkOperation> benchmark(GetTestCase(_testCase));
+
+  if (benchmark == nullptr) {
+    ARANGOB = nullptr;
+    LOG(FATAL) << "invalid test case name '" << _testCase << "'";
+    FATAL_ERROR_EXIT();
+  }
+
+  status("starting threads...");
+
+  BenchmarkCounter<unsigned long> operationsCounter(0,
+                                                    (unsigned long)_operations);
+  ConditionVariable startCondition;
+
+  std::vector<Endpoint*> endpoints;
+  std::vector<BenchmarkThread*> threads;
+
+  double const stepSize = (double)_operations / (double)_concurreny;
+  int64_t realStep = (int64_t)stepSize;
+
+  if (stepSize - (double)((int64_t)stepSize) > 0.0) {
+    realStep++;
+  }
+
+  if (realStep % 1000 != 0) {
+    realStep += 1000 - (realStep % 1000);
+  }
+
+  // add some more offset so we don't get into trouble with threads of different
+  // speed
+  realStep += 10000;
+
+  // start client threads
+  for (uint64_t i = 0; i < _concurreny; ++i) {
+    Endpoint* endpoint = Endpoint::clientFactory(client->endpoint());
+    endpoints.push_back(endpoint);
+
+    BenchmarkThread* thread = new BenchmarkThread(
+        benchmark.get(), &startCondition, &ArangobFeature::updateStartCounter,
+        i, (unsigned long)_batchSize, &operationsCounter, client, _keepAlive,
+        _async, _verbose);
+
+    threads.push_back(thread);
+    thread->setOffset((size_t)(i * realStep));
+    thread->start();
+  }
+
+  // give all threads a chance to start so they will not miss the broadcast
+  while (getStartCounter() < (int)_concurreny) {
+    usleep(5000);
+  }
+
+  if (_delay) {
+    status("sleeping (startup delay)...");
+    sleep(10);
+  }
+
+  status("executing tests...");
+
+  double start = TRI_microtime();
+
+  // broadcast the start signal to all threads
+  {
+    CONDITION_LOCKER(guard, startCondition);
+    guard.broadcast();
+  }
+
+  size_t const stepValue = _operations / 20;
+  size_t nextReportValue = stepValue;
+
+  if (nextReportValue < 100) {
+    nextReportValue = 100;
+  }
+
+  while (true) {
+    size_t const numOperations = operationsCounter.getDone();
+
+    if (numOperations >= (size_t)_operations) {
+      break;
+    }
+
+    if (_progress && numOperations >= nextReportValue) {
+      LOG(INFO) << "number of operations: " << nextReportValue;
+      nextReportValue += stepValue;
+    }
+
+    usleep(20000);
+  }
+
+  double time = TRI_microtime() - start;
+  double requestTime = 0.0;
+
+  for (uint64_t i = 0; i < _concurreny; ++i) {
+    requestTime += threads[i]->getTime();
+  }
+
+  size_t failures = operationsCounter.failures();
+  size_t incomplete = operationsCounter.incompleteFailures();
+
+  std::cout << std::endl;
+
+  std::cout << "Total number of operations: " << _operations
+            << ", keep alive: " << (_keepAlive ? "yes" : "no")
+            << ", async: " << (_async ? "yes" : "no")
+            << ", batch size: " << _batchSize
+            << ", concurrency level (threads): " << _concurreny << std::endl;
+
+  std::cout << "Test case: " << _testCase << ", complexity: " << _complexity
+            << ", database: '" << client->databaseName() << "', collection: '"
+            << _collection << "'" << std::endl;
+
+  std::cout << "Total request/response duration (sum of all threads): "
+            << std::fixed << requestTime << " s" << std::endl;
+
+  std::cout << "Request/response duration (per thread): " << std::fixed
+            << (requestTime / (double)_concurreny) << " s" << std::endl;
+
+  std::cout << "Time needed per operation: " << std::fixed
+            << (time / _operations) << " s" << std::endl;
+
+  std::cout << "Time needed per operation per thread: " << std::fixed
+            << (time / (double)_operations * (double)_concurreny) << " s"
+            << std::endl;
+
+  std::cout << "Operations per second rate: " << std::fixed
+            << ((double)_operations / time) << std::endl;
+
+  std::cout << "Elapsed time since start: " << std::fixed << time << " s"
+            << std::endl
+            << std::endl;
+
+  if (failures > 0) {
+    LOG(WARN) << "WARNING: " << failures << " arangob request(s) failed!";
+  }
+  if (incomplete > 0) {
+    LOG(WARN) << "WARNING: " << incomplete
+              << " arangob requests with incomplete results!";
+  }
+
+  benchmark->tearDown();
+
+  for (uint64_t i = 0; i < _concurreny; ++i) {
+    delete threads[i];
+    delete endpoints[i];
+  }
+
+  if (failures > 0) {
+    ret = EXIT_FAILURE;
+  }
+
+  *_result = ret;
+}
+
+void ArangobFeature::stop() {
+  LOG_TOPIC(TRACE, Logger::STARTUP) << name() << "::stop";
+
+  ARANGOB = nullptr;
+}