Merge branch 'devel' of https://github.com/arangodb/arangodb into feature/alliterator

Documentation/Books/Manual/SUMMARY.md

@@ -205,6 +205,7 @@
 #
 * [Release notes](ReleaseNotes/README.md)
   * [Whats New in 3.2](ReleaseNotes/NewFeatures32.md)
+  * [Known Issues in 3.2](ReleaseNotes/KnownIssues32.md)
   * [Incompatible changes in 3.2](ReleaseNotes/UpgradingChanges32.md)
   * [Whats New in 3.1](ReleaseNotes/NewFeatures31.md)
   * [Incompatible changes in 3.1](ReleaseNotes/UpgradingChanges31.md)

arangod/Aql/AqlTransaction.h

@@ -29,6 +29,7 @@
 #include "Aql/Collection.h"
 #include "Transaction/StandaloneContext.h"
 #include "Transaction/Methods.h"
+#include "Transaction/Options.h"
 #include "VocBase/vocbase.h"
 
 namespace arangodb {
@@ -41,9 +42,11 @@ class AqlTransaction final : public transaction::Methods {
   AqlTransaction(
       std::shared_ptr<transaction::Context> const& transactionContext, 
       std::map<std::string, aql::Collection*> const* collections,
+      transaction::Options const& options,
       bool isMainTransaction)
-      : transaction::Methods(transactionContext),
-        _collections(*collections) {
+      : transaction::Methods(transactionContext, options),
+        _collections(*collections),
+        _options(options) {
     if (!isMainTransaction) {
       addHint(transaction::Hints::Hint::LOCK_NEVER);
     } else {
@@ -91,7 +94,7 @@ class AqlTransaction final : public transaction::Methods {
   /// AQL query running on the coordinator
   transaction::Methods* clone() const override {
     return new AqlTransaction(transaction::StandaloneContext::Create(vocbase()),
-        &_collections, false);
+        &_collections, _options, false);
   }
 
   /// @brief lockCollections, this is needed in a corner case in AQL: we need
@@ -106,6 +109,7 @@ class AqlTransaction final : public transaction::Methods {
   /// operation
  private:
   std::map<std::string, aql::Collection*> _collections;
+  transaction::Options _options;
 };
 
 }

arangod/Aql/Query.cpp

@@ -342,6 +342,7 @@ void Query::prepare(QueryRegistry* registry, uint64_t queryHash) {
       // create the transaction object, but do not start it yet
       AqlTransaction* trx = new AqlTransaction(
         createTransactionContext(), _collections.collections(),
+        _queryOptions.transactionOptions,
         _part == PART_MAIN);
       _trx = trx;
 
@@ -429,8 +430,8 @@ ExecutionPlan* Query::prepare() {
 
   // create the transaction object, but do not start it yet
   AqlTransaction* trx = new AqlTransaction(
-      createTransactionContext(), _collections.collections(),
-      _part == PART_MAIN);
+      createTransactionContext(), _collections.collections(), 
+      _queryOptions.transactionOptions, _part == PART_MAIN);
   _trx = trx;
     
   // As soon as we start du instantiate the plan we have to clean it
@@ -919,7 +920,8 @@ QueryResult Query::explain() {
 
     // create the transaction object, but do not start it yet
     _trx = new AqlTransaction(createTransactionContext(),
-                              _collections.collections(), true);
+                              _collections.collections(), 
+                              _queryOptions.transactionOptions, true);
 
     // we have an AST
     Result res = _trx->begin();

arangod/Aql/QueryOptions.cpp

@@ -172,6 +172,9 @@ void QueryOptions::fromVelocyPack(VPackSlice const& slice) {
       it.next();
     }
   }
+
+  // also handle transaction options
+  transactionOptions.fromVelocyPack(slice);
 }
 
 void QueryOptions::toVelocyPack(VPackBuilder& builder, bool disableOptimizerRules) const {
@@ -216,6 +219,9 @@ void QueryOptions::toVelocyPack(VPackBuilder& builder, bool disableOptimizerRule
     }
     builder.close(); // shardIds
   }
+  
+  // also handle transaction options
+  transactionOptions.toVelocyPack(builder);
 
   builder.close();
 }

arangod/Aql/QueryOptions.h

@@ -25,6 +25,7 @@
 #define ARANGOD_AQL_QUERY_OPTIONS_H 1
 
 #include "Basics/Common.h"
+#include "Transaction/Options.h"
 
 namespace arangodb {
 namespace velocypack {
@@ -58,6 +59,8 @@ struct QueryOptions {
   bool inspectSimplePlans;
   std::vector<std::string> optimizerRules;
   std::unordered_set<std::string> shardIds;
+
+  transaction::Options transactionOptions;
 };
 
 }

arangod/Cluster/TraverserEngine.cpp

@@ -101,7 +101,7 @@ BaseEngine::BaseEngine(TRI_vocbase_t* vocbase, VPackSlice info)
 
   _trx = new arangodb::aql::AqlTransaction(
       arangodb::transaction::StandaloneContext::Create(vocbase),
-      _collections.collections(), true);
+      _collections.collections(), transaction::Options(), true);
   // true here as last argument is crucial: it leads to the fact that the
   // created transaction is considered a "MAIN" part and will not switch
   // off collection locking completely!

arangod/RestServer/arangod.cpp

@@ -247,13 +247,13 @@ static void WINAPI ServiceMain(DWORD dwArgc, LPSTR* lpszArgv) {
       RegisterServiceCtrlHandlerA(lpszArgv[0], (LPHANDLER_FUNCTION)ServiceCtrl);
 
   // set start pending
-  SetServiceStatus(SERVICE_START_PENDING, 0, 1, 10000);
+  SetServiceStatus(SERVICE_START_PENDING, 0, 1, 10000, 0);
 
   ArangoGlobalContext context(ARGC, ARGV, SBIN_DIRECTORY);
   runServer(ARGC, ARGV, context);
 
   // service has stopped
-  SetServiceStatus(SERVICE_STOPPED, NO_ERROR, 0, 0);
+  SetServiceStatus(SERVICE_STOPPED, NO_ERROR, 0, 0, 0);
   TRI_CloseWindowsEventlog();
 }
 

arangod/RocksDBEngine/RocksDBKey.cpp

@@ -197,7 +197,7 @@ std::pair<bool, int32_t> RocksDBKey::geoValues(rocksdb::Slice const& slice) {
   uint64_t val =
       uint64FromPersistent(slice.data() + sizeof(char) + sizeof(uint64_t));
   bool isSlot = ((val & 0xFFULL) > 0);  // lowest byte is 0xFF if true
-  return std::pair<bool, int32_t>(isSlot, (val >> 32));
+  return std::pair<bool, int32_t>(isSlot, static_cast<int32_t>(val >> 32));
 }
 
 std::string const& RocksDBKey::string() const { return _buffer; }

arangod/RocksDBEngine/RocksDBTransactionState.cpp

@@ -438,8 +438,6 @@ RocksDBOperationResult RocksDBTransactionState::addOperation(
   // "transaction size" counters have reached their limit
   if (_options.intermediateCommitCount <= numOperations ||
       _options.intermediateCommitSize <= newSize) {
-
-    LOG_TOPIC(ERR, Logger::FIXME) << "INTERMEDIATE COMMIT!";
       internalCommit();
       _numInserts = 0;
       _numUpdates = 0;

arangod/Transaction/Options.cpp

@@ -23,6 +23,10 @@
 
 #include "Options.h"
 
+#include <velocypack/Builder.h>
+#include <velocypack/Slice.h>
+#include <velocypack/velocypack-aliases.h>
+
 using namespace arangodb::transaction;
   
 uint64_t Options::defaultMaxTransactionSize = UINT64_MAX;
@@ -42,3 +46,44 @@ void Options::setLimits(uint64_t maxTransactionSize, uint64_t intermediateCommit
   defaultIntermediateCommitSize = intermediateCommitSize;
   defaultIntermediateCommitCount = intermediateCommitCount;
 }
+
+void Options::fromVelocyPack(arangodb::velocypack::Slice const& slice) {
+  VPackSlice value;
+  
+  value = slice.get("lockTimeout");
+  if (value.isNumber()) {
+    lockTimeout = value.getNumber<double>();
+  }
+  value = slice.get("maxTransactionSize");
+  if (value.isNumber()) {
+    maxTransactionSize = value.getNumber<uint64_t>();
+  }
+  value = slice.get("intermediateCommitSize");
+  if (value.isNumber()) {
+    intermediateCommitSize = value.getNumber<uint64_t>();
+  }
+  value = slice.get("intermediateCommitCount");
+  if (value.isNumber()) {
+    intermediateCommitCount = value.getNumber<uint64_t>();
+  }
+  value = slice.get("allowImplicitCollections");
+  if (value.isBool()) {
+    allowImplicitCollections = value.getBool();
+  }
+  value = slice.get("waitForSync");
+  if (value.isBool()) {
+    waitForSync = value.getBool();
+  }
+}
+ 
+/// @brief add the options to an opened vpack builder 
+void Options::toVelocyPack(arangodb::velocypack::Builder& builder) const {
+  TRI_ASSERT(builder.isOpenObject());
+
+  builder.add("lockTimeout", VPackValue(lockTimeout));
+  builder.add("maxTransactionSize", VPackValue(maxTransactionSize));
+  builder.add("intermediateCommitSize", VPackValue(intermediateCommitSize));
+  builder.add("intermediateCommitCount", VPackValue(intermediateCommitCount));
+  builder.add("allowImplicitCollections", VPackValue(allowImplicitCollections));
+  builder.add("waitForSync", VPackValue(waitForSync));
+}

arangod/Transaction/Options.h

@@ -27,12 +27,25 @@
 #include "Basics/Common.h"
 
 namespace arangodb {
+namespace velocypack {
+class Builder;
+class Slice;
+}
+
 namespace transaction {
 
 struct Options {
   Options();
   
+  /// @brief adjust the global default values for transactions
   static void setLimits(uint64_t maxTransactionSize, uint64_t intermediateCommitSize, uint64_t intermediateCommitCount);
+
+  /// @brief read the options from a vpack slice
+  void fromVelocyPack(arangodb::velocypack::Slice const&);
+
+  /// @brief add the options to an opened vpack builder 
+  void toVelocyPack(arangodb::velocypack::Builder&) const;
+
   static constexpr double defaultLockTimeout = 900.0;
   static uint64_t defaultMaxTransactionSize;
   static uint64_t defaultIntermediateCommitSize;

arangod/V8Server/v8-actions.cpp

@@ -1401,7 +1401,7 @@ void TRI_InitV8Actions(v8::Isolate* isolate, v8::Handle<v8::Context> context) {
 ////////////////////////////////////////////////////////////////////////////////
 
 #ifdef ARANGODB_ENABLE_FAILURE_TESTS
-static bool clusterSendToAllServers(
+static int clusterSendToAllServers(
     std::string const& dbname,
     std::string const& path,  // Note: Has to be properly encoded!
     arangodb::rest::RequestType const& method, std::string const& body) {

arangosh/Import/ImportHelper.cpp

@@ -816,7 +816,7 @@ void ImportHelper::sendJsonBuffer(char const* str, size_t len, bool isObject) {
 
 SenderThread* ImportHelper::findSender() {
   while (!_senderThreads.empty()) {
-    for (std::unique_ptr<SenderThread>& t : _senderThreads) {
+    for (auto const& t : _senderThreads) {
       if (t->hasError()) {
         _hasError = true;
         _errorMessage = t->errorMessage();
@@ -825,7 +825,7 @@ SenderThread* ImportHelper::findSender() {
         return t.get();
       }
     }
-    usleep(500000);
+    usleep(100000);
   }
   return nullptr;
 }
@@ -833,7 +833,7 @@ SenderThread* ImportHelper::findSender() {
 void ImportHelper::waitForSenders() {
   while (!_senderThreads.empty()) {
     uint32_t numIdle = 0;
-    for (std::unique_ptr<SenderThread>& t : _senderThreads) {
+    for (auto const& t : _senderThreads) {
       if (t->idle() || t->hasError()) {
         numIdle++;
       }

arangosh/Import/SenderThread.cpp

@@ -68,8 +68,8 @@ void SenderThread::sendData(std::string const& url,
   _data.swap(data);
 
   // wake up the thread that may be waiting in run()
-  _idle = false;
   CONDITION_LOCKER(guard, _condition);
+  _idle = false;
   guard.broadcast();
 }
 
@@ -80,6 +80,8 @@ void SenderThread::run() {
       guard.wait();
     }
     if (isStopping()) {
+      CONDITION_LOCKER(guard, _condition);
+      _idle = true;
       return;
     }
     try {
@@ -96,8 +98,10 @@ void SenderThread::run() {
         _url.clear();
         _data.reset();
       }
+      CONDITION_LOCKER(guard, _condition);
       _idle = true;
     } catch (...) {
+      CONDITION_LOCKER(guard, _condition);
       _hasError = true;
       _idle = true;
     }

arangosh/Import/SenderThread.h

@@ -57,11 +57,11 @@ class SenderThread : public arangodb::Thread {
 
   void sendData(std::string const& url, basics::StringBuffer* sender);
 
-  bool idle() { return _idle; }
+  bool idle() const { return _idle; }
 
-  bool hasError() { return _hasError; }
+  bool hasError() const { return _hasError; }
 
-  std::string const& errorMessage() { return _errorMessage; }
+  std::string const& errorMessage() const { return _errorMessage; }
 
   void beginShutdown() override;
 

lib/ApplicationFeatures/ApplicationServer.cpp

@@ -594,7 +594,7 @@ void ApplicationServer::start() {
           }
         }
       }
-      
+      shutdownFatalError();
       // throw exception so the startup aborts
       THROW_ARANGO_EXCEPTION_MESSAGE(res, std::string("startup aborted: ") + TRI_errno_string(res));
     }

lib/ApplicationFeatures/ApplicationServer.h

@@ -36,6 +36,7 @@ class ProgramOptions;
 namespace application_features {
 class ApplicationFeature;
 
+// handled i.e. in WindowsServiceFeature.cpp
 enum class ServerState {
   UNINITIALIZED,
   IN_COLLECT_OPTIONS,

lib/ApplicationFeatures/WindowsServiceFeature.cpp

@@ -368,7 +368,7 @@ void DeleteService (bool force) {
 ////////////////////////////////////////////////////////////////////////////////
 
 void SetServiceStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode,
-                      DWORD dwCheckPoint, DWORD dwWaitHint) {
+                      DWORD dwCheckPoint, DWORD dwWaitHint, DWORD exitCode) {
   // disable control requests until the service is started
   SERVICE_STATUS ss;
 
@@ -381,9 +381,11 @@ void SetServiceStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode,
 
   // initialize ss structure
   ss.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
-  ss.dwServiceSpecificExitCode = 0;
   ss.dwCurrentState = dwCurrentState;
+
   ss.dwWin32ExitCode = dwWin32ExitCode;
+  ss.dwServiceSpecificExitCode = exitCode;
+
   ss.dwCheckPoint = dwCheckPoint;
   ss.dwWaitHint = dwWaitHint;
 
@@ -407,7 +409,7 @@ void SetServiceStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode,
 ///        really up and running.
 //////////////////////////////////////////////////////////////////////////////
 void WindowsServiceFeature::startupProgress () {
-  SetServiceStatus(SERVICE_START_PENDING, NO_ERROR, _progress++, 20000);
+  SetServiceStatus(SERVICE_START_PENDING, NO_ERROR, _progress++, 20000, 0);
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -416,7 +418,7 @@ void WindowsServiceFeature::startupProgress () {
 //////////////////////////////////////////////////////////////////////////////
 void WindowsServiceFeature::startupFinished () {
   // startup finished - signalize we're running.
-  SetServiceStatus(SERVICE_RUNNING, NO_ERROR, 0, 0);
+  SetServiceStatus(SERVICE_RUNNING, NO_ERROR, 0, 0, 0);
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -425,7 +427,7 @@ void WindowsServiceFeature::startupFinished () {
 //////////////////////////////////////////////////////////////////////////////
 void WindowsServiceFeature::shutDownBegins () {
   // startup finished - signalize we're running.
-  SetServiceStatus(SERVICE_STOP_PENDING, NO_ERROR, 0, 0);
+  SetServiceStatus(SERVICE_STOP_PENDING, NO_ERROR, 0, 0, 0);
 }
 
 
@@ -435,7 +437,7 @@ void WindowsServiceFeature::shutDownBegins () {
 //////////////////////////////////////////////////////////////////////////////
 void WindowsServiceFeature::shutDownComplete () {
   // startup finished - signalize we're running.
-  SetServiceStatus(SERVICE_STOPPED, NO_ERROR, 0, 0);
+  SetServiceStatus(SERVICE_STOPPED, NO_ERROR, 0, 0, 0);
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -444,7 +446,16 @@ void WindowsServiceFeature::shutDownComplete () {
 //////////////////////////////////////////////////////////////////////////////
 void WindowsServiceFeature::shutDownFailure () {
   // startup finished - signalize we're running.
-  SetServiceStatus(SERVICE_STOP, ERROR_FAIL_RESTART, 0, 0);
+  SetServiceStatus(SERVICE_STOP, ERROR_SERVICE_SPECIFIC_ERROR, 0, 0, 1);
+}
+
+//////////////////////////////////////////////////////////////////////////////
+/// @brief wrap ArangoDB server so we can properly emmit a status on shutdown
+///        starting
+//////////////////////////////////////////////////////////////////////////////
+void WindowsServiceFeature::abortFailure () {
+  // startup finished - signalize we're running.
+  SetServiceStatus(SERVICE_STOP, ERROR_SERVICE_SPECIFIC_ERROR, 0, 0, 2);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -470,7 +481,7 @@ void WINAPI ServiceCtrl(DWORD dwCtrlCode) {
   // stop service
   if (dwCtrlCode == SERVICE_CONTROL_STOP ||
       dwCtrlCode == SERVICE_CONTROL_SHUTDOWN) {
-    SetServiceStatus(SERVICE_STOP_PENDING, NO_ERROR, 0, 0);
+    SetServiceStatus(SERVICE_STOP_PENDING, NO_ERROR, 0, 0, 0);
 
     if (ArangoInstance != nullptr && ArangoInstance->_server != nullptr) {
       ArangoInstance->_server->beginShutdown();
@@ -480,28 +491,10 @@ void WINAPI ServiceCtrl(DWORD dwCtrlCode) {
       }
     }
   } else {
-    SetServiceStatus(dwState, NO_ERROR, 0, 0);
+    SetServiceStatus(dwState, NO_ERROR, 0, 0, 0);
   }
 }
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief parse windows specific commandline options
-////////////////////////////////////////////////////////////////////////////////
-/*
-bool TRI_ParseMoreArgs(int argc, char* argv[]) {
-  SetUnhandledExceptionFilter(unhandledExceptionHandler);
-
-    } else if (TRI_EqualString(argv[1], "--uninstall-service")) {
-      bool force = ((argc > 2) && !strcmp(argv[2], "--force"));
-      DeleteService(argc, argv, force);
-      exit(EXIT_SUCCESS);
-    }
-  }
-  return false;
-}
-*/
-
-
 WindowsServiceFeature::WindowsServiceFeature(application_features::ApplicationServer* server)
   : ApplicationFeature(server, "WindowsService"),
     _server(server){
@@ -556,6 +549,14 @@ void WindowsServiceFeature::collectOptions(std::shared_ptr<ProgramOptions> optio
                            new BooleanParameter(&_stopWaitService));
 }
 
+void WindowsServiceFeature::abortService() {
+  if (ArangoInstance != nullptr) {
+    ArangoInstance->_server = nullptr;
+    ArangoInstance->abortFailure();
+  }
+  exit(EXIT_FAILURE);
+}
+
 void WindowsServiceFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
   if (!TRI_InitWindowsEventLog()) {
     exit(EXIT_FAILURE);
@@ -570,6 +571,8 @@ void WindowsServiceFeature::validateOptions(std::shared_ptr<ProgramOptions> opti
   else if (_forceUninstall) {
   }
   else if (_startAsService) {
+    TRI_SetWindowsServiceAbortFunction(abortService);
+    
     ProgressHandler reporter{
       [this](ServerState state) {
         switch (state) {
@@ -585,6 +588,9 @@ void WindowsServiceFeature::validateOptions(std::shared_ptr<ProgramOptions> opti
         case ServerState::IN_START:
           this->startupProgress();
           break;
+        case ServerState::ABORT:
+          this->shutDownFailure();
+          break;
         case ServerState::UNINITIALIZED:
         case ServerState::STOPPED:
           break;

lib/ApplicationFeatures/WindowsServiceFeature.h

@@ -28,7 +28,7 @@
 extern SERVICE_STATUS_HANDLE ServiceStatus;
 
 void SetServiceStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode,
-                      DWORD dwCheckPoint, DWORD dwWaitHint);
+                      DWORD dwCheckPoint, DWORD dwWaitHint, DWORD exitCode);
 
 void WINAPI ServiceCtrl(DWORD dwCtrlCode);
 
@@ -51,7 +51,8 @@ class WindowsServiceFeature final : public application_features::ApplicationFeat
   void shutDownBegins ();
   void shutDownComplete ();
   void shutDownFailure ();
-
+  void abortFailure();
+  static void abortService();
  public:
   bool _installService = false;
   bool _unInstallService = false;