Merge branch 'devel' of github.com:arangodb/arangodb into devel

arangod/Agency/AgencyComm.cpp

@@ -46,12 +46,20 @@
 #include "SimpleHttpClient/SimpleHttpResult.h"
 
 #include <thread>
+#ifdef DEBUG_SYNC_REPLICATION
+#include <atomic>
+#endif
 
 using namespace arangodb;
 using namespace arangodb::application_features;
 using namespace arangodb::httpclient;
 using namespace arangodb::rest;
 
+#ifdef DEBUG_SYNC_REPLICATION
+static std::atomic<uint64_t> debugUniqId(1);
+bool AgencyComm::syncReplDebug = false;
+#endif
+
 static void addEmptyVPackObject(std::string const& name,
                                 VPackBuilder& builder) {
   builder.add(VPackValue(name));
@@ -862,6 +870,11 @@ AgencyCommResult AgencyComm::casValue(std::string const& key,
 }
 
 uint64_t AgencyComm::uniqid(uint64_t count, double timeout) {
+#ifdef DEBUG_SYNC_REPLICATION
+  if (AgencyComm::syncReplDebug == true) {
+    return debugUniqId++;
+  }
+#endif
   static int const maxTries = 1000000;
   // this is pretty much forever, but we simply cannot continue at all
   // if we do not get a unique id from the agency.

arangod/Agency/AgencyComm.h

@@ -477,6 +477,11 @@ class AgencyComm {
   static uint64_t const INITIAL_SLEEP_TIME = 5000;
   static uint64_t const MAX_SLEEP_TIME = 50000;
 
+#ifdef DEBUG_SYNC_REPLICATION
+ public:
+  static bool syncReplDebug;
+#endif
+
  public:
   AgencyCommResult sendServerState(double ttl);
 

arangod/Cluster/v8-cluster.cpp

@@ -1318,6 +1318,26 @@ static void JS_CoordinatorConfigServerState(
   TRI_V8_TRY_CATCH_END
 }
 
+#ifdef DEBUG_SYNC_REPLICATION
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set arangoserver state to initialized
+////////////////////////////////////////////////////////////////////////////////
+
+static void JS_EnableSyncReplicationDebug(
+    v8::FunctionCallbackInfo<v8::Value> const& args) {
+  TRI_V8_TRY_CATCH_BEGIN(isolate);
+  v8::HandleScope scope(isolate);
+
+  if (args.Length() != 0) {
+    TRI_V8_THROW_EXCEPTION_USAGE("enableSyncReplicationDebug()");
+  }
+
+  ServerState::instance()->setInitialized();
+  AgencyComm::syncReplDebug = true;
+  TRI_V8_TRY_CATCH_END
+}
+#endif
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief return whether the cluster is initialized
 ////////////////////////////////////////////////////////////////////////////////
@@ -2227,6 +2247,10 @@ void TRI_InitV8Cluster(v8::Isolate* isolate, v8::Handle<v8::Context> context) {
                        JS_DBserverConfigServerState);
   TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("coordinatorConfig"),
                        JS_CoordinatorConfigServerState);
+#ifdef DEBUG_SYNC_REPLICATION
+  TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("enableSyncReplicationDebug"),
+                       JS_EnableSyncReplicationDebug);
+#endif
   TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("initialized"),
                        JS_InitializedServerState);
   TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("isCoordinator"),

arangod/RestServer/VocbaseContext.cpp

@@ -96,7 +96,6 @@ rest::ResponseCode VocbaseContext::authenticate() {
   if (username.empty()) {
     // mop: set user to root so that the foxx stuff
     // knows about us
-    _request->setUser("root");
     return rest::ResponseCode::OK;
   }
 

arangod/StorageEngine/MMFilesEngine.cpp

@@ -24,6 +24,7 @@
 #include "MMFilesEngine.h"
 #include "Basics/FileUtils.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/ReadLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
@@ -525,7 +526,10 @@ int MMFilesEngine::dropDatabase(TRI_vocbase_t* vocbase) {
   // stop compactor thread
   shutdownDatabase(vocbase);
 
-  _collectionPaths.erase(vocbase->id());
+  {
+    WRITE_LOCKER(locker, _pathsLock);
+    _collectionPaths.erase(vocbase->id());
+  }
 
   return dropDatabaseDirectory(databaseDirectory(vocbase->id()));
 }
@@ -681,96 +685,79 @@ void MMFilesEngine::dropCollection(TRI_vocbase_t* vocbase, arangodb::LogicalColl
   RocksDBFeature::dropCollection(vocbase->id(), collection->cid());
 
   // rename collection directory
-  if (!collection->path().empty()) {
-    std::string const collectionPath = collection->path();
+  if (collection->path().empty()) {
+    return;
+  }
+
+  std::string const collectionPath = collection->path();
 
 #ifdef _WIN32
-    size_t pos = collectionPath.find_last_of('\\');
+  size_t pos = collectionPath.find_last_of('\\');
 #else
-    size_t pos = collectionPath.find_last_of('/');
+  size_t pos = collectionPath.find_last_of('/');
 #endif
 
-    bool invalid = false;
+  bool invalid = false;
 
-    if (pos == std::string::npos || pos + 1 >= collectionPath.size()) {
+  if (pos == std::string::npos || pos + 1 >= collectionPath.size()) {
+    invalid = true;
+  }
+
+  std::string path;
+  std::string relName;
+  if (!invalid) {
+    // extract path part
+    if (pos > 0) {
+      path = collectionPath.substr(0, pos); 
+    }
+
+    // extract relative filename
+    relName = collectionPath.substr(pos + 1);
+
+    if (!StringUtils::isPrefix(relName, "collection-") || 
+        StringUtils::isSuffix(relName, ".tmp")) {
       invalid = true;
     }
+  }
 
-    std::string path;
-    std::string relName;
-    if (!invalid) {
-      // extract path part
-      if (pos > 0) {
-        path = collectionPath.substr(0, pos); 
-      }
+  if (invalid) {
+    LOG(ERR) << "cannot rename dropped collection '" << name
+              << "': unknown path '" << collection->path() << "'";
+  } else {
+    // prefix the collection name with "deleted-"
 
-      // extract relative filename
-      relName = collectionPath.substr(pos + 1);
+    std::string const newFilename = 
+      FileUtils::buildFilename(path, "deleted-" + relName.substr(std::string("collection-").size()));
 
-      if (!StringUtils::isPrefix(relName, "collection-") || 
-          StringUtils::isSuffix(relName, ".tmp")) {
-        invalid = true;
-      }
+    // check if target directory already exists
+    if (TRI_IsDirectory(newFilename.c_str())) {
+      // remove existing target directory
+      TRI_RemoveDirectory(newFilename.c_str());
     }
 
-    if (invalid) {
-      LOG(ERR) << "cannot rename dropped collection '" << name
-               << "': unknown path '" << collection->path() << "'";
-    } else {
-      // prefix the collection name with "deleted-"
+    // perform the rename
+    LOG(TRACE) << "renaming collection directory from '"
+                << collection->path() << "' to '" << newFilename << "'";
 
-      std::string const newFilename = 
-        FileUtils::buildFilename(path, "deleted-" + relName.substr(std::string("collection-").size()));
-
-      // check if target directory already exists
-      if (TRI_IsDirectory(newFilename.c_str())) {
-        // remove existing target directory
-        TRI_RemoveDirectory(newFilename.c_str());
+    std::string systemError;
+    int res = TRI_RenameFile(collection->path().c_str(), newFilename.c_str(), nullptr, &systemError);
+      
+    if (res != TRI_ERROR_NO_ERROR) {
+      if (!systemError.empty()) {
+        systemError = ", error details: " + systemError;
       }
+      LOG(ERR) << "cannot rename directory of dropped collection '" << name
+                << "' from '" << collection->path() << "' to '"
+                << newFilename << "': " << TRI_errno_string(res) << systemError;
+    } else {
+      LOG(DEBUG) << "wiping dropped collection '" << name
+                  << "' from disk";
 
-      // perform the rename
-      LOG(TRACE) << "renaming collection directory from '"
-                 << collection->path() << "' to '" << newFilename << "'";
+      res = TRI_RemoveDirectory(newFilename.c_str());
 
-      std::string systemError;
-      int res = TRI_RenameFile(collection->path().c_str(), newFilename.c_str(), nullptr, &systemError);
-        
       if (res != TRI_ERROR_NO_ERROR) {
-        if (!systemError.empty()) {
-          systemError = ", error details: " + systemError;
-        }
-        LOG(ERR) << "cannot rename directory of dropped collection '" << name
-                 << "' from '" << collection->path() << "' to '"
-                 << newFilename << "': " << TRI_errno_string(res) << systemError
-                 << ", source exists: " << TRI_IsDirectory(collection->path().c_str()) 
-                 << ", dest exists: " << TRI_IsDirectory(newFilename.c_str()) 
-                 << ", status: " << collection->statusString();
-
-        std::vector<std::string> files = TRI_FilesDirectory(collection->path().c_str());
-        LOG(ERR) << "ALL FILES: " << files;
-        for (auto const& f : files) {
-          bool isDir = TRI_IsDirectory(f.c_str());
-          std::string full = basics::FileUtils::buildFilename(collection->path(), f);
-          LOG(ERR) << "- found: " << f << ", IS DIR: " << isDir;
-          if (isDir) {
-            LOG(ERR) << "- removing dir: " << TRI_RemoveDirectory(full.c_str());
-          } else {
-            LOG(ERR) << "- file: " << full << ", size: " << TRI_SizeFile(full.c_str());
-            LOG(ERR) << "- removing file: " << TRI_UnlinkFile(full.c_str());
-          }
-        }
-        LOG(ERR) << "ALL FILES AGAIN: " << TRI_FilesDirectory(collection->path().c_str());
-
-      } else {
-        LOG(DEBUG) << "wiping dropped collection '" << name
-                   << "' from disk";
-
-        res = TRI_RemoveDirectory(newFilename.c_str());
-
-        if (res != TRI_ERROR_NO_ERROR) {
-          LOG(ERR) << "cannot wipe dropped collection '" << name
-                   << "' from disk: " << TRI_errno_string(res);
-        }
+        LOG(ERR) << "cannot wipe dropped collection '" << name
+                  << "' from disk: " << TRI_errno_string(res);
       }
     }
   }
@@ -1167,6 +1154,8 @@ std::string MMFilesEngine::databaseParametersFilename(TRI_voc_tick_t id) const {
 }
 
 std::string MMFilesEngine::collectionDirectory(TRI_voc_tick_t databaseId, TRI_voc_cid_t id) const {
+  READ_LOCKER(locker, _pathsLock);
+
   auto it = _collectionPaths.find(databaseId);
 
   if (it == _collectionPaths.end()) {
@@ -1307,6 +1296,8 @@ std::string MMFilesEngine::createCollectionDirectoryName(std::string const& base
 }
 
 void MMFilesEngine::registerCollectionPath(TRI_voc_tick_t databaseId, TRI_voc_cid_t id, std::string const& path) {
+  WRITE_LOCKER(locker, _pathsLock);
+
   auto it = _collectionPaths.find(databaseId);
 
   if (it == _collectionPaths.end()) {
@@ -1316,12 +1307,16 @@ void MMFilesEngine::registerCollectionPath(TRI_voc_tick_t databaseId, TRI_voc_ci
 }
 
 void MMFilesEngine::unregisterCollectionPath(TRI_voc_tick_t databaseId, TRI_voc_cid_t id) {
+  /*
+  WRITE_LOCKER(locker, _pathsLock);
+
   auto it = _collectionPaths.find(databaseId);
 
   if (it == _collectionPaths.end()) {
     return;
   }
-//  (*it).second.erase(id);
+  (*it).second.erase(id);
+  */
 }
 
 void MMFilesEngine::saveCollectionInfo(TRI_vocbase_t* vocbase,

arangod/StorageEngine/MMFilesEngine.h

@@ -339,6 +339,7 @@ class MMFilesEngine final : public StorageEngine {
   TRI_voc_tick_t _maxTick;
   std::vector<std::pair<std::string, std::string>> _deleted;
 
+  arangodb::basics::ReadWriteLock mutable _pathsLock;
   std::unordered_map<TRI_voc_tick_t, std::unordered_map<TRI_voc_cid_t, std::string>> _collectionPaths;
 
   struct CompactionBlocker {
@@ -350,7 +351,7 @@ class MMFilesEngine final : public StorageEngine {
   };
 
   // lock for compaction blockers
-  arangodb::basics::ReadWriteLock _compactionBlockersLock;
+  arangodb::basics::ReadWriteLock mutable _compactionBlockersLock;
   // cross-database map of compaction blockers, protected by _compactionBlockersLock
   std::unordered_map<TRI_vocbase_t*, std::vector<CompactionBlocker>> _compactionBlockers;
   

arangod/VocBase/LogicalCollection.cpp

@@ -1038,6 +1038,9 @@ void LogicalCollection::drop() {
   if (_revisionsCache != nullptr) {
     _revisionsCache->clear();
   }
+  
+  // make sure collection has been closed
+  this->close();
 
   TRI_ASSERT(!ServerState::instance()->isCoordinator());
   StorageEngine* engine = EngineSelectorFeature::ENGINE;

js/common/tests/shell/shell-fs.js

@@ -714,7 +714,7 @@ function FileSystemSuite () {
         fail();
       }
       catch (err) {
-        assertEqual(ERRORS.ERROR_FILE_NOT_FOUND.code, err.errorNum);
+        assertEqual(ERRORS.ERROR_SYS_ERROR.code, err.errorNum);
       }
     },
 

js/server/modules/@arangodb/sync-replication-debug.js

@@ -0,0 +1,47 @@
+/* jshint strict: false */
+/* global ArangoServerState */
+
+// //////////////////////////////////////////////////////////////////////////////
+// / DISCLAIMER
+// /
+// / Copyright 2017 ArangoDB GmbH, Cologne, Germany
+// /
+// / Licensed under the Apache License, Version 2.0 (the "License")
+// / you may not use this file except in compliance with the License.
+// / You may obtain a copy of the License at
+// /
+// /     http://www.apache.org/licenses/LICENSE-2.0
+// /
+// / Unless required by applicable law or agreed to in writing, software
+// / distributed under the License is distributed on an "AS IS" BASIS,
+// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// / See the License for the specific language governing permissions and
+// / limitations under the License.
+// /
+// / Copyright holder is triAGENS GmbH, Cologne, Germany
+// /
+// / @author Andreas Streichardt
+// //////////////////////////////////////////////////////////////////////////////
+
+exports.setup = function() {
+  global.ArangoServerState.enableSyncReplicationDebug();
+  ArangoServerState.setRole('PRIMARY');
+  global.ArangoAgency.set = function() { return true; };
+  global.ArangoAgency.write = function() { return true; };
+  global.ArangoAgency.increaseVersion = function() { return true; };
+  global.ArangoAgency.get = function(path) {
+    let value = {};
+    let pathSegments = path.split('/');
+    let keyValue = 1;
+    value.arango = pathSegments.reverse().reduce((v, key) => {
+      let kv = {};
+      kv[key] = v;
+      return kv;
+    }, keyValue);
+    return value;
+  };
+  global.ArangoAgency.lockRead = function() { return true; };
+  global.ArangoAgency.lockWrite = function() { return true; };
+  global.ArangoAgency.unlockRead = function() { return true; };
+  global.ArangoAgency.unlockWrite = function() { return true; };
+};