Merge branch 'devel' of github.com:arangodb/arangodb into devel

arangod/Agency/AgencyComm.cpp

@@ -46,12 +46,20 @@
 #include "SimpleHttpClient/SimpleHttpResult.h"
 
 #include <thread>
+#ifdef DEBUG_SYNC_REPLICATION
+#include <atomic>
+#endif
 
 using namespace arangodb;
 using namespace arangodb::application_features;
 using namespace arangodb::httpclient;
 using namespace arangodb::rest;
 
+#ifdef DEBUG_SYNC_REPLICATION
+static std::atomic<uint64_t> debugUniqId(1);
+bool AgencyComm::syncReplDebug = false;
+#endif
+
 static void addEmptyVPackObject(std::string const& name,
                                 VPackBuilder& builder) {
   builder.add(VPackValue(name));
@@ -862,6 +870,11 @@ AgencyCommResult AgencyComm::casValue(std::string const& key,
 }
 
 uint64_t AgencyComm::uniqid(uint64_t count, double timeout) {
+#ifdef DEBUG_SYNC_REPLICATION
+  if (AgencyComm::syncReplDebug == true) {
+    return debugUniqId++;
+  }
+#endif
   static int const maxTries = 1000000;
   // this is pretty much forever, but we simply cannot continue at all
   // if we do not get a unique id from the agency.

arangod/Agency/AgencyComm.h

@@ -477,6 +477,11 @@ class AgencyComm {
   static uint64_t const INITIAL_SLEEP_TIME = 5000;
   static uint64_t const MAX_SLEEP_TIME = 50000;
 
+#ifdef DEBUG_SYNC_REPLICATION
+ public:
+  static bool syncReplDebug;
+#endif
+
  public:
   AgencyCommResult sendServerState(double ttl);
 

arangod/Cluster/v8-cluster.cpp

@@ -1318,6 +1318,26 @@ static void JS_CoordinatorConfigServerState(
   TRI_V8_TRY_CATCH_END
 }
 
+#ifdef DEBUG_SYNC_REPLICATION
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set arangoserver state to initialized
+////////////////////////////////////////////////////////////////////////////////
+
+static void JS_EnableSyncReplicationDebug(
+    v8::FunctionCallbackInfo<v8::Value> const& args) {
+  TRI_V8_TRY_CATCH_BEGIN(isolate);
+  v8::HandleScope scope(isolate);
+
+  if (args.Length() != 0) {
+    TRI_V8_THROW_EXCEPTION_USAGE("enableSyncReplicationDebug()");
+  }
+
+  ServerState::instance()->setInitialized();
+  AgencyComm::syncReplDebug = true;
+  TRI_V8_TRY_CATCH_END
+}
+#endif
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief return whether the cluster is initialized
 ////////////////////////////////////////////////////////////////////////////////
@@ -2227,6 +2247,10 @@ void TRI_InitV8Cluster(v8::Isolate* isolate, v8::Handle<v8::Context> context) {
                        JS_DBserverConfigServerState);
   TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("coordinatorConfig"),
                        JS_CoordinatorConfigServerState);
+#ifdef DEBUG_SYNC_REPLICATION
+  TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("enableSyncReplicationDebug"),
+                       JS_EnableSyncReplicationDebug);
+#endif
   TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("initialized"),
                        JS_InitializedServerState);
   TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING("isCoordinator"),

arangod/RestServer/VocbaseContext.cpp

@@ -96,7 +96,6 @@ rest::ResponseCode VocbaseContext::authenticate() {
   if (username.empty()) {
     // mop: set user to root so that the foxx stuff
     // knows about us
-    _request->setUser("root");
     return rest::ResponseCode::OK;
   }
 

arangod/StorageEngine/MMFilesEngine.cpp

@@ -24,6 +24,7 @@
 #include "MMFilesEngine.h"
 #include "Basics/FileUtils.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/ReadLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
@@ -525,7 +526,10 @@ int MMFilesEngine::dropDatabase(TRI_vocbase_t* vocbase) {
   // stop compactor thread
   shutdownDatabase(vocbase);
 
+  {
+    WRITE_LOCKER(locker, _pathsLock);
     _collectionPaths.erase(vocbase->id());
+  }
 
   return dropDatabaseDirectory(databaseDirectory(vocbase->id()));
 }
@@ -681,7 +685,10 @@ void MMFilesEngine::dropCollection(TRI_vocbase_t* vocbase, arangodb::LogicalColl
   RocksDBFeature::dropCollection(vocbase->id(), collection->cid());
 
   // rename collection directory
-  if (!collection->path().empty()) {
+  if (collection->path().empty()) {
+    return;
+  }
+
   std::string const collectionPath = collection->path();
 
 #ifdef _WIN32
@@ -741,26 +748,7 @@ void MMFilesEngine::dropCollection(TRI_vocbase_t* vocbase, arangodb::LogicalColl
       }
       LOG(ERR) << "cannot rename directory of dropped collection '" << name
                 << "' from '" << collection->path() << "' to '"
-                 << newFilename << "': " << TRI_errno_string(res) << systemError
-                 << ", source exists: " << TRI_IsDirectory(collection->path().c_str()) 
-                 << ", dest exists: " << TRI_IsDirectory(newFilename.c_str()) 
-                 << ", status: " << collection->statusString();
-
-        std::vector<std::string> files = TRI_FilesDirectory(collection->path().c_str());
-        LOG(ERR) << "ALL FILES: " << files;
-        for (auto const& f : files) {
-          bool isDir = TRI_IsDirectory(f.c_str());
-          std::string full = basics::FileUtils::buildFilename(collection->path(), f);
-          LOG(ERR) << "- found: " << f << ", IS DIR: " << isDir;
-          if (isDir) {
-            LOG(ERR) << "- removing dir: " << TRI_RemoveDirectory(full.c_str());
-          } else {
-            LOG(ERR) << "- file: " << full << ", size: " << TRI_SizeFile(full.c_str());
-            LOG(ERR) << "- removing file: " << TRI_UnlinkFile(full.c_str());
-          }
-        }
-        LOG(ERR) << "ALL FILES AGAIN: " << TRI_FilesDirectory(collection->path().c_str());
-
+                << newFilename << "': " << TRI_errno_string(res) << systemError;
     } else {
       LOG(DEBUG) << "wiping dropped collection '" << name
                   << "' from disk";
@@ -774,7 +762,6 @@ void MMFilesEngine::dropCollection(TRI_vocbase_t* vocbase, arangodb::LogicalColl
     }
   }
 }
-}
 
 // asks the storage engine to change properties of the collection as specified in 
 // the VPack Slice object and persist them. If this operation fails 
@@ -1167,6 +1154,8 @@ std::string MMFilesEngine::databaseParametersFilename(TRI_voc_tick_t id) const {
 }
 
 std::string MMFilesEngine::collectionDirectory(TRI_voc_tick_t databaseId, TRI_voc_cid_t id) const {
+  READ_LOCKER(locker, _pathsLock);
+
   auto it = _collectionPaths.find(databaseId);
 
   if (it == _collectionPaths.end()) {
@@ -1307,6 +1296,8 @@ std::string MMFilesEngine::createCollectionDirectoryName(std::string const& base
 }
 
 void MMFilesEngine::registerCollectionPath(TRI_voc_tick_t databaseId, TRI_voc_cid_t id, std::string const& path) {
+  WRITE_LOCKER(locker, _pathsLock);
+
   auto it = _collectionPaths.find(databaseId);
 
   if (it == _collectionPaths.end()) {
@@ -1316,12 +1307,16 @@ void MMFilesEngine::registerCollectionPath(TRI_voc_tick_t databaseId, TRI_voc_ci
 }
 
 void MMFilesEngine::unregisterCollectionPath(TRI_voc_tick_t databaseId, TRI_voc_cid_t id) {
+  /*
+  WRITE_LOCKER(locker, _pathsLock);
+
   auto it = _collectionPaths.find(databaseId);
 
   if (it == _collectionPaths.end()) {
     return;
   }
-//  (*it).second.erase(id);
+  (*it).second.erase(id);
+  */
 }
 
 void MMFilesEngine::saveCollectionInfo(TRI_vocbase_t* vocbase,

arangod/StorageEngine/MMFilesEngine.h

@@ -339,6 +339,7 @@ class MMFilesEngine final : public StorageEngine {
   TRI_voc_tick_t _maxTick;
   std::vector<std::pair<std::string, std::string>> _deleted;
 
+  arangodb::basics::ReadWriteLock mutable _pathsLock;
   std::unordered_map<TRI_voc_tick_t, std::unordered_map<TRI_voc_cid_t, std::string>> _collectionPaths;
 
   struct CompactionBlocker {
@@ -350,7 +351,7 @@ class MMFilesEngine final : public StorageEngine {
   };
 
   // lock for compaction blockers
-  arangodb::basics::ReadWriteLock _compactionBlockersLock;
+  arangodb::basics::ReadWriteLock mutable _compactionBlockersLock;
   // cross-database map of compaction blockers, protected by _compactionBlockersLock
   std::unordered_map<TRI_vocbase_t*, std::vector<CompactionBlocker>> _compactionBlockers;
   

arangod/VocBase/LogicalCollection.cpp

@@ -1039,6 +1039,9 @@ void LogicalCollection::drop() {
     _revisionsCache->clear();
   }
   
+  // make sure collection has been closed
+  this->close();
+
   TRI_ASSERT(!ServerState::instance()->isCoordinator());
   StorageEngine* engine = EngineSelectorFeature::ENGINE;
   engine->dropCollection(_vocbase, this);

js/common/tests/shell/shell-fs.js

@@ -714,7 +714,7 @@ function FileSystemSuite () {
         fail();
       }
       catch (err) {
-        assertEqual(ERRORS.ERROR_FILE_NOT_FOUND.code, err.errorNum);
+        assertEqual(ERRORS.ERROR_SYS_ERROR.code, err.errorNum);
       }
     },
 

js/server/modules/@arangodb/sync-replication-debug.js

@@ -0,0 +1,47 @@
+/* jshint strict: false */
+/* global ArangoServerState */
+
+// //////////////////////////////////////////////////////////////////////////////
+// / DISCLAIMER
+// /
+// / Copyright 2017 ArangoDB GmbH, Cologne, Germany
+// /
+// / Licensed under the Apache License, Version 2.0 (the "License")
+// / you may not use this file except in compliance with the License.
+// / You may obtain a copy of the License at
+// /
+// /     http://www.apache.org/licenses/LICENSE-2.0
+// /
+// / Unless required by applicable law or agreed to in writing, software
+// / distributed under the License is distributed on an "AS IS" BASIS,
+// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// / See the License for the specific language governing permissions and
+// / limitations under the License.
+// /
+// / Copyright holder is triAGENS GmbH, Cologne, Germany
+// /
+// / @author Andreas Streichardt
+// //////////////////////////////////////////////////////////////////////////////
+
+exports.setup = function() {
+  global.ArangoServerState.enableSyncReplicationDebug();
+  ArangoServerState.setRole('PRIMARY');
+  global.ArangoAgency.set = function() { return true; };
+  global.ArangoAgency.write = function() { return true; };
+  global.ArangoAgency.increaseVersion = function() { return true; };
+  global.ArangoAgency.get = function(path) {
+    let value = {};
+    let pathSegments = path.split('/');
+    let keyValue = 1;
+    value.arango = pathSegments.reverse().reduce((v, key) => {
+      let kv = {};
+      kv[key] = v;
+      return kv;
+    }, keyValue);
+    return value;
+  };
+  global.ArangoAgency.lockRead = function() { return true; };
+  global.ArangoAgency.lockWrite = function() { return true; };
+  global.ArangoAgency.unlockRead = function() { return true; };
+  global.ArangoAgency.unlockWrite = function() { return true; };
+};