Added test to ensure RocksDB recovers keygen data outside WAL. (#3072)

arangod/RocksDBEngine/RocksDBEngine.cpp

@@ -1260,6 +1260,27 @@ Result RocksDBEngine::createLoggerState(TRI_vocbase_t* vocbase,
   return Result{};
 }
 
+std::vector<std::string> RocksDBEngine::currentWalFiles() {
+  rocksdb::VectorLogPtr files;
+  std::vector<std::string> names;
+
+  auto status = _db->GetSortedWalFiles(files);
+  if (!status.ok()) {
+    return names;  // TODO: error here?
+  }
+
+  for (size_t current = 0; current < files.size(); current++) {
+    auto f = files[current].get();
+    try {
+      names.push_back(f->PathName());
+    } catch (...) {
+      return names;
+    }
+  }
+
+  return names;
+}
+
 void RocksDBEngine::determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep) {
   rocksdb::VectorLogPtr files;
 

arangod/RocksDBEngine/RocksDBEngine.h

@@ -226,6 +226,7 @@ class RocksDBEngine final : public StorageEngine {
   std::pair<TRI_voc_tick_t, TRI_voc_cid_t> mapObjectToCollection(
       uint64_t) const;
 
+  std::vector<std::string> currentWalFiles();
   void determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep);
   void pruneWalFiles();
 

arangod/V8Server/v8-collection.cpp

@@ -3152,6 +3152,7 @@ static void JS_CompletionsVocbase(
   result->Set(j++, TRI_V8_ASCII_STRING("_createEdgeCollection()"));
   result->Set(j++, TRI_V8_ASCII_STRING("_createView()"));
   result->Set(j++, TRI_V8_ASCII_STRING("_createStatement()"));
+  result->Set(j++, TRI_V8_ASCII_STRING("_currentWalFiles()"));
   result->Set(j++, TRI_V8_ASCII_STRING("_document()"));
   result->Set(j++, TRI_V8_ASCII_STRING("_drop()"));
   result->Set(j++, TRI_V8_ASCII_STRING("_dropDatabase()"));

arangod/V8Server/v8-vocbase.cpp

@@ -59,6 +59,7 @@
 #include "Rest/Version.h"
 #include "RestServer/ConsoleThread.h"
 #include "RestServer/DatabaseFeature.h"
+#include "RocksDBEngine/RocksDBEngine.h"
 #include "Statistics/StatisticsFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
@@ -2043,6 +2044,32 @@ void JS_ArangoDBContext(v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_END;
 }
 
+/// @brief return a list of all wal files (empty list if not rocksdb)
+static void JS_CurrentWalFiles(v8::FunctionCallbackInfo<v8::Value> const& args) {
+  TRI_V8_TRY_CATCH_BEGIN(isolate);
+  v8::HandleScope scope(isolate);
+
+  std::vector<std::string> names;
+  StorageEngine* engine = EngineSelectorFeature::ENGINE;
+  bool haveRocks = engine->typeName() == RocksDBEngine::EngineName;
+  if (haveRocks) {
+    names = static_cast<RocksDBEngine*>(engine)->currentWalFiles();
+  }
+  std::sort(names.begin(), names.end());
+
+  // already create an array of the correct size
+  v8::Handle<v8::Array> result = v8::Array::New(isolate);
+
+  size_t const n = names.size();
+
+  for (size_t i = 0; i < n; ++i) {
+    result->Set(static_cast<uint32_t>(i), TRI_V8_STD_STRING(names[i]));
+  }
+
+  TRI_V8_RETURN(result);
+  TRI_V8_TRY_CATCH_END
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief creates a TRI_vocbase_t global context
 ////////////////////////////////////////////////////////////////////////////////
@@ -2094,6 +2121,8 @@ void TRI_InitV8VocBridge(v8::Isolate* isolate, v8::Handle<v8::Context> context,
                        JS_NameDatabase);
   TRI_AddMethodVocbase(isolate, ArangoNS, TRI_V8_ASCII_STRING("_path"),
                        JS_PathDatabase);
+  TRI_AddMethodVocbase(isolate, ArangoNS, TRI_V8_ASCII_STRING("_currentWalFiles"),
+                       JS_CurrentWalFiles);
   TRI_AddMethodVocbase(isolate, ArangoNS, TRI_V8_ASCII_STRING("_versionFilename"),
                        JS_VersionFilenameDatabase, true);
   TRI_AddMethodVocbase(isolate, ArangoNS,
@@ -2263,3 +2292,4 @@ void TRI_InitV8VocBridge(v8::Isolate* isolate, v8::Handle<v8::Context> context,
   context->Global()->ForceSet(TRI_V8_ASCII_STRING("_AQL"),
                               v8::Undefined(isolate), v8::DontEnum);
 }
+

js/server/tests/recovery/collection-keygen-rocksdb.js

@@ -0,0 +1,121 @@
+/* jshint globalstrict:false, strict:false, unused: false */
+/* global assertTrue, assertFalse, assertEqual */
+// //////////////////////////////////////////////////////////////////////////////
+// / @brief tests for transactions
+// /
+// / @file
+// /
+// / DISCLAIMER
+// /
+// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
+// /
+// / Licensed under the Apache License, Version 2.0 (the "License")
+// / you may not use this file except in compliance with the License.
+// / You may obtain a copy of the License at
+// /
+// /     http://www.apache.org/licenses/LICENSE-2.0
+// /
+// / Unless required by applicable law or agreed to in writing, software
+// / distributed under the License is distributed on an "AS IS" BASIS,
+// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// / See the License for the specific language governing permissions and
+// / limitations under the License.
+// /
+// / Copyright holder is triAGENS GmbH, Cologne, Germany
+// /
+// / @author Jan Steemann
+// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
+// //////////////////////////////////////////////////////////////////////////////
+
+var db = require('@arangodb').db;
+var internal = require('internal');
+var jsunity = require('jsunity');
+
+function runSetup () {
+  'use strict';
+  internal.debugClearFailAt();
+  var c, i;
+
+  // write some documents with autoincrement keys
+  db._drop('UnitTestsRecovery1');
+  c = db._create('UnitTestsRecovery1', { keyOptions: { type: 'autoincrement',
+    offset: 0, increment: 10 } } );
+  for (i = 0; i < 1000; i++) {
+    c.save({ value: i });
+  }
+  var wals = db._currentWalFiles().map(function(f) {
+    // strip off leading `/` or `/archive/` if it exists
+    var p = f.split('/');
+    return p[p.length - 1];
+  });
+
+  // write to other collection until all documents from first collection are
+  // out of the wall
+  db._drop('UnitTestsRecovery2');
+  c = db._create('UnitTestsRecovery2');
+  var keepWriting = true;
+  while (keepWriting) {
+    var padding = 'aaa';
+    for (i = 0; i < 10000; i++) {
+      var padding = padding.concat('aaa');
+      c.save({ value: i , text: padding });
+    }
+
+    keepWriting = false;
+    var walsLeft = db._currentWalFiles().map(function(f) {
+      // strip off leading `/` or `/archive/` if it exists
+      var p = f.split('/');
+      return p[p.length - 1];
+    });
+    for (var j = 0; j < wals.length; j++) {
+      if (walsLeft.indexOf(wals[j]) !== -1) { // still have old wal file
+        keepWriting = true;
+      }
+    }
+  }
+  c.save({ value: 0 }, { waitForSync: true });
+
+  internal.debugSegfault('crashing server');
+}
+
+// //////////////////////////////////////////////////////////////////////////////
+// / @brief test suite
+// //////////////////////////////////////////////////////////////////////////////
+
+function recoverySuite () {
+  'use strict';
+  jsunity.jsUnity.attachAssertions();
+
+  return {
+    setUp: function () {},
+    tearDown: function () {},
+
+    // //////////////////////////////////////////////////////////////////////////////
+    // / @brief test whether we still pick up the right autoincrement value
+    // //////////////////////////////////////////////////////////////////////////////
+
+    testCollectionKeyGen: function () {
+      var c, d;
+
+      c = db._collection('UnitTestsRecovery1');
+      d = c.save({ value: 1001});
+      assertEqual("10010", d._key);
+    }
+
+  };
+}
+
+// //////////////////////////////////////////////////////////////////////////////
+// / @brief executes the test suite
+// //////////////////////////////////////////////////////////////////////////////
+
+function main (argv) {
+  'use strict';
+  if (argv[1] === 'setup') {
+    runSetup();
+    return 0;
+  } else {
+    jsunity.run(recoverySuite);
+    return jsunity.done().status ? 0 : 1;
+  }
+}