fixed arango-dfdb issues

arangod/V8Server/v8-query.cpp

@@ -2044,9 +2044,7 @@ static v8::Handle<v8::Value> JS_ByExampleQuery (v8::Arguments const& argv) {
       TRI_DestroyVector(&filtered);
       TRI_V8_EXCEPTION_MEMORY(scope);
     }
-  }
+    
-
-  if (0 < total) {
     assert(barrier != 0);
 
     size_t s, e;
@@ -2054,7 +2052,7 @@ static v8::Handle<v8::Value> JS_ByExampleQuery (v8::Arguments const& argv) {
 
     for (size_t j = s; j < e; ++j) {
       TRI_doc_mptr_t* mptr = (TRI_doc_mptr_t*) TRI_AtVector(&filtered, j);
-
+      
       v8::Handle<v8::Value> doc = WRAP_SHAPED_JSON(trx, col->_cid, mptr, barrier);
 
       if (doc.IsEmpty()) {

arangod/V8Server/v8-vocbase.cpp

@@ -6076,6 +6076,7 @@ static v8::Handle<v8::Value> JS_DatafileScanVocbaseCol (v8::Arguments const& arg
   result->Set(v8::String::New("endPosition"), v8::Number::New(scan._endPosition));
   result->Set(v8::String::New("numberMarkers"), v8::Number::New(scan._numberMarkers));
   result->Set(v8::String::New("status"), v8::Number::New(scan._status));
+  result->Set(v8::String::New("isSealed"), v8::Boolean::New(scan._isSealed));
 
   v8::Handle<v8::Array> entries = v8::Array::New();
   result->Set(v8::String::New("entries"), entries);

etc/arangodb/arango-dfdb.conf.in

@@ -5,6 +5,10 @@ directory= @LOCALSTATEDIR@/lib/arangodb
 # maximal-journal-size=33554432
 # remove-on-drop=true
 
+[server]
+# set number of threads to 1 so we don't have concurrency
+threads = 1
+
 [javascript]
 startup-directory = @PKGDATADIR@/js
 app-path = @LOCALSTATEDIR@/lib/arangodb-apps

etc/relative/arango-dfdb.conf

@@ -7,6 +7,8 @@ no-upgrade = true
 
 [server]
 disable-authentication = true
+# set number of threads to 1 so we don't have concurrency
+threads = 1
 
 [javascript]
 startup-directory = ./js

js/server/arango-dfdb.js

@@ -61,34 +61,52 @@ function WipeDatafile (collection, type, datafile, lastGoodPos) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief checks a journal
+/// @brief queries user to wipe a datafile
 ////////////////////////////////////////////////////////////////////////////////
 
-function DeepCheckJournal (collection, type, datafile, scan, lastGoodPos) {
+function QueryWipeDatafile (collection, type, datafile, scan, lastGoodPos) {
   var entries = scan.entries;
 
   if (entries.length == 0) {
-    printf("WARNING: The journal is empty. Even the header is missing. Going\n");
+    if (type === "journal" || type === "compactor") {
-    printf("         to remove the datafile.\n");
+      printf("WARNING: The journal is empty. Even the header is missing. Going\n");
+      printf("         to remove the file.\n");
+    }
+    else {
+      printf("WARNING: The datafile is empty. Even the header is missing. Going\n");
+      printf("         to remove the datafile. This should never happen. Datafiles\n");
+      printf("         are append-only. Make sure your hard disk does not contain\n");
+      printf("         any hardware errors.\n");
+    }
     printf("\n");
 
     RemoveDatafile(collection, type, datafile);
-
     return;
   }
 
-  if (entries.length === lastGoodPos + 3 && entries[lastGoodPos + 2].status === 2) {
+  var ask = true;
-    printf("WARNING: The journal was not closed properly, the last entry is corrupted.\n");
+  if (type === "journal") {
-    printf("         This might happen ArangoDB was killed and the last entry was not\n");
+    if (entries.length === lastGoodPos + 3 && entries[lastGoodPos + 2].status === 2) {
-    printf("         fully written to disk. Going to remove the last entry.\n");
+      printf("WARNING: The journal was not closed properly, the last entry is corrupted.\n");
-    printf("\n");
+      printf("         This might happen ArangoDB was killed and the last entry was not\n");
+      printf("         fully written to disk. Going to remove the last entry.\n");
+      ask = false;
+    }
+    else {
+      printf("WARNING: The journal was not closed properly, the last entries are corrupted.\n");
+      printf("         This might happen ArangoDB was killed and the last entries were not\n");
+      printf("         fully written to disk.\n");
+    }
   }
   else {
-    printf("WARNING: The journal was not closed properly, the last entries is corrupted.\n");
+    printf("WARNING: The datafile contains corrupt entries. This should never happen.\n");
-    printf("         This might happen ArangoDB was killed and the last entries were not\n");
+    printf("         Datafiles are append-only. Make sure your hard disk does not contain\n");
-    printf("         fully written to disk.\n");
+    printf("         any hardware errors.\n");
-    printf("\n");
+  }
 
+  printf("\n");
+
+  if (ask) {
     printf("Wipe the last entries (Y/N)? ");
     var line = console.getline();
 
@@ -104,57 +122,34 @@ function DeepCheckJournal (collection, type, datafile, scan, lastGoodPos) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief checks a datafile
+/// @brief prints details about entries
 ////////////////////////////////////////////////////////////////////////////////
 
-function DeepCheckDatafile (collection, type, datafile, scan, lastGoodPos) {
+function PrintEntries (entries, amount) {
-  var entries = scan.entries;
+  var start, end;
 
-  if (entries.length == 0) {
+  if (amount > 0) {
-    printf("WARNING: The datafile is empty. Even the header is missing. Going\n");
+    start = 0;
-    printf("         to remove the datafile. This should never happen. Datafiles\n");
+    end = amount; 
-    printf("         are append-only. Make sure your hard disk does not contain\n");
+    if (end > entries.length) {
-    printf("         any hardware errors.\n");
+      end = entries.length;
-    printf("\n");
+    }
+  }
+  else {
+    start = entries.length + amount - 1;
+    if (start < 0) {
+      return;
+    }
+    if (start < Math.abs(amount)) {
+      return;
+    }
 
-    RemoveDatafile(collection, type, datafile);
+    end = entries.length;
-
-    return;
   }
 
-  printf("WARNING: The datafile contains corrupt entries. This should never happen.\n");
+  for (var i = start;  i < end;  ++i) {
-  printf("         Datafiles are append-only. Make sure your hard disk does not contain\n");
-  printf("         any hardware errors.\n");
-  printf("\n");
-
-  printf("Wipe the last entries (Y/N)? ");
-  var line = console.getline();
-
-  if (line !== "yes" && line !== "YES" && line !== "y" && line !== "Y") {
-    printf("ABORTING\n");
-    return;
-  }
-
-  var entry = entries[lastGoodPos];
-
-  WipeDatafile(collection, type, datafile, entry.position + entry.size);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief checks a datafile deeply
-////////////////////////////////////////////////////////////////////////////////
-
-function DeepCheckDatafile (collection, type, datafile, scan) {
-  var entries = scan.entries;
-
-  printf("Entries\n");
-
-  var lastGood = 0;
-  var lastGoodPos = 0;
-  var stillGood = true;
-
-  for (var i = 0;  i < entries.length;  ++i) {
     var entry = entries[i];
+
     var s = "unknown";
 
     switch (entry.status) {
@@ -165,7 +160,31 @@ function DeepCheckDatafile (collection, type, datafile, scan) {
       case 5: s = "FAILED (crc mismatch)";  break;
     }
 
-    printf("  %d: status %s type %d size %d\n", i, s, entry.type, entry.size);
+    printf("  %d: status %s type %d size %d, tick %s\n", i, s, entry.type, entry.size, entry.tick);
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief checks a datafile deeply
+////////////////////////////////////////////////////////////////////////////////
+
+function DeepCheckDatafile (collection, type, datafile, scan, details) {
+  var entries = scan.entries;
+
+  printf("Entries\n");
+
+  var lastGood = 0;
+  var lastGoodPos = 0;
+  var stillGood = true;
+
+  if (details) {
+    // print details
+    PrintEntries(entries, 10);
+    PrintEntries(entries, -10);
+  }
+
+  for (var i = 0;  i < entries.length;  ++i) {
+    var entry = entries[i];
 
     if (entry.status === 1 || entry.status === 2) {
       if (stillGood) {
@@ -182,12 +201,7 @@ function DeepCheckDatafile (collection, type, datafile, scan) {
     printf("  Last good position: %d\n", lastGood.position + lastGood.size);
     printf("\n");
 
-    if (type === "journal" || type === "compactor") {
+    QueryWipeDatafile(collection, type, datafile, scan, lastGoodPos);
-      DeepCheckJournal(collection, type, datafile, scan, lastGoodPos);
-    }
-    else {
-      DeepCheckDatafile(collection, type, datafile, scan, lastGoodPos);
-    }
   }
 
   printf("\n");
@@ -197,7 +211,7 @@ function DeepCheckDatafile (collection, type, datafile, scan) {
 /// @brief checks a datafile
 ////////////////////////////////////////////////////////////////////////////////
 
-function CheckDatafile (collection, type, datafile, issues) {
+function CheckDatafile (collection, type, datafile, issues, details) {
   printf("Datafile\n");
   printf("  path: %s\n", datafile);
   printf("  type: %s\n", type);
@@ -208,6 +222,8 @@ function CheckDatafile (collection, type, datafile, issues) {
   printf("  maximal size: %d\n", scan.maximalSize);
   printf("  total used: %d\n", scan.endPosition);
   printf("  # of entries: %d\n", scan.numberMarkers);
+  printf("  status: %d\n", scan.status);
+  printf("  isSealed: %s\n", scan.isSealed ? "yes" : "no");
 
   // set default value to unknown
   var statusMessage = "UNKNOWN (" + scan.status + ")"; 
@@ -217,9 +233,12 @@ function CheckDatafile (collection, type, datafile, issues) {
     case 1:
       statusMessage = "OK";
       color = internal.COLORS.COLOR_GREEN;
+      if (! scan.isSealed && type === "datafile") {
+        color = internal.COLORS.COLOR_YELLOW;
+      }
       break;
 
-    case 1:
+    case 2:
       statusMessage = "NOT OK (reached empty marker)";
       color = internal.COLORS.COLOR_RED;
       break;
@@ -296,7 +315,7 @@ function CheckDatafile (collection, type, datafile, issues) {
     return;
   }
 
-  if (scan.entries.length == 2 && scan.entries[1].type !== 2000) {
+  if (scan.entries.length === 2 && scan.entries[1].type !== 2000) {
     // asserting a TRI_COL_MARKER_HEADER as second marker
     statusMessage = "datafile contains no collection header marker at pos #1!";
     color = internal.COLORS.COLOR_YELLOW;
@@ -316,19 +335,40 @@ function CheckDatafile (collection, type, datafile, issues) {
     RemoveDatafile(collection, type, datafile);
     return;
   }
+  
+  if (type !== "journal" && scan.entries.length === 3 && scan.entries[2].type === 0) {
+    // got the two initial header markers but nothing else...
+    statusMessage = "datafile is empty but not sealed";
+    color = internal.COLORS.COLOR_YELLOW;
 
-  if (scan.status === 1) {
+    issues.push({
+      collection: collection.name(), 
+      path: datafile, 
+      type: type, 
+      status: scan.status, 
+      message: statusMessage,
+      color: color
+    });
+
+    printf(color);
+    printf("WARNING: %s\n", statusMessage);
+    printf(internal.COLORS.COLOR_RESET);
+    RemoveDatafile(collection, type, datafile);
     return;
   }
 
-  DeepCheckDatafile(collection, type, datafile, scan);
+  if (scan.status === 1 && scan.isSealed) {
+    return;
+  }
+
+  DeepCheckDatafile(collection, type, datafile, scan, details);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief checks a collection
 ////////////////////////////////////////////////////////////////////////////////
 
-function CheckCollection (collection, issues) {
+function CheckCollection (collection, issues, details) {
   printf("Database\n");
   printf("  name: %s\n", internal.db._name());
   printf("  path: %s\n", internal.db._path());
@@ -348,15 +388,15 @@ function CheckCollection (collection, issues) {
   printf("\n");
 
   for (var i = 0;  i < datafiles.journals.length;  ++i) {
-    CheckDatafile(collection, "journal", datafiles.journals[i], issues);
+    CheckDatafile(collection, "journal", datafiles.journals[i], issues, details);
   }
 
   for (var i = 0;  i < datafiles.datafiles.length;  ++i) {
-    CheckDatafile(collection, "datafiles", datafiles.datafiles[i], issues);
+    CheckDatafile(collection, "datafile", datafiles.datafiles[i], issues, details);
   }
 
   for (var i = 0;  i < datafiles.compactors.length;  ++i) {
-    CheckDatafile(collection, "compactor", datafiles.compactors[i], issues);
+    CheckDatafile(collection, "compactor", datafiles.compactors[i], issues, details);
   }
 }
 
@@ -378,7 +418,7 @@ function main (argv) {
 
     return 0;
   };
-
+  
   printf("%s\n", "    ___      _         __ _ _           ___  ___    ___ ");
   printf("%s\n", "   /   \\__ _| |_ __ _ / _(_) | ___     /   \\/ __\\  / _ \\");
   printf("%s\n", "  / /\\ / _` | __/ _` | |_| | |/ _ \\   / /\\ /__\\// / /_\\/");
@@ -398,9 +438,11 @@ function main (argv) {
     printf("  %d: %s\n", i, databases[i]);
   }
   
+  var line;
+  
   printf("Database to check: ");
   while (true) {
-    var line = console.getline();
+    line = console.getline();
    
     if (line == "") {
       printf("Exiting. Please wait.\n");
@@ -445,7 +487,7 @@ function main (argv) {
   var a = [];
   
   while (true) {
-    var line = console.getline();
+    line = console.getline();
 
     if (line == "") {
       printf("Exiting. Please wait.\n");
@@ -471,6 +513,23 @@ function main (argv) {
       }
     }
   }
+  
+  printf("\n");
+  printf("Prints details (yes/no)? ");
+
+  var details = false;
+  while (true) {
+    line = console.getline();
+    if (line === "") {
+      printf("Exiting. Please wait.\n");
+      return;
+    }
+
+    if (line === "yes" || line === "YES" || line === "y" || line === "Y") {
+      details = true;
+    }
+    break;
+  }
 
   var issues = [ ];
 
@@ -496,7 +555,7 @@ function main (argv) {
 
     printf("\n");
 
-    CheckCollection(collection, issues);
+    CheckCollection(collection, issues, details);
   }
 
  

lib/V8/v8-utils.cpp

@@ -2275,7 +2275,7 @@ static v8::Handle<v8::Value> JS_Wait (v8::Arguments const& argv) {
   if (gc) {
     // wait with gc
     v8::V8::LowMemoryNotification();
-    while(! v8::V8::IdleNotification()) {
+    while (! v8::V8::IdleNotification()) {
     }
 
     size_t i = 0;