Merge branch 'devel' of ssh://github.com/triAGENS/ArangoDB into devel

CHANGELOG

@@ -10,6 +10,9 @@ v2.1.0 (XXXX-XX-XX)
 v2.0.1 (XXXX-XX-XX)
 -------------------
 
+* bumped version in `X-Arango-Version` compatibility header sent by arangosh and other
+  client tools from `1.5` to `2.0`.
+
 * fixed startup options for arango-dfdb, added details option for arango-dfdb
 
 * fixed display of missing error messages and codes in arangosh

Documentation/UserManual/Aql.md

@@ -1495,9 +1495,9 @@ AQL has the following functions to traverse graphs:
   - `filterVertices`: an optional list of example vertex documents that the traversal will
     treat specially. If no examples are given, the traversal will handle all encountered
     vertices equally. If one or many vertex examples are given, the traversal will exclude
-    the vertex from the result and/or not descend into it. Optionally, `filterVertices` can 
-    contain the name of a user-defined AQL function that should be responsible for filtering.
-    If so, the AQL function is expected to have the following signature:
+    any non-matching vertex from the result and/or not descend into it. Optionally,
+    `filterVertices` can contain the name of a user-defined AQL function that should be responsible
+    for filtering. If so, the AQL function is expected to have the following signature:
 
         function (config, vertex, path)
 

UnitTests/HttpInterface/arangodb.rb

@@ -8,6 +8,7 @@ $address = ENV['ARANGO_SERVER'] || '127.0.0.1:8529'
 $user = ENV['ARANGO_USER']
 $password = ENV['ARANGO_PASSWORD']
 $ssl = ENV['ARANGO_SSL']
+$silent = ENV['ARANGO_NO_LOG'] || ''
 
 begin
   $address = RSpec.configuration.ARANGO_SERVER
@@ -162,6 +163,9 @@ class ArangoDB
 ################################################################################
 
   def self.log (args)
+    # disable logging if requested
+    return if not ($silent.nil? || $silent.empty? || $silent == "0")
+
     if args.key?(:output)
       logfile = File.new("logs/#{args[:output]}", "a")
     else

UnitTests/Makefile.unittests

@@ -10,6 +10,7 @@ SKIP_GEO = 0
 SKIP_AHUACATL = 0
 SKIP_RANGES = 0
 VALGRIND = 
+NO_LOG = 
 USERNAME = root
 PASSWORD =
 PROTO = http
@@ -575,7 +576,7 @@ unittests-http-server:
 	@echo "================================================================================"
 	@echo
 
-	cd @top_srcdir@/UnitTests/HttpInterface && ARANGO_SERVER="$(VOCHOST):$(VOCPORT)" ARANGO_SSL=0 ARANGO_USER="$(USERNAME)" ARANGO_PASSWORD="$(PASSWORD)" ./run-tests || test "x$(FORCE)" == "x1"
+	cd @top_srcdir@/UnitTests/HttpInterface && ARANGO_NO_LOG="$(NO_LOG)" ARANGO_SERVER="$(VOCHOST):$(VOCPORT)" ARANGO_SSL=0 ARANGO_USER="$(USERNAME)" ARANGO_PASSWORD="$(PASSWORD)" ./run-tests || test "x$(FORCE)" == "x1"
 
 	kill `cat $(PIDFILE)`
 

arangod/VocBase/datafile.c

@@ -630,13 +630,40 @@ static bool CheckDatafile (TRI_datafile_t* datafile) {
   return true;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief extract the numeric part from a filename
+/// the filename must look like this: /.*type-abc\.ending$/, where abc is
+/// a number, and type and ending are arbitrary letters
+////////////////////////////////////////////////////////////////////////////////
+
+static uint64_t GetNumericFilenamePart (const char* filename) {
+  char* pos1;
+  char* pos2;
+
+  pos1 = strrchr(filename, '.');
+
+  if (pos1 == NULL) {
+    return 0;
+  }
+
+  pos2 = strrchr(filename, '-');
+
+  if (pos2 == NULL || pos2 > pos1) {
+    return 0;
+  }
+
+  return TRI_UInt64String2(pos2 + 1, pos1 - pos2 - 1);
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief opens a datafile
 ////////////////////////////////////////////////////////////////////////////////
 
-static TRI_datafile_t* OpenDatafile (char const* filename, bool ignoreErrors) {
+static TRI_datafile_t* OpenDatafile (char const* filename, 
+                                     bool ignoreErrors) {
   TRI_datafile_t* datafile;
   TRI_voc_size_t size;
+  TRI_voc_fid_t fid;
   bool ok;
   void* data;
   char* ptr;
@@ -650,6 +677,8 @@ static TRI_datafile_t* OpenDatafile (char const* filename, bool ignoreErrors) {
   // this function must not be called for non-physical datafiles
   assert(filename != NULL);
 
+  fid = GetNumericFilenamePart(filename);
+
   // ..........................................................................
   // attempt to open a datafile file
   // ..........................................................................
@@ -664,7 +693,6 @@ static TRI_datafile_t* OpenDatafile (char const* filename, bool ignoreErrors) {
     return NULL;
   }
 
-
   // compute the size of the file
   res = fstat(fd, &status);
 
@@ -767,7 +795,7 @@ static TRI_datafile_t* OpenDatafile (char const* filename, bool ignoreErrors) {
                mmHandle,
                size,
                size,
-               header._fid,
+               fid,
                data);
 
   return datafile;
@@ -1253,12 +1281,16 @@ int TRI_WriteElementDatafile (TRI_datafile_t* datafile,
     }
   }
 
-  if (datafile->_tickMin == 0) {
-    datafile->_tickMin = tick;
-  }
+  if (type != TRI_DF_MARKER_ATTRIBUTE &&
+      type != TRI_DF_MARKER_SHAPE) {
 
-  if (datafile->_tickMax < tick) {
-    datafile->_tickMax = tick;
+    if (datafile->_tickMin == 0) {
+      datafile->_tickMin = tick;
+    }
+
+    if (datafile->_tickMax < tick) {
+      datafile->_tickMax = tick;
+    }
   }
    
   assert(markerSize > 0);

arangod/VocBase/server.c

@@ -1866,6 +1866,9 @@ int TRI_StartServer (TRI_server_t* server,
 
   server->_wasShutdownCleanly = (res == TRI_ERROR_NO_ERROR);
 
+  if (! server->_wasShutdownCleanly) {
+    LOG_INFO("server was not shut down cleanly. scanning datafile markers");
+  }
 
   // .............................................................................
   // verify existence of "databases" subdirectory

arangod/VocBase/voc-shaper.c

@@ -1301,6 +1301,8 @@ int TRI_MoveMarkerVocShaper (TRI_shaper_t* s,
     TRI_shape_t* l = (TRI_shape_t*) p;
     void* f;
 
+    TRI_LockMutex(&shaper->_shapeLock);
+
     // remove the old marker
     f = TRI_RemoveKeyAssociativeSynced(&shaper->_shapeIds, &l->_sid);
     assert(f != NULL);
@@ -1316,11 +1318,15 @@ int TRI_MoveMarkerVocShaper (TRI_shaper_t* s,
     // re-insert 
     f = TRI_InsertElementAssociativeSynced(&shaper->_shapeDictionary, l);
     assert(f == NULL);
+    
+    TRI_UnlockMutex(&shaper->_shapeLock);
   }
   else if (marker->_type == TRI_DF_MARKER_ATTRIBUTE) {
     TRI_df_attribute_marker_t* m = (TRI_df_attribute_marker_t*) marker;
     char* p = ((char*) m) + sizeof(TRI_df_attribute_marker_t);
     void* f;
+    
+    TRI_LockMutex(&shaper->_attributeLock);
  
     // remove attribute by name (p points to new location of name, but names
     // are identical in old and new marker) 
@@ -1338,6 +1344,8 @@ int TRI_MoveMarkerVocShaper (TRI_shaper_t* s,
     // now re-insert same attribute with adjusted pointer
     f = TRI_InsertKeyAssociativeSynced(&shaper->_attributeIds, &m->_aid, m);
     assert(f == NULL);
+    
+    TRI_UnlockMutex(&shaper->_attributeLock);
   }
 
   return TRI_ERROR_NO_ERROR;
@@ -1384,10 +1392,28 @@ int TRI_InsertAttributeVocShaper (TRI_shaper_t* s,
   LOG_TRACE("found attribute '%s', aid: %lu", p, (unsigned long) m->_aid);
 
   f = TRI_InsertKeyAssociativeSynced(&shaper->_attributeNames, p, m);
-  assert(f == NULL);
+
+  if (f != NULL) {
+    char const* name = shaper->_collection->base.base._info._name;
+
+#ifdef TRI_ENABLE_MAINTAINER_MODE
+    LOG_WARNING("found duplicate attribute name '%s' in collection '%s'", p, name);
+#else
+    LOG_TRACE("found duplicate attribute name '%s' in collection '%s'", p, name); 
+#endif    
+  }
 
   f = TRI_InsertKeyAssociativeSynced(&shaper->_attributeIds, &m->_aid, m);
-  assert(f == NULL);
+  
+  if (f != NULL) {
+    char const* name = shaper->_collection->base.base._info._name;
+
+#ifdef TRI_ENABLE_MAINTAINER_MODE
+    LOG_WARNING("found duplicate attribute id '%llu' in collection '%s'", (unsigned long long) m->_aid, name);
+#else
+    LOG_TRACE("found duplicate attribute id '%llu' in collection '%s'", (unsigned long long) m->_aid, name);
+#endif    
+  }
 
   // no lock is necessary here as we are the only users of the shaper at this time
   if (shaper->_nextAid <= m->_aid) {

arangod/VocBase/vocbase.c

@@ -823,6 +823,10 @@ static int ScanPath (TRI_vocbase_t* vocbase,
   files = TRI_FilesDirectory(path);
   n = files._length;
 
+  if (iterateMarkers) {
+    LOG_TRACE("scanning all collection markers in database '%s", vocbase->_name);
+  }
+
   for (i = 0;  i < n;  ++i) {
     char* name;
     char* file;

js/server/arango-dfdb.js

@@ -168,21 +168,13 @@ function PrintEntries (entries, amount) {
 /// @brief checks a datafile deeply
 ////////////////////////////////////////////////////////////////////////////////
 
-function DeepCheckDatafile (collection, type, datafile, scan, details) {
+function DeepCheckDatafile (collection, type, datafile, scan) {
   var entries = scan.entries;
 
-  printf("Entries\n");
-
   var lastGood = 0;
   var lastGoodPos = 0;
   var stillGood = true;
 
-  if (details) {
-    // print details
-    PrintEntries(entries, 10);
-    PrintEntries(entries, -10);
-  }
-
   for (var i = 0;  i < entries.length;  ++i) {
     var entry = entries[i];
 
@@ -356,12 +348,19 @@ function CheckDatafile (collection, type, datafile, issues, details) {
     RemoveDatafile(collection, type, datafile);
     return;
   }
+  
+  if (details) {
+    // print details
+    printf("Entries\n");
+    PrintEntries(scan.entries, 10);
+    PrintEntries(scan.entries, -10);
+  }
 
   if (scan.status === 1 && scan.isSealed) {
     return;
   }
 
-  DeepCheckDatafile(collection, type, datafile, scan, details);
+  DeepCheckDatafile(collection, type, datafile, scan);
 }
 
 ////////////////////////////////////////////////////////////////////////////////

lib/SimpleHttpClient/SimpleHttpClient.cpp

@@ -313,7 +313,7 @@ namespace triagens {
         _writeBuffer.appendText("Connection: Close\r\n");
       }
       _writeBuffer.appendText("User-Agent: ArangoDB\r\n");
-      _writeBuffer.appendText("X-Arango-Version: 1.5\r\n");
+      _writeBuffer.appendText("X-Arango-Version: 2.0\r\n");
       _writeBuffer.appendText("Accept-Encoding: deflate\r\n");
       
       // do basic authorization