Merge branch 'devel' of github.com:triAGENS/ArangoDB into devel

CHANGELOG

@@ -1,3 +1,21 @@
+v2.1.0 (XXXX-XX-XX)
+-------------------
+
+* removed arangod command line option `--database.remove-on-compacted`
+
+
+v2.0.0 (2014-03-10)
+-------------------
+
+* first 2.0 release
+
+
+v2.0.0-rc2 (2014-03-07)
+-----------------------
+
+* fixed cluster authorization
+
+
 v2.0.0-rc1 (2014-02-28)
 -----------------------
 
@@ -217,7 +235,16 @@ v2.0.0-rc1 (2014-02-28)
 * added collection status "loading"
 
 
-v1.4.12 (XXXX-XX-XX)
+v1.4.13 (XXXX-XX-XX)
+--------------------
+
+* fixed too eager compaction
+  
+  The compaction will now wait for several seconds before trying to re-compact the same
+  collection. Additionally, some other limits have been introduced for the compaction.
+ 
+
+v1.4.12 (2014-03-05)
 --------------------
 
 * fixed display bug in web interface which caused the following problems:

Documentation/ImplementorManual/HttpSystem.md

@@ -27,27 +27,7 @@ monitoring of the server.
 @copydetails JSF_get_admin_statistics_description
 
 @CLEARPAGE
-@anchor HttpSystemAdminDescriptionShardingGET
+@anchor HttpSystemAdminServerRole
-@copydetails JSF_cluster_test_GET
+@copydetails JSF_get_admin_server_role
-
-@CLEARPAGE
-@anchor HttpSystemAdminDescriptionShardingPUT
-@copydetails JSF_cluster_test_PUT
-
-@CLEARPAGE
-@anchor HttpSystemAdminDescriptionShardingPOST
-@copydetails JSF_cluster_test_POST
-
-@CLEARPAGE
-@anchor HttpSystemAdminDescriptionShardingDELETE
-@copydetails JSF_cluster_test_DELETE
-
-@CLEARPAGE
-@anchor HttpSystemAdminDescriptionShardingPATCH
-@copydetails JSF_cluster_test_PATCH
-
-@CLEARPAGE
-@anchor HttpSystemAdminDescriptionShardingHEAD
-@copydetails JSF_cluster_test_HEAD
 
 @BNAVIGATE_HttpSystem

Documentation/ImplementorManual/HttpSystemTOC.md

@@ -7,9 +7,4 @@ TOC {#HttpSystemTOC}
   - @ref HttpSystemFlushServerModules "POST /_admin/modules/flush"
   - @ref HttpSystemAdminStatistics "GET /_admin/statistics"
   - @ref HttpSystemAdminStatisticsDescription "GET /_admin/statistics-descriptions"
-  - @ref HttpSystemAdminDescriptionShardingGET "GET /_admin/sharding-test"
+  - @ref HttpSystemAdminServerRole "GET /_admin/server/role"
-  - @ref HttpSystemAdminDescriptionShardingPUT "PUT /_admin/sharding-test"
-  - @ref HttpSystemAdminDescriptionShardingPOST "POST /_admin/sharding-test"
-  - @ref HttpSystemAdminDescriptionShardingDELETE "DELETE /_admin/sharding-test"
-  - @ref HttpSystemAdminDescriptionShardingPATCH "PATCH /_admin/sharding-test"
-  - @ref HttpSystemAdminDescriptionShardingHEAD "HEAD /_admin/sharding-test"

Documentation/UserManual/CommandLineTOC.md

@@ -38,7 +38,6 @@ TOC {#CommandLineTOC}
     - @ref CommandLineArangoMaximalJournalSize "database.maximal-journal-size"
     - @ref CommandLineArangoWaitForSync "database.wait-for-sync"
     - @ref CommandLineArangoForceSyncProperties "database.force-sync-properties"
-    - @ref CommandLineArangoRemoveOnDrop "database.remove-on-compacted"
     - @ref CommandLineArangoJsGcFrequency "javascript.gc-frequency"
     - @ref CommandLineArangoJsGcInterval "javascript.gc-interval"
     - @ref CommandLineArangoJsV8Options "javascript.v8-options"

Makefile.am

@@ -213,6 +213,8 @@ pkglibexec_SCRIPTS =
 
 if ENABLE_ALL_IN_ONE_ETCD
 pkglibexec_SCRIPTS += bin/etcd-arango$(PROGRAM_SUFFIX)
+
+CLEANUP += bin/etcd-arango$(PROGRAM_SUFFIX)
 endif
 
 ################################################################################

UnitTests/Makefile.unittests

@@ -392,7 +392,7 @@ SHELL_SERVER_ONLY = \
                @top_srcdir@/js/server/tests/shell-compaction-noncluster.js \
                @top_srcdir@/js/server/tests/shell-transactions-noncluster.js \
                @top_srcdir@/js/server/tests/shell-routing.js \
-               @top_srcdir@/js/server/tests/shell-any.js \
+               @top_srcdir@/js/server/tests/shell-any-noncluster.js \
                @top_srcdir@/js/server/tests/shell-bitarray-index.js \
                @top_srcdir@/js/server/tests/shell-database-noncluster.js \
                @top_srcdir@/js/server/tests/shell-foxx.js \

arangod/Ahuacatl/ahuacatl-grammar.h

@@ -1,8 +1,8 @@
-/* A Bison parser, made by GNU Bison 2.5.  */
+/* A Bison parser, made by GNU Bison 3.0.2.  */
 
 /* Bison interface for Yacc-like parsers in C
 
-      Copyright (C) 1984, 1989-1990, 2000-2011 Free Software Foundation, Inc.
+   Copyright (C) 1984, 1989-1990, 2000-2013 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -30,13 +30,21 @@
    This special exception was added by the Free Software Foundation in
    version 2.2 of Bison.  */
 
+#ifndef YY_AHUACATL_ARANGOD_AHUACATL_AHUACATL_GRAMMAR_H_INCLUDED
+# define YY_AHUACATL_ARANGOD_AHUACATL_AHUACATL_GRAMMAR_H_INCLUDED
+/* Debug traces.  */
+#ifndef YYDEBUG
+# define YYDEBUG 0
+#endif
+#if YYDEBUG
+extern int Ahuacatldebug;
+#endif
 
-/* Tokens.  */
+/* Token type.  */
 #ifndef YYTOKENTYPE
 # define YYTOKENTYPE
-   /* Put the tokens into the symbol table, so that GDB and other debuggers
+  enum yytokentype
-      know about them.  */
+  {
-   enum yytokentype {
     T_END = 0,
     T_FOR = 258,
     T_LET = 259,
@@ -84,52 +92,48 @@
     T_DOC_CLOSE = 301,
     T_LIST_OPEN = 302,
     T_LIST_CLOSE = 303,
-     UPLUS = 304,
+    UMINUS = 304,
-     UMINUS = 305,
+    UPLUS = 305,
     FUNCCALL = 306,
     REFERENCE = 307,
     INDEXED = 308
   };
 #endif
 
-
+/* Value type.  */
-
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef union YYSTYPE
+typedef union YYSTYPE YYSTYPE;
+union YYSTYPE
 {
-
+#line 26 "arangod/Ahuacatl/ahuacatl-grammar.y" /* yacc.c:1915  */
-/* Line 2132 of yacc.c  */
-#line 26 "arangod/Ahuacatl/ahuacatl-grammar.y"
 
   TRI_aql_node_t* node;
   char* strval;
   bool boolval;
   int64_t intval;
 
-
+#line 116 "arangod/Ahuacatl/ahuacatl-grammar.h" /* yacc.c:1915  */
-
+};
-/* Line 2132 of yacc.c  */
-#line 113 "arangod/Ahuacatl/ahuacatl-grammar.h"
-} YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
 # define YYSTYPE_IS_DECLARED 1
 #endif
 
-
+/* Location type.  */
-
 #if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
-typedef struct YYLTYPE
+typedef struct YYLTYPE YYLTYPE;
+struct YYLTYPE
 {
   int first_line;
   int first_column;
   int last_line;
   int last_column;
-} YYLTYPE;
+};
-# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
 # define YYLTYPE_IS_DECLARED 1
 # define YYLTYPE_IS_TRIVIAL 1
 #endif
 
 
 
+int Ahuacatlparse (TRI_aql_context_t* const context);
+
+#endif /* !YY_AHUACATL_ARANGOD_AHUACATL_AHUACATL_GRAMMAR_H_INCLUDED  */

arangod/RestServer/ArangoServer.cpp

@@ -313,7 +313,6 @@ ArangoServer::ArangoServer (int argc, char** argv)
     _unusedForceSyncShapes(false),
     _disableReplicationLogger(false),
     _disableReplicationApplier(false),
-    _removeOnCompacted(true),
     _removeOnDrop(true),
     _server(0) {
 
@@ -529,10 +528,6 @@ void ArangoServer::buildApplicationServer () {
     ("database.force-sync-properties", &_forceSyncProperties, "force syncing of collection properties to disk, will use waitForSync value of collection when turned off")
   ;
 
-  additional["DATABASE Options:help-devel"]
-    ("database.remove-on-compacted", &_removeOnCompacted, "wipe a datafile from disk after compaction")
-  ;
-
   // deprecated options
   additional[ApplicationServer::OPTIONS_HIDDEN]
     ("database.force-sync-shapes", &_unusedForceSyncShapes, "force syncing of shape data to disk, will use waitForSync value of collection when turned off (deprecated)")
@@ -1016,7 +1011,6 @@ void ArangoServer::openDatabases () {
   // override with command-line options
   defaults.defaultMaximalSize               = _defaultMaximalSize;
   defaults.removeOnDrop                     = _removeOnDrop;
-  defaults.removeOnCompacted                = _removeOnCompacted;
   defaults.defaultWaitForSync               = _defaultWaitForSync;
   defaults.forceSyncProperties              = _forceSyncProperties;
   defaults.requireAuthentication            = ! _disableAuthentication;

arangod/RestServer/ArangoServer.h

@@ -449,20 +449,6 @@ namespace triagens {
 
         bool _disableReplicationApplier;
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief remove on compaction
-///
-/// @CMDOPT{\--database.remove-on-compaction @CA{flag}}
-///
-/// Normally the garbage collection will removed compacted datafile. For debug
-/// purposes you can use this option to keep the old datafiles. You should
-/// never set it to @LIT{false} on a live system.
-///
-/// The default is @LIT{true}.
-////////////////////////////////////////////////////////////////////////////////
-
-        bool _removeOnCompacted;
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief remove on drop
 ///

arangod/V8Server/v8-vocbase.cpp

@@ -9490,7 +9490,6 @@ static v8::Handle<v8::Value> JS_CreateDatabase (v8::Arguments const& argv) {
   TRI_GetDatabaseDefaultsServer((TRI_server_t*) v8g->_server, &defaults);
   
   v8::Local<v8::String> keyRemoveOnDrop = v8::String::New("removeOnDrop");
-  v8::Local<v8::String> keyRemoveOnCompacted = v8::String::New("removeOnCompacted");
   v8::Local<v8::String> keyDefaultMaximalSize = v8::String::New("defaultMaximalSize");
   v8::Local<v8::String> keyDefaultWaitForSync = v8::String::New("defaultWaitForSync");
   v8::Local<v8::String> keyForceSyncProperties = v8::String::New("forceSyncProperties");
@@ -9506,10 +9505,6 @@ static v8::Handle<v8::Value> JS_CreateDatabase (v8::Arguments const& argv) {
       defaults.removeOnDrop = options->Get(keyRemoveOnDrop)->BooleanValue();
     }
 
-    if (options->Has(keyRemoveOnCompacted)) {
-      defaults.removeOnCompacted = options->Get(keyRemoveOnCompacted)->BooleanValue();
-    }
-
     if (options->Has(keyDefaultMaximalSize)) {
       defaults.defaultMaximalSize = (TRI_voc_size_t) options->Get(keyDefaultMaximalSize)->IntegerValue();
     }

arangod/VocBase/compactor.c

@@ -77,12 +77,32 @@
 
 #define COMPACTOR_MAX_FILES 4
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief maximum multiple of journal filesize of a compacted file
+/// a value of 3 means that the maximum filesize of the compacted file is
+/// 3 x (collection->journalSize)
+////////////////////////////////////////////////////////////////////////////////
+
+#define COMPACTOR_MAX_SIZE_FACTOR (3)
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief maximum filesize of resulting compacted file
+////////////////////////////////////////////////////////////////////////////////
+
+#define COMPACTOR_MAX_RESULT_FILESIZE (128 * 1024 * 1024)
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief datafiles smaller than the following value will be merged with others
 ////////////////////////////////////////////////////////////////////////////////
 
 #define COMPACTOR_MIN_SIZE (128 * 1024)
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief re-try compaction of a specific collection in this interval (in s)
+////////////////////////////////////////////////////////////////////////////////
+
+#define COMPACTOR_COLLECTION_INTERVAL (15.0)
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief compactify interval in microseconds
 ////////////////////////////////////////////////////////////////////////////////
@@ -266,6 +286,7 @@ static void DropDatafileCallback (TRI_datafile_t* datafile, void* data) {
   TRI_FreeString(TRI_CORE_MEM_ZONE, name);
 
   if (datafile->isPhysical(datafile)) {
+    // copy the current filename
     copy = TRI_DuplicateStringZ(TRI_CORE_MEM_ZONE, datafile->_filename);
 
     ok = TRI_RenameDatafile(datafile, filename);
@@ -288,7 +309,6 @@ static void DropDatafileCallback (TRI_datafile_t* datafile, void* data) {
               TRI_last_error());
   }
   else if (datafile->isPhysical(datafile)) {
-    if (primary->base._vocbase->_settings.removeOnCompacted) {
      int res;
 
      LOG_DEBUG("wiping compacted datafile from disk");
@@ -316,7 +336,6 @@ static void DropDatafileCallback (TRI_datafile_t* datafile, void* data) {
       }
     }
   }
-  }
 
   TRI_FreeDatafile(datafile);
   TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
@@ -1031,6 +1050,7 @@ static void CompactifyDatafiles (TRI_document_collection_t* document,
 static bool CompactifyDocumentCollection (TRI_document_collection_t* document) {
   TRI_primary_collection_t* primary;
   TRI_vector_t vector;
+  uint64_t maxSize;
   int64_t numAlive;
   size_t i, n;
   bool compactNext;
@@ -1057,6 +1077,15 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) {
     return false;
   }
 
+  // get maximum size of result file
+  maxSize = (uint64_t) COMPACTOR_MAX_SIZE_FACTOR * (uint64_t) primary->base._info._maximalSize;
+  if (maxSize < 8 * 1024 * 1024) {
+    maxSize = 8 * 1024 * 1024;
+  }
+  if (maxSize >= COMPACTOR_MAX_RESULT_FILESIZE) {
+    maxSize = COMPACTOR_MAX_RESULT_FILESIZE;
+  }
+
   // copy datafile information
   TRI_InitVector(&vector, TRI_UNKNOWN_MEM_ZONE, sizeof(compaction_info_t));
   numAlive = 0;
@@ -1065,6 +1094,7 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) {
     TRI_datafile_t* df;
     TRI_doc_datafile_info_t* dfi;
     compaction_info_t compaction;
+    uint64_t totalSize = 0;
     bool shouldCompact;
   
     df = primary->base._datafiles._buffer[i];
@@ -1086,8 +1116,8 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) {
       shouldCompact = true;
       compactNext = true;
     }
-    else if (numAlive == 0 && dfi->_numberDeletion > 0) {
+    else if (numAlive == 0 && dfi->_numberAlive == 0 && dfi->_numberDeletion > 0) {
-      // compact first datafile already if it has got some deletions
+      // compact first datafile(s) already if they have some deletions
       shouldCompact = true;
       compactNext = true;
     }
@@ -1137,6 +1167,8 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) {
               (unsigned long long) dfi->_sizeShapes,
               (unsigned long long) dfi->_sizeAttributes);
     
+    totalSize += (uint64_t) df->_maximalSize;
+
     compaction._datafile = df;
     compaction._keepDeletions = (numAlive > 0 && i > 0);
 
@@ -1149,7 +1181,8 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) {
     // delete the collection in the middle of compaction, but the compactor
     // will not pick this up as it is read-locking the collection status)
 
-    if (TRI_LengthVector(&vector) >= COMPACTOR_MAX_FILES) {
+    if (TRI_LengthVector(&vector) >= COMPACTOR_MAX_FILES ||
+        totalSize >= maxSize) {
       // found enough to compact
       break;
     }
@@ -1458,6 +1491,7 @@ void TRI_CompactorVocBase (void* data) {
     // check if compaction is currently disallowed
     if (CheckAndLockCompaction(vocbase)) {
       // compaction is currently allowed
+      double now = TRI_microtime();
       size_t i, n;
       numCompacted = 0;
 
@@ -1503,6 +1537,7 @@ void TRI_CompactorVocBase (void* data) {
             continue;
           }
 
+          if (primary->_lastCompaction + COMPACTOR_COLLECTION_INTERVAL <= now) {
             ce = TRI_CreateBarrierCompaction(&primary->_barrierList);
 
             if (ce == NULL) {
@@ -1512,8 +1547,15 @@ void TRI_CompactorVocBase (void* data) {
             else {
               worked = CompactifyDocumentCollection((TRI_document_collection_t*) primary);
                 
+              if (! worked) {
+                // set compaction stamp
+                primary->_lastCompaction = now;
+              }
+              // if we worked, then we don't set the compaction stamp to force another round of compaction
+
               TRI_FreeBarrier(ce);
             }
+          }
         
           // read-unlock the compaction lock
           TRI_WriteUnlockReadWriteLock(&primary->_compactionLock);

arangod/VocBase/primary-collection.c

@@ -582,6 +582,7 @@ int TRI_InitPrimaryCollection (TRI_primary_collection_t* primary,
   primary->_capConstraint      = NULL;
   primary->_keyGenerator       = NULL;
   primary->_numberDocuments    = 0;
+  primary->_lastCompaction     = 0.0;
 
   primary->figures             = Figures;
   primary->size                = Count;

arangod/VocBase/primary-collection.h

@@ -311,6 +311,7 @@ typedef struct TRI_primary_collection_s {
 
   int64_t                      _numberDocuments;
   TRI_read_write_lock_t        _compactionLock;
+  double                       _lastCompaction;
 
   int (*beginRead) (struct TRI_primary_collection_s*);
   int (*endRead) (struct TRI_primary_collection_s*);

arangod/VocBase/vocbase-defaults.c

@@ -46,7 +46,6 @@ void TRI_ApplyVocBaseDefaults (TRI_vocbase_t* vocbase,
                                TRI_vocbase_defaults_t const* defaults) {
   vocbase->_settings.defaultMaximalSize               = defaults->defaultMaximalSize;
   vocbase->_settings.removeOnDrop                     = defaults->removeOnDrop;
-  vocbase->_settings.removeOnCompacted                = defaults->removeOnCompacted;
   vocbase->_settings.defaultWaitForSync               = defaults->defaultWaitForSync;
   vocbase->_settings.forceSyncProperties              = defaults->forceSyncProperties;
   vocbase->_settings.requireAuthentication            = defaults->requireAuthentication;    
@@ -69,7 +68,6 @@ TRI_json_t* TRI_JsonVocBaseDefaults (TRI_memory_zone_t* zone,
   }
 
   TRI_Insert3ArrayJson(zone, json, "removeOnDrop", TRI_CreateBooleanJson(zone, defaults->removeOnDrop));
-  TRI_Insert3ArrayJson(zone, json, "removeOnCompacted", TRI_CreateBooleanJson(zone, defaults->removeOnCompacted));
   TRI_Insert3ArrayJson(zone, json, "waitForSync", TRI_CreateBooleanJson(zone, defaults->defaultWaitForSync));
   TRI_Insert3ArrayJson(zone, json, "forceSyncProperties", TRI_CreateBooleanJson(zone, defaults->forceSyncProperties));
   TRI_Insert3ArrayJson(zone, json, "requireAuthentication", TRI_CreateBooleanJson(zone, defaults->requireAuthentication));
@@ -98,12 +96,6 @@ void TRI_FromJsonVocBaseDefaults (TRI_vocbase_defaults_t* defaults,
     defaults->removeOnDrop = optionJson->_value._boolean;
   }
   
-  optionJson = TRI_LookupArrayJson(json, "removeOnCompacted");
-
-  if (TRI_IsBooleanJson(optionJson)) {
-    defaults->removeOnCompacted = optionJson->_value._boolean;
-  }
-  
   optionJson = TRI_LookupArrayJson(json, "waitForSync");
 
   if (TRI_IsBooleanJson(optionJson)) {

arangod/VocBase/vocbase-defaults.h

@@ -54,7 +54,6 @@ struct TRI_vocbase_s;
 typedef struct TRI_vocbase_defaults_s {
   TRI_voc_size_t    defaultMaximalSize;
   bool              removeOnDrop;
-  bool              removeOnCompacted;
   bool              defaultWaitForSync;
   bool              forceSyncProperties;
   bool              requireAuthentication;

arangosh/V8Client/arangosh.cpp

@@ -1667,8 +1667,8 @@ static bool RunJsLint (v8::Handle<v8::Context> context) {
     sysTestFiles->Set((uint32_t) i, v8::String::New(JsLint[i].c_str()));
   }
 
-  TRI_AddGlobalVariableVocbase(context, "SYS_UNIT_TESTS", sysTestFiles);
+  context->Global()->Set(v8::String::New("SYS_UNIT_TESTS"), sysTestFiles);
-  TRI_AddGlobalVariableVocbase(context, "SYS_UNIT_TESTS_RESULT", v8::True());
+  context->Global()->Set(v8::String::New("SYS_UNIT_TESTS_RESULT"), v8::True());
 
   // run tests
   char const* input = "require(\"jslint\").runCommandLineTests({ });";

js/Makefile.files

@@ -70,14 +70,17 @@ JAVASCRIPT_JSLINT = \
 .PHONY: jslint
 
 jslint:
-	@for file in $(JAVASCRIPT_JSLINT); do \
+	@RESULT=0; \
+	for file in $(JAVASCRIPT_JSLINT); do \
 		@builddir@/bin/arangosh \
 			-c none \
 			--log.level error \
 			--server.password "" \
 			--javascript.startup-directory @srcdir@/js \
 			--jslint $$file; \
-	done
+                if [ "$$?x" != "0x" ]; then RESULT=1; fi; \
+	done; \
+	exit $$RESULT
 
 ################################################################################
 ### @brief sets up the directories

js/actions/api-system.js

@@ -133,7 +133,27 @@ actions.defineHttp({
 });
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief returns the current server role
+/// @fn JSF_get_admin_server_role
+/// @brief returns the role of a server in a cluster
+///
+/// @RESTHEADER{GET /_admin/server/role,returns the role of a server in a cluster}
+///
+/// @RESTDESCRIPTION
+///
+/// Returns the role of a server in a cluster.
+/// The role is returned in the `role` attribute of the result.
+/// Possible return values for `role` are:
+/// - `COORDINATOR`: the server is a coordinator in a cluster
+/// - `PRIMARY`: the server is a primary database server in a cluster
+/// - `SECONDARY`: the server is a secondary database server in a cluster
+/// - `UNKNOWN`: in a cluster, `UNKNOWN` is returned if the server role cannot be 
+///    determined. On a single server, `UNKNOWN` is the only possible return
+///    value.
+///
+/// @RESTRETURNCODES
+///
+/// @RESTRETURNCODE{200}
+/// Is returned in all cases.
 ////////////////////////////////////////////////////////////////////////////////
 
 actions.defineHttp({

js/apps/system/aardvark/frontend/css/queryView.css

@@ -28,6 +28,7 @@
   border-bottom: 0px;
   height: 27px;
   background-color: #F0F0F0;
+  font-size: 20px;
 }
 
 #editorToolbar {
@@ -301,7 +302,6 @@
 }
 
 #queryDiv .icon_arangodb {
-  font-size: 20px;
   margin-bottom: 5px;
 }
 

js/apps/system/aardvark/frontend/js/templates/documentsView.ejs

@@ -102,7 +102,7 @@
                 <th><input type="text" id="newCapSize" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="The maximal number of documents for the collection.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="The maximal number of documents for the collection.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -113,7 +113,7 @@
                 <th><input type="text" id="newCapByteSize" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="The maximal size of the active document data in the collection. (min = 16384)">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="The maximal size of the active document data in the collection. (min = 16384)">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -128,7 +128,7 @@
                 <th><input type="text" id="newGeoFields" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="A list with one or two attribute paths.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="A list with one or two attribute paths.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -141,7 +141,7 @@
                 </th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="If a geo-spatial index on a location is constructed and geoJson is true, then the order within the list is longitude followed by latitude.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="If a geo-spatial index on a location is constructed and geoJson is true, then the order within the list is longitude followed by latitude.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -154,7 +154,7 @@
                 </th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="If constraint is true, then a geo-spatial constraint is created.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="If constraint is true, then a geo-spatial constraint is created.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -167,7 +167,7 @@
                 </th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="If a geo-spatial constraint is created and ignoreNull is true, then documents with a null in location or at least one null in latitude or longitude are ignored.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="If a geo-spatial constraint is created and ignoreNull is true, then documents with a null in location or at least one null in latitude or longitude are ignored.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -183,7 +183,7 @@
                 <th><input type="text" id="newHashFields" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="A list of attribute paths.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="A list of attribute paths.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -196,7 +196,7 @@
                 </th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="If true, then create a unique index.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="If true, then create a unique index.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -211,7 +211,7 @@
                 <th><input type="text" id="newFulltextFields" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title='A list of attribute names. Currently, the list is limited to exactly one attribute, so the value of fields should look like this for example: [ "text" ].'>
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title='A list of attribute names. Currently, the list is limited to exactly one attribute, so the value of fields should look like this for example: [ "text" ].'>
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -222,7 +222,7 @@
                 <th><input type="text" id="newFulltextMinLength" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="Minimum character length of words to index. Will default to a server-defined value if unspecified. It is thus recommended to set this value explicitly when creating the index.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="Minimum character length of words to index. Will default to a server-defined value if unspecified. It is thus recommended to set this value explicitly when creating the index.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -239,7 +239,7 @@
                 <th><input type="text" id="newSkiplistFields" value=""/></th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="A list of attribute paths.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="A list of attribute paths.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>
@@ -252,7 +252,7 @@
                 </th>
                 <th class="tooltipInfoTh">
                   <div>
-                    <a class="indexTooltip" data-toggle="tooltip" data-placement="left" title="If true, then create a unique index.">
+                    <a class="index-tooltip" data-toggle="tooltip" data-placement="left" title="If true, then create a unique index.">
                       <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
                     </a>
                   </div>

js/apps/system/aardvark/frontend/js/templates/userBarView.ejs

@@ -3,7 +3,7 @@
 
   <li class="dropdown user-menu userImg">
     <a href="#" class="tab userImg" id="user" >
-      <img class="userMenuImg" src="<%=img%>" id="userimage" /> <b class="caret"></b>
+      <img class="user-menu-img" src="<%=img%>" id="userimage" /> <b class="caret"></b>
     </a>
     <ul class="user-dropdown-menu" id="user_dropdown">
       <li class="dropdown-header" style="text-transform: none">

js/apps/system/aardvark/frontend/js/templates/userProfileView.ejs

@@ -44,6 +44,14 @@
         <tr>
           <th>Gravatar account (Mail):</th>
           <th><input type="text" id="editUserProfileImg" name="img" value="" placeholder="myAccount@gravatar.com"/></th>
+          <th class="tooltipInfoTh">
+            <div>
+              <a class="modalInfoTooltips" data-toggle="tooltip" data-placement="left"
+                 title="Mailaddress or its md5 representation of your gravatar account. The address will be converted into a md5 string. Only the md5 string will be stored, not the mailaddress.">
+                <span rel="tooltip" class="arangoicon icon_arangodb_info"></span>
+              </a>
+            </div>
+          </th>
         </tr>
       </table>
     </div>

js/apps/system/aardvark/frontend/js/views/userProfileView.js

@@ -30,6 +30,13 @@
         username : this.user.get("user")
 
       }));
+
+      $("[data-toggle=tooltip]").tooltip();
+
+      $('.modalInfoTooltips').tooltip({
+        placement: "left"
+      });
+
       return this;
     },
 

js/apps/system/aardvark/frontend/scss/_buttons.scss

@@ -10,8 +10,8 @@
   @extend %bare-btn;
   border: 0;
   color: $c-white;
-  margin-left: 10px;
   padding: 5px 16px;
+  margin-left: 10px;
 }
 
 .button-neutral {

js/apps/system/aardvark/frontend/scss/_tooltips.scss

@@ -4,7 +4,7 @@
   word-wrap: break-word !important;
 }
 
-.indexTooltip {
+.index-tooltip {
   color: $c-btn-inverse;
 
   &:hover {

js/apps/system/aardvark/frontend/scss/_userMenu.scss

@@ -1,4 +1,4 @@
-.userMenuImg {
+.user-menu-img {
   background-color: $c-inactive;
   @include border-radius(3px);
   border: 2px solid $c-positive;

js/apps/system/aardvark/frontend/scss/generated.css

@@ -83,7 +83,7 @@ nav.navbar, footer.footer {
 .button-inactive {
   background-color: lightgrey; }
   .button-inactive:hover {
-    background-color: grey; }
+    background-color: gray; }
 
 ul.link-dropdown-menu, ul.user-dropdown-menu, ul.gv-dropdown-menu {
   -moz-border-radius: 3px;
@@ -1007,7 +1007,7 @@ select.filterSelect {
 .createModalDialog table {
   width: 100%; }
 
-.userMenuImg {
+.user-menu-img {
   background-color: lightgrey;
   -moz-border-radius: 3px;
   -webkit-border-radius: 3px;
@@ -1020,11 +1020,11 @@ select.filterSelect {
   white-space: normal !important;
   word-wrap: break-word !important; }
 
-.indexTooltip {
+.index-tooltip {
   color: #736b68; }
-  .indexTooltip:hover {
+  .index-tooltip:hover {
     color: black; }
-  .indexTooltip span.arangoicon {
+  .index-tooltip span.arangoicon {
     font-size: 18px !important; }
 
 .dbselection {

js/apps/system/aardvark/test/specs/views/collectionsViewSpec.js

@@ -70,11 +70,11 @@
       it("Check if System Collections should be drawn", function() {
         myView.render();
         var wasRendered;
-        $('#checkSystem').click();
 
         spyOn(myView, 'render').andCallFake(function(request) {
           wasRendered = true;
         });
+        $('#checkSystem').click();
         myView.checkSystem();
         expect(wasRendered).toBeTruthy();
       });
@@ -93,11 +93,11 @@
       it("Check if Edge Collections should be drawn", function() {
         myView.render();
         var wasRendered;
-        $('#checkEdge').click();
-
         spyOn(myView, 'render').andCallFake(function(request) {
           wasRendered = true;
         });
+
+        $('#checkEdge').click();
         myView.checkEdge();
         expect(wasRendered).toBeTruthy();
       });
@@ -116,11 +116,11 @@
       it("Check if Document Collections should be drawn", function() {
         myView.render();
         var wasRendered;
-        $('#checkDocument').click();
-
         spyOn(myView, 'render').andCallFake(function(request) {
           wasRendered = true;
         });
+
+        $('#checkDocument').click();
         myView.checkDocument();
         expect(wasRendered).toBeTruthy();
       });
@@ -139,11 +139,11 @@
       it("Check if Loaded Collections should be drawn", function() {
         myView.render();
         var wasRendered;
-        $('#checkLoaded').click();
-
         spyOn(myView, 'render').andCallFake(function(request) {
           wasRendered = true;
         });
+
+        $('#checkLoaded').click();
         myView.checkLoaded();
         expect(wasRendered).toBeTruthy();
       });
@@ -162,13 +162,15 @@
       it("Check if Unloaded Collections should be drawn", function() {
         myView.render();
         var wasRendered;
-        $('#checkUnloaded').click();
-
         spyOn(myView, 'render').andCallFake(function(request) {
           wasRendered = true;
         });
-        myView.checkUnloaded();
+
+        $('#checkUnloaded').click();
         expect(wasRendered).toBeTruthy();
+        wasRendered =false;
+        myView.checkUnloaded();
+        expect(wasRendered).toBeFalsy();
       });
 
       it("Check if Name Sort is inactive", function() {

js/server/modules/org/arangodb/cluster/kickstarter.js

@@ -338,9 +338,10 @@ launchActions.createSystemColls = function (dispatchers, cmd, isRelaunch) {
   url = cmd.url + "/_admin/execute?returnAsJSON=true";
   var body = 'load=require("internal").load;\n'+
              'UPGRADE_ARGS=undefined;\n'+
-             'return load("'+fs.join(ArangoServerState.javaScriptPath(),
+             'return load('+JSON.stringify(
-                                     makePath("server/version-check.js"))+
+                                fs.join(ArangoServerState.javaScriptPath(),
-             '");\n';
+                                        makePath("server/version-check.js")))+
+             ');\n';
   var o = { method: "POST", timeout: 90, headers: hdrs };
   r = download(url, body, o);
   if (r.code === 200) {

js/server/tests/shell-any-cluster.js

@@ -0,0 +1,202 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test the random document selector 
+///
+/// @file
+///
+/// DISCLAIMER
+///
+/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is triAGENS GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+var jsunity = require("jsunity");
+
+var arangodb = require("org/arangodb");
+var db = arangodb.db;
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                               any
+// -----------------------------------------------------------------------------
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test suite
+////////////////////////////////////////////////////////////////////////////////
+
+function AnySuite () {
+  var cn = "example";
+  var c;
+
+  var threshold = 15; // maximum tolerated stddev for distribution
+
+  var stddev = function (dist) {
+    var v;
+    var sum = 0;
+    var count = 0;
+
+    for (v in dist) {
+      if (dist.hasOwnProperty(v)) {
+        sum += dist[v];
+        count++;
+      }
+    }
+
+    var avg = sum / count;
+    var sum2 = 0;
+
+    for (v in dist) {
+      if (dist.hasOwnProperty(v)) {
+        var d = dist[v] - avg;
+        sum2 += d * d;
+      }
+    }
+
+    return Math.sqrt(sum2 / count);
+  };
+
+  var getDistribution = function (n, rng) {
+    var dist = { };
+    var i;
+
+    for (i = 0; i < n; ++i) {
+      var pick = rng();
+
+      if (dist.hasOwnProperty(pick)) {
+        dist[pick]++;
+      }
+      else {
+        dist[pick] = 1;
+      }
+    }
+
+    return dist;
+  };
+
+  return {
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set up
+////////////////////////////////////////////////////////////////////////////////
+
+    setUp : function () {
+      db._drop(cn);
+      c = db._create(cn);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief tear down
+////////////////////////////////////////////////////////////////////////////////
+
+    tearDown : function () {
+      db._drop(cn);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief check entropy of Math.random()
+////////////////////////////////////////////////////////////////////////////////
+
+    testCheckEntropyNative : function () {
+      var i, n;
+
+      n = 100;
+
+      for (i = 0; i < n; ++i) {
+        c.save({ value: i });
+      }
+
+      var dist = getDistribution(n * 100, function () {
+        return parseInt(Math.random() * 100, 10);
+      });
+
+      assertTrue(stddev(dist) < threshold);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief check entropy of any(), just one document
+////////////////////////////////////////////////////////////////////////////////
+
+    testCheckEntropyCollectionOne : function () {
+      c.save({ value: 1 });
+
+      var dist = getDistribution(100, function () {
+        return c.any().value;
+      });
+      
+      assertTrue(stddev(dist) <= 0.01);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief check entropy of any(), few picks
+////////////////////////////////////////////////////////////////////////////////
+
+    testCheckEntropyCollectionFew1 : function () {
+      var i, n;
+
+      n = 3;
+
+      for (i = 0; i < n; ++i) {
+        c.save({ value: i });
+      }
+
+      var dist = getDistribution(n * 200, function () {
+        return c.any().value;
+      });
+      
+      assertTrue(stddev(dist) < threshold * 3.0);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief check entropy of any(), few picks
+////////////////////////////////////////////////////////////////////////////////
+
+    testCheckEntropyCollectionFew2 : function () {
+      var i, n;
+
+      n = 10;
+
+      for (i = 0; i < n; ++i) {
+        c.save({ value: i });
+      }
+
+      var dist = getDistribution(n * 100, function () {
+        return c.any().value;
+      });
+      
+      assertTrue(stddev(dist) < threshold * 1.5);
+    },
+
+  };
+}
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                              main
+// -----------------------------------------------------------------------------
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief executes the test suite
+////////////////////////////////////////////////////////////////////////////////
+
+jsunity.run(AnySuite);
+
+return jsunity.done();
+
+// Local Variables:
+// mode: outline-minor
+// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)"
+// End:
+

js/server/version-check.js

@@ -1,4 +1,4 @@
-/*jslint indent: 2, nomen: true, maxlen: 100, sloppy: true, vars: true, white: true, plusplus: true, stupid: true, continue: true, regexp: true */
+/*jslint indent: 2, nomen: true, maxlen: 100, sloppy: true, vars: true, white: true, plusplus: true, stupid: true, continue: true, regexp: true nonpropdel: true*/
 /*global require, exports, module, UPGRADE_ARGS, UPGRADE_STARTED: true */
 
 ////////////////////////////////////////////////////////////////////////////////

lib/BasicsC/socket-utils.c

@@ -73,14 +73,19 @@ int TRI_closesocket (TRI_socket_t s) {
     if (res != 0) {
       LOG_WARNING("socket close error: %d", WSAGetLastError());
     }
-    if (s.fileDescriptor != -1) {
+    // We patch libev on Windows lightly to not really distinguish between
-      res = _close(s.fileDescriptor);
+    // socket handles and file descriptors, therefore, we do not have to do the 
-      /*
+    // following any more:
-      To close a file opened with _open_osfhandle, call _close.
+    // if (s.fileDescriptor != -1) {
-      The underlying handle is also closed by a call to _close,
+    //   res = _close(s.fileDescriptor);
-      so it is not necessary to call the Win32 function CloseHandle on the original handle.
+         // "To close a file opened with _open_osfhandle, call _close."
-      */
+         // The underlying handle is also closed by a call to _close,
-    }
+         // so it is not necessary to call the Win32 function CloseHandle 
+         // on the original handle.
+    // However, we do want to do the special shutdown/recv magic above
+    // because only then we can reuse the port quickly, which we want
+    // to do directly after a port test.
+    // }
   }
 #else
     if (s.fileDescriptor != TRI_INVALID_SOCKET) {

lib/Rest/Version.cpp

@@ -77,6 +77,7 @@ void Version::initialise () {
   Values["build-date"] = getBuildDate();
   Values["repository-version"] = getRepositoryVersion();
   Values["sizeof int"] = triagens::basics::StringUtils::itoa(sizeof(int));
+  Values["sizeof void*"] = triagens::basics::StringUtils::itoa(sizeof(void*));
 }
 
 ////////////////////////////////////////////////////////////////////////////////

lib/Scheduler/ListenTask.cpp

@@ -113,7 +113,13 @@ bool ListenTask::setup (Scheduler* scheduler, EventLoop loop) {
     return false;
   }
 
-  int res = _open_osfhandle (_listenSocket.fileHandle, 0);
+  // For the official version of libev we would do this:
+  // int res = _open_osfhandle (_listenSocket.fileHandle, 0);
+  // However, this opens a whole lot of problems and in general one should
+  // never use _open_osfhandle for sockets.
+  // Therefore, we do the following, although it has the potential to
+  // lose the higher bits of the socket handle:
+  int res = (int)_listenSocket.fileHandle;
 
   if (res == - 1) {
     LOG_ERROR("In ListenTask::setup could not convert socket handle to socket descriptor -- _open_osfhandle(...) failed");

lib/Scheduler/SocketTask.cpp

@@ -429,7 +429,13 @@ bool SocketTask::setup (Scheduler* scheduler, EventLoop loop) {
     return false;
   }
 
-  int res = _open_osfhandle(_commSocket.fileHandle, 0);
+  // For the official version of libev we would do this:
+  // int res = _open_osfhandle(_commSocket.fileHandle, 0);
+  // However, this opens a whole lot of problems and in general one should
+  // never use _open_osfhandle for sockets.
+  // Therefore, we do the following, although it has the potential to
+  // lose the higher bits of the socket handle:
+  int res = (int)_commSocket.fileHandle;
 
   if (res == -1) {
     LOG_ERROR("In SocketTask::setup could not convert socket handle to socket descriptor -- _open_osfhandle(...) failed");