Merge branch 'devel' of https://github.com/triAGENS/ArangoDB into devel

CHANGELOG

@@ -1,6 +1,10 @@
 v1.4
 ----
 
+* added AQL MINUS function
+
+* added AQL UNION_DISTINCT function (more efficient than `UNIQUE(UNION())`)
+
 * updated mruby to 2013-08-22
 
 * issue #587: Add db._create() in help for startup arangosh

Documentation/UserManual/Aql.md

@@ -1111,8 +1111,8 @@ AQL supports the following functions to operate on list values:
   The function expects at least two list values as its arguments. The result is a list
   of values in an undefined order.
 
-  Note: no duplicates will be removed. In order to remove duplicates, please use the
+  Note: no duplicates will be removed. In order to remove duplicates, please use either
-  @LIT{UNIQUE} function.
+  @LIT{UNION_DISTINCT} function or apply the @LIT{UNIQUE} on the result of @LIT{union}.
 
   Example:
     RETURN UNION(
@@ -1135,6 +1135,16 @@ AQL supports the following functions to operate on list values:
   will produce:
     [ [ 1, 2, 3 ] ]
 
+- @FN{UNION_DISTINCT(@FA{list1, list2, ...})}: returns the union of distinct values of
+  all lists specified. The function expects at least two list values as its arguments. 
+  The result is a list of values in an undefined order.
+
+- @FN{MINUS(@FA{list1, list2, ...})}: returns the difference of all lists specified.
+  The function expects at least two list values as its arguments.
+  The result is a list of values that occur in the first list but not in any of the
+  subsequent lists. The order of the result list is undefined and should not be relied on.
+  Note: duplicates will be removed.
+
 - @FN{INTERSECTION(@FA{list1, list2, ...})}: returns the intersection of all lists specified.
   The function expects at least two list values as its arguments.
   The result is a list of values that occur in all arguments. The order of the result list

arangod/Ahuacatl/ahuacatl-functions.c

@@ -645,6 +645,8 @@ TRI_associative_pointer_t* TRI_InitialiseFunctionsAql (void) {
   // list functions
   REGISTER_FUNCTION("RANGE", "RANGE", true, false, "n,n|n", NULL);
   REGISTER_FUNCTION("UNION", "UNION", true, false, "l,l|+", NULL);
+  REGISTER_FUNCTION("UNION_DISTINCT", "UNION_DISTINCT", true, false, "l,l|+", NULL);
+  REGISTER_FUNCTION("MINUS", "MINUS", true, false, "l,l|+", NULL);
   REGISTER_FUNCTION("INTERSECTION", "INTERSECTION", true, false, "l,l|+", NULL);
   REGISTER_FUNCTION("LENGTH", "LENGTH", true, true, "las", NULL);
   REGISTER_FUNCTION("MIN", "MIN", true, true, "l", NULL);

arangod/VocBase/replication-applier.c

@@ -1032,7 +1032,7 @@ int TRI_RemoveStateReplicationApplier (TRI_vocbase_t* vocbase) {
 
 int TRI_SaveStateReplicationApplier (TRI_vocbase_t* vocbase,
                                      TRI_replication_applier_state_t const* state,
-                                     bool sync) {
+                                     bool doSync) {
   TRI_json_t* json;
   char* filename;
   int res;
@@ -1046,7 +1046,7 @@ int TRI_SaveStateReplicationApplier (TRI_vocbase_t* vocbase,
   filename = GetStateFilename(vocbase);
   LOG_TRACE("saving replication applier state to file '%s'", filename);
 
-  if (! TRI_SaveJson(filename, json, sync)) {
+  if (! TRI_SaveJson(filename, json, doSync)) {
     res = TRI_errno();
   }
   else {
@@ -1238,7 +1238,7 @@ int TRI_RemoveConfigurationReplicationApplier (TRI_vocbase_t* vocbase) {
 
 int TRI_SaveConfigurationReplicationApplier (TRI_vocbase_t* vocbase,
                                              TRI_replication_applier_configuration_t const* config,
-                                             bool sync) {
+                                             bool doSync) {
   TRI_json_t* json;
   char* filename;
   int res;
@@ -1251,7 +1251,7 @@ int TRI_SaveConfigurationReplicationApplier (TRI_vocbase_t* vocbase,
 
   filename = GetConfigurationFilename(vocbase);
 
-  if (! TRI_SaveJson(filename, json, sync)) {
+  if (! TRI_SaveJson(filename, json, doSync)) {
     res = TRI_errno();
   }
   else {

js/apps/aardvark/frontend/src/mode-aql.js

@@ -97,7 +97,7 @@ var AqlHighlightRules = function() {
         "like|floor|ceil|round|abs|sqrt|rand|length|min|max|average|sum|median|variance_population|" +
         "variance_sample|first|last|unique|matches|merge|merge_recursive|has|attributes|unset|keep|" +
         "near|within|fulltext|paths|traversal|traversal_tree|edges|not_null|first_list|first_document|" +
-        "collections|document|stddev_population|stddev_sample|neighbors)"
+        "collections|document|stddev_population|stddev_sample|neighbors|union|union_distinct|intersection)"
     );
 
     var keywordMapper = this.createKeywordMapper({

js/server/modules/org/arangodb/ahuacatl.js

@@ -2507,7 +2507,11 @@ function UNIQUE (values) {
 
   values.forEach(function (value) {
     var normalized = NORMALIZE(value);
-    keys[JSON.stringify(normalized)] = normalized;
+    var key = JSON.stringify(normalized);
+
+    if (! keys.hasOwnProperty(key)) {
+      keys[key] = normalized;
+    }
   });
 
   for (a in keys) {
@@ -2526,7 +2530,7 @@ function UNIQUE (values) {
 function UNION () {
   "use strict";
 
-  var result = [ ], i, a;
+  var result = [ ], i;
 
   for (i in arguments) {
     if (arguments.hasOwnProperty(i)) {
@@ -2536,14 +2540,102 @@ function UNION () {
         THROW(INTERNAL.errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, "UNION");
       }
 
-      for (a in element) {
+      var n = element.length, j;
-        if (element.hasOwnProperty(a)) {
+
-          result.push(element[a]);
+      for (j = 0; j < n; ++j) {
+        result.push(element[j]);
+      }
+    }
+  }
+
+  return result; 
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create the union (distinct) of all arguments
+////////////////////////////////////////////////////////////////////////////////
+
+function UNION_DISTINCT () {
+  "use strict";
+
+  var keys = { }, i;
+
+  for (i in arguments) {
+    if (arguments.hasOwnProperty(i)) {
+      var element = arguments[i];
+
+      if (TYPEWEIGHT(element) !== TYPEWEIGHT_LIST) {
+        THROW(INTERNAL.errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, "UNION_DISTINCT");
+      }
+      
+      var n = element.length, j;
+
+      for (j = 0; j < n; ++j) {
+        var normalized = NORMALIZE(element[j]);
+        var key = JSON.stringify(normalized);
+
+        if (! keys.hasOwnProperty(key)) {
+          keys[key] = normalized;
         }
       }
     }
   }
 
+  var result = [ ];
+  for (i in keys) {
+    if (keys.hasOwnProperty(i)) {
+      result.push(keys[i]);
+    }
+  }
+
+  return result; 
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief subtract lists from other lists
+////////////////////////////////////////////////////////////////////////////////
+
+function MINUS () {
+  "use strict";
+
+  var keys = { }, i, first = true;
+
+  for (i in arguments) {
+    if (arguments.hasOwnProperty(i)) {
+      var element = arguments[i];
+
+      if (TYPEWEIGHT(element) !== TYPEWEIGHT_LIST) {
+        THROW(INTERNAL.errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, "MINUS");
+      }
+      
+      var n = element.length, j;
+
+      for (j = 0; j < n; ++j) {
+        var normalized = NORMALIZE(element[j]);
+        var key = JSON.stringify(normalized);
+        var contained = keys.hasOwnProperty(key);
+
+        if (first) {
+          if (! contained) {
+            keys[key] = normalized;
+          }
+        }
+        else if (contained) {
+          delete keys[key];
+        }
+      }
+
+      first = false;
+    }
+  }
+
+  var result = [ ];
+  for (i in keys) {
+    if (keys.hasOwnProperty(i)) {
+      result.push(keys[i]);
+    }
+  }
+
   return result; 
 }
 
@@ -3896,6 +3988,8 @@ exports.REVERSE = REVERSE;
 exports.RANGE = RANGE;
 exports.UNIQUE = UNIQUE;
 exports.UNION = UNION;
+exports.UNION_DISTINCT = UNION_DISTINCT;
+exports.MINUS = MINUS;
 exports.INTERSECTION = INTERSECTION;
 exports.MAX = MAX;
 exports.MIN = MIN;

js/server/tests/ahuacatl-functions.js

@@ -1291,8 +1291,8 @@ function ahuacatlFunctionsTestSuite () {
 ////////////////////////////////////////////////////////////////////////////////
     
     testUnion1 : function () {
-      var expected = [ [ 1, 2, 3, 1, 2, 3 ], [ 1, 2, 3, 1, 2, 3 ] ];
+      var expected = [ [ 1, 2, 3, 1, 2, 3 ] ];
-      var actual = getQueryResults("FOR u IN [ 1, 2 ] return UNION([ 1, 2, 3 ], [ 1, 2, 3 ])");
+      var actual = getQueryResults("RETURN UNION([ 1, 2, 3 ], [ 1, 2, 3 ])");
       assertEqual(expected, actual);
     },
 
@@ -1301,8 +1301,8 @@ function ahuacatlFunctionsTestSuite () {
 ////////////////////////////////////////////////////////////////////////////////
     
     testUnion2 : function () {
-      var expected = [ [ 1, 2, 3, 3, 2, 1 ], [ 1, 2, 3, 3, 2, 1 ] ];
+      var expected = [ [ 1, 2, 3, 3, 2, 1 ] ];
-      var actual = getQueryResults("FOR u IN [ 1, 2 ] return UNION([ 1, 2, 3 ], [ 3, 2, 1 ])");
+      var actual = getQueryResults("RETURN UNION([ 1, 2, 3 ], [ 3, 2, 1 ])");
       assertEqual(expected, actual);
     },
 
@@ -1312,7 +1312,7 @@ function ahuacatlFunctionsTestSuite () {
     
     testUnion3 : function () {
       var expected = [ "Fred", "John", "John", "Amy" ];
-      var actual = getQueryResults("FOR u IN UNION([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"]) return u");
+      var actual = getQueryResults("FOR u IN UNION([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"]) RETURN u");
       assertEqual(expected, actual);
     },
 
@@ -1370,6 +1370,90 @@ function ahuacatlFunctionsTestSuite () {
       assertEqual(expected, actual);
     },
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testUnionDistinct1 : function () {
+      var expected = [ [ 1, 2, 3, ] ];
+      var actual = getQueryResults("RETURN UNION_DISTINCT([ 1, 2, 3 ], [ 1, 2, 3 ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testUnionDistinct2 : function () {
+      var expected = [ [ 1, 2, 3 ] ];
+      var actual = getQueryResults("RETURN UNION_DISTINCT([ 1, 2, 3 ], [ 3, 2, 1 ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testUnionDistinct3 : function () {
+      var expected = [ "Fred", "John", "Amy" ];
+      var actual = getQueryResults("FOR u IN UNION_DISTINCT([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"]) RETURN u");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testUnionDistinct4 : function () {
+      var expected = [ [ 1, 2, 3, 4, 5, 6 ] ];
+      var actual = getQueryResults("RETURN UNION_DISTINCT([ 1, 2, 3 ], [ 3, 2, 1 ], [ 4 ], [ 5, 6, 1 ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testUnionDistinct5 : function () {
+      var expected = [ [ ] ];
+      var actual = getQueryResults("RETURN UNION_DISTINCT([ ], [ ], [ ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testUnionDistinct6 : function () {
+      var expected = [ [ false, true ] ];
+      var actual = getQueryResults("RETURN UNION_DISTINCT([ ], [ false ], [ ], [ true ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test union_distinct function
+////////////////////////////////////////////////////////////////////////////////
+
+    testUnionDistinctInvalid : function () {
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN UNION_DISTINCT()"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN UNION_DISTINCT([ ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], null)"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], true)"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], 3)"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], \"yes\")"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], { })"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], [ ], null)"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], [ ], true)"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], [ ], 3)"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], [ ], \"yes\")"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT([ ], [ ], { })"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT(null, [ ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT(true, [ ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT(3, [ ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT(\"yes\", [ ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT({ }, [ ])"); 
+    },
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test range function
 ////////////////////////////////////////////////////////////////////////////////
@@ -1482,6 +1566,66 @@ function ahuacatlFunctionsTestSuite () {
       assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN RANGE(-1, 1, -1)");
     },
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test minus function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testMinus1 : function () {
+      var expected = [ [ 'b', 'd' ] ];
+      var actual = getQueryResults("RETURN MINUS([ 'a', 'b', 'c', 'd' ], [ 'c' ], [ 'a', 'c', 'e' ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test minus function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testMinus2 : function () {
+      var expected = [ [ 'a', 'b' ] ];
+      var actual = getQueryResults("RETURN MINUS([ 'a', 'b', 'c' ], [ 'c', 'd', 'e' ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test minus function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testMinus3 : function () {
+      var expected = [ [ 'a', 'b', 'c' ] ];
+      var actual = getQueryResults("RETURN MINUS([ 'a', 'b', 'c' ], [ 1, 2, 3 ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test minus function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testMinus4 : function () {
+      var expected = [ [ 'a', 'b', 'c' ] ];
+      var actual = getQueryResults("RETURN MINUS([ 'a', 'b', 'c' ], [ 1, 2, 3 ], [ 1, 2, 3 ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test minus function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testMinus5 : function () {
+      var expected = [ [ ] ];
+      var actual = getQueryResults("RETURN MINUS([ 2 ], [ 'a', 'b', 'c' ], [ 1, 2, 3 ], [ 1, 2, 3 ])");
+      assertEqual(expected, actual);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test minus function
+////////////////////////////////////////////////////////////////////////////////
+    
+    testMinus6 : function () {
+      var expected = [ [ ] ];
+      var actual = getQueryResults("RETURN MINUS([ ], [ 'a', 'b', 'c' ], [ 1, 2, 3 ], [ 1, 2, 3 ])");
+      assertEqual(expected, actual);
+    },
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test intersect function
 ////////////////////////////////////////////////////////////////////////////////