Merge branch 'devel' of github.com:triAGENS/ArangoDB into devel

CHANGELOG

@@ -21,6 +21,8 @@ v2.3.0 (XXXX-XX-XX)
   JSON directly. If authentication is turned on, the link requires authentication,
   too.
 
+* documentation updates
+
 
 v2.3.0-beta1 (2014-11-01)
 -------------------------

Documentation/Books/Users/Aql/Basics.mdpp

@@ -46,17 +46,17 @@ AQL supports two types of comments:
 !SUBSECTION Keywords
 
 On the top level, AQL offers the following operations:
-- FOR: list iteration
+- `FOR`: list iteration
-- RETURN: results projection
+- `RETURN`: results projection
-- FILTER: results filtering
+- `FILTER`: results filtering
-- SORT: result sorting
+- `SORT`: result sorting
-- LIMIT: result slicing
+- `LIMIT`: result slicing
-- LET: variable assignment
+- `LET`: variable assignment
-- COLLECT: result grouping
+- `COLLECT`: result grouping
-- INSERT: insertion of new documents
+- `INSERT`: insertion of new documents
-- UPDATE: (partial) update of existing documents
+- `UPDATE`: (partial) update of existing documents
-- REPLACE: replacement of existing documents
+- `REPLACE`: replacement of existing documents
-- REMOVE: removal of existing documents
+- `REMOVE`: removal of existing documents
 
 Each of the above operations can be initiated in a query by using a keyword of
 the same name. An AQL query can (and typically does) consist of multiple of the

Documentation/Books/Users/Aql/Operators.mdpp

@@ -19,8 +19,7 @@ The following comparison operators are supported:
 - *IN* test if a value is contained in a list
 - *NOT IN* test if a value is not contained in a list
 
-The *IN* and *NOT IN* operators expect the second operand to be of type list. 
+These operators accept any data types for the first and second operands.
-All other operators accept any data types for the first and second operands.
 
 Each of the comparison operators returns a boolean value if the comparison can
 be evaluated and returns *true* if the comparison evaluates to true, and *false*
@@ -29,49 +28,131 @@ otherwise.
 Some examples for comparison operations in AQL:
 
 ```
-1 > 0
+1 > 0                     // true
-true != null
+true != null              // true
-45 <= "yikes!"
+45 <= "yikes!"            // true
-65 != "65"
+65 != "65"                // true
-65 == 65
+65 == 65                  // true
-1.23 < 1.32
+1.23 > 1.32               // false
-1.5 IN [ 2, 3, 1.5 ]
+1.5 IN [ 2, 3, 1.5 ]      // true
-42 NOT IN [ 17, 40, 50 ]
+"foo" IN null             // false
+42 NOT IN [ 17, 40, 50 ]  // true
 ```
 
 !SUBSUBSECTION Logical operators
 
-Logical operators combine two boolean operands in a logical operation and return
+The following logical operators are supported in AQL:
-a boolean result value.
-
-The following logical operators are supported:
 
 - *&&* logical and operator
 - *||* logical or operator
 - *!* logical not/negation operator
 
-Some examples for logical operations in AQL:
+AQL also supports the following alternative forms for the logical operators:
-
-    u.age > 15 && u.address.city != ""
-    true || false
-    !u.isInvalid
-
-The *&&*, *||*, and *!* operators expect their input operands to be boolean
-values each. If a non-boolean operand is used, the operation will fail with an
-error. In case all operands are valid, the result of each logical operator is a
-boolean value.
-
-Both the *&&* and *||* operators use short-circuit evaluation and only evaluate
-the second operand if the result of the operation cannot be determined by
-checking the first operand alone.
-
-ArangoDB also supports the following alternative forms for the logical operators:
 
 - *AND* logical and operator
 - *OR* logical or operator
 - *NOT* logical not/negation operator
 
-The alternative forms are functionally equivalent to the regular operators.
+The alternative forms are aliases and functionally equivalent to the regular 
+operators.
+
+The two-operand logical operators in AQL will be executed with short-circuit 
+evaluation. The result of the logical operators in AQL is defined as follows:
+
+- `lhs && rhs` will return `lhs` if it is `false` or would be `false` when converted
+  into a boolean. If `lhs` is `true` or would be `true` when converted to a boolean,
+  `rhs` will be returned.
+- `lhs || rhs` will return `lhs` if it is `true` or would be `true` when converted
+  into a boolean. If `lhs` is `false` or would be `false` when converted to a boolean,
+  `rhs` will be returned.
+- `! value` will return the negated value of `value` converted into a boolean
+
+Some examples for logical operations in AQL:
+
+    u.age > 15 && u.address.city != ""
+    true || false
+    ! u.isInvalid
+    1 || ! 0
+
+Older versions of ArangoDB required the operands of all logical operators to
+be boolean values and failed when non-boolean values were passed into the 
+operators. Additionally, the result of any logical operation always was a
+boolean value.
+
+This behavior has changed in ArangoDB 2.3. Passing non-boolean values to a
+logical operator is now allowed. Any-non boolean operands will be casted
+to boolean implicity by the operator, without making the query abort. The
+result of logical and and logical or operations can now have any data type and
+it not necessarily a boolean value.
+
+For example, the following logical operations will return a boolean values:
+
+    25 > 1 && 42 != 7                          // true
+    22 IN [ 23, 42 ] || 23 NOT IN [ 22, 7 ]    // true
+    25 != 25                                   // false
+
+whereas the following logical operations will not return boolean values:
+
+    1 || 7                                     // 1
+    null || "foo"                              // "foo"
+    null && true                               // null
+    true && 23                                 // 23
+   
+
+!SUBSUBSECTION Type conversion
+
+In some cases, an operator needs to perform an implicit type conversion of 
+its operand. For example, the logical negation operator (`!`) will cast its
+operand to a boolean if it is not already a boolean.
+
+The arithmetic operators will also cast their operands to numbers before
+performing the arithmetic operation.
+
+The *conversion to a boolean value* works as follows:
+- `null` will be converted to `false`
+- boolean values remain unchanged
+- all numbers unequal to zero are `true`, zero is `false`
+- the empty string is `false`, all other strings are `true`
+- lists (`[ ]`) and documents (`{ }`) are `true`, regardless of their contents
+
+The *conversion to a numeric value* works as follows:
+- `null` will be converted to `0`
+- `false` will be converted to `0`, true will be converted to `1`
+- a numeric value remains unchanged - NaN and Infinity are converted to `null`
+- string values are converted to a number if they contain a valid string representation
+  of a number. Any whitespace at the start or the end of the string is ignored. Strings
+  with any other contents are converted to `null`
+- an empty list is converted to `0`, a list with one member is converted to the numeric
+  representation of this one list member, and lists with more members are converted 
+  to `null`
+- documents are converted to `null`
+
+The *conversion to a string value* works as follows:
+- `null` will be converted to the string `"null"`
+- `false` and `true` will be converted to the strings `"false"` and `"true"` resp.
+- numbers will be converted into strings. Scientific notation may be used
+- a string value remains unchanged
+- an empty list will be converted into the empty string, a list with a single member
+  will be converted into the string representation of this one list member. A list
+  with more than one member will be converted into a comma-separated list of the
+  string representations of the list's members
+- documents will be converted to the string literal `"[object Object]"`
+
+The *conversion to a list* works as follows:
+- `null` will be converted to the empty list (`[ ]`)
+- a boolean value will be converted to a single-member list with the boolean value
+- a numeric value will be converted to a single-member list with the number value
+- a string value will be converted to a single-member list with the string value
+- a list value remains unchanged
+- a numeric value will be conve
+- `false` and `true` will be converted to the strings `"false"` and `"true"` resp.
+- numbers will be converted into strings. Scientific notation may be used.
+- strings will keep their value
+- an empty list will be converted into the empty string, a list with a single member
+  will be converted into the string representation of this one list member. A list
+  with more than one member will be converted into a comma-separated list of the
+  string representations of the list's members.
+- documents will be converted to the string literal `"[object Object]"`
 
 !SUBSUBSECTION Arithmetic operators
 

UnitTests/Makefile.unittests

@@ -554,6 +554,7 @@ SHELL_SERVER_AQL = @top_srcdir@/js/server/tests/aql-arithmetic.js \
 			@top_srcdir@/js/server/tests/aql-optimizer-rule-remove-unnecessary-calculations.js \
 			@top_srcdir@/js/server/tests/aql-optimizer-rule-remove-unnecessary-filters.js \
 			@top_srcdir@/js/server/tests/aql-optimizer-rule-use-index-for-sort.js \
+			@top_srcdir@/js/server/tests/aql-optimizer-rule-replace-or-with-in.js \
 			@top_srcdir@/js/server/tests/aql-parse.js \
 			@top_srcdir@/js/server/tests/aql-primary-index-noncluster.js \
 			@top_srcdir@/js/server/tests/aql-queries-collection.js \

arangod/Aql/AstNode.cpp

@@ -1073,12 +1073,18 @@ bool AstNode::isComparisonOperator () const {
 ////////////////////////////////////////////////////////////////////////////////
 
 bool AstNode::canThrow () const {
+  if (hasFlag(FLAG_THROWS)) {
+    // fast track exit
+    return true;
+  }
+  
   // check sub-nodes first
   size_t const n = numMembers();
   for (size_t i = 0; i < n; ++i) {
     auto member = getMember(i);
     if (member->canThrow()) {
       // if any sub-node may throw, the whole branch may throw
+      setFlag(FLAG_THROWS);
       return true;
     }
   }
@@ -1093,13 +1099,16 @@ bool AstNode::canThrow () const {
     // potentially throwing. This is not correct on the one hand, but on
     // the other hand we must not optimize or move non-deterministic functions
     // during optimization
-    // TODO: move the check for isDeterministic into a function of its
+    if (func->canThrow) {
-    // own and check it from the optimizer rules
+      setFlag(FLAG_THROWS);
-    return func->canThrow || ! func->isDeterministic;
+      return true;
+    }
+    return false;
   }
   
   if (type == NODE_TYPE_FCALL_USER) {
     // user functions can always throw
+    setFlag(FLAG_THROWS);
     return true;
   }
 
@@ -1144,12 +1153,18 @@ bool AstNode::canRunOnDBServer () const {
 ////////////////////////////////////////////////////////////////////////////////
 
 bool AstNode::isDeterministic () const {
+  if (hasFlag(FLAG_NONDETERMINISTIC)) {
+    // fast track exit
+    return false;
+  }
+
   // check sub-nodes first
   size_t const n = numMembers();
   for (size_t i = 0; i < n; ++i) {
     auto member = getMember(i);
     if (! member->isDeterministic()) {
       // if any sub-node is non-deterministic, we are neither
+      setFlag(FLAG_NONDETERMINISTIC);
       return false;
     }
   }
@@ -1157,11 +1172,16 @@ bool AstNode::isDeterministic () const {
   if (type == NODE_TYPE_FCALL) {
     // built-in functions may or may not be deterministic
     auto func = static_cast<Function*>(getData());
-    return func->isDeterministic;
+    if (! func->isDeterministic) {
+      setFlag(FLAG_NONDETERMINISTIC);
+      return false;
+    }
+    return true;
   }
   
   if (type == NODE_TYPE_FCALL_USER) {
     // user functions are always non-deterministic
+    setFlag(FLAG_NONDETERMINISTIC);
     return false;
   }
 

arangod/Aql/AstNode.h

@@ -59,11 +59,12 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
     enum AstNodeFlagType : uint8_t {
-      FLAG_SORTED   = 1,   // node is a list and its members are sorted asc.
+      FLAG_SORTED           = 1,   // node is a list and its members are sorted asc.
-      FLAG_CONSTANT = 2,   // node value is constant (i.e. not dynamic)
+      FLAG_CONSTANT         = 2,   // node value is constant (i.e. not dynamic)
-      FLAG_DYNAMIC  = 4,   // node value is dynamic (i.e. not constant)
+      FLAG_DYNAMIC          = 4,   // node value is dynamic (i.e. not constant)
-      FLAG_SIMPLE   = 8    // node value is simple (i.e. for use in a simple expression)
+      FLAG_SIMPLE           = 8,   // node value is simple (i.e. for use in a simple expression)
-
+      FLAG_THROWS           = 16,  // node can throws an exception
+      FLAG_NONDETERMINISTIC = 32   // node produces non-deterministic result (e.g. function call nodes)
     };
 
 ////////////////////////////////////////////////////////////////////////////////

arangod/Aql/ExecutionNode.cpp

@@ -1824,6 +1824,10 @@ SortInformation SortNode::getSortInformation (ExecutionPlan* plan,
       // variable introduced by a calculation
       auto expression = static_cast<CalculationNode*>(setter)->expression();
 
+      if (! expression->isDeterministic()) {
+        result.isDeterministic = false;
+      }
+
       if (! expression->isAttributeAccess() &&
           ! expression->isReference()) {
         result.isComplex = true;

arangod/Aql/ExecutionNode.h

@@ -1718,9 +1718,10 @@ namespace triagens {
       };
 
       std::vector<std::tuple<ExecutionNode const*, std::string, bool>> criteria;
-      bool isValid   = true;
+      bool isValid         = true;
-      bool isComplex = false;
+      bool isDeterministic = true;
-      bool canThrow  = false;
+      bool isComplex       = false;
+      bool canThrow        = false;
           
       Match isCoveredBy (SortInformation const& other) {
         if (! isValid || ! other.isValid) {

arangod/Aql/Executor.cpp

@@ -104,63 +104,63 @@ std::unordered_map<std::string, Function const> const Executor::FunctionNames{
   { "TO_LIST",                     Function("TO_LIST",                     "AQL_TO_LIST", ".", true, false, true) },
   
   // string functions
-  { "CONCAT",                      Function("CONCAT",                      "AQL_CONCAT", "sz,sz|+", true, true, true) },
+  { "CONCAT",                      Function("CONCAT",                      "AQL_CONCAT", "sz,sz|+", true, false, true) },
-  { "CONCAT_SEPARATOR",            Function("CONCAT_SEPARATOR",            "AQL_CONCAT_SEPARATOR", "s,sz,sz|+", true, true, true) },
+  { "CONCAT_SEPARATOR",            Function("CONCAT_SEPARATOR",            "AQL_CONCAT_SEPARATOR", "s,sz,sz|+", true, false, true) },
-  { "CHAR_LENGTH",                 Function("CHAR_LENGTH",                 "AQL_CHAR_LENGTH", "s", true, true, true) },
+  { "CHAR_LENGTH",                 Function("CHAR_LENGTH",                 "AQL_CHAR_LENGTH", "s", true, false, true) },
-  { "LOWER",                       Function("LOWER",                       "AQL_LOWER", "s", true, true, true) },
+  { "LOWER",                       Function("LOWER",                       "AQL_LOWER", "s", true, false, true) },
-  { "UPPER",                       Function("UPPER",                       "AQL_UPPER", "s", true, true, true) },
+  { "UPPER",                       Function("UPPER",                       "AQL_UPPER", "s", true, false, true) },
-  { "SUBSTRING",                   Function("SUBSTRING",                   "AQL_SUBSTRING", "s,n|n", true, true, true) },
+  { "SUBSTRING",                   Function("SUBSTRING",                   "AQL_SUBSTRING", "s,n|n", true, false, true) },
-  { "CONTAINS",                    Function("CONTAINS",                    "AQL_CONTAINS", "s,s|b", true, true, true) },
+  { "CONTAINS",                    Function("CONTAINS",                    "AQL_CONTAINS", "s,s|b", true, false, true) },
-  { "LIKE",                        Function("LIKE",                        "AQL_LIKE", "s,r|b", true, true, true) },
+  { "LIKE",                        Function("LIKE",                        "AQL_LIKE", "s,r|b", true, false, true) },
-  { "LEFT",                        Function("LEFT",                        "AQL_LEFT", "s,n", true, true, true) },
+  { "LEFT",                        Function("LEFT",                        "AQL_LEFT", "s,n", true, false, true) },
-  { "RIGHT",                       Function("RIGHT",                       "AQL_RIGHT", "s,n", true, true, true) },
+  { "RIGHT",                       Function("RIGHT",                       "AQL_RIGHT", "s,n", true, false, true) },
-  { "TRIM",                        Function("TRIM",                        "AQL_TRIM", "s|n", true, true, true) },
+  { "TRIM",                        Function("TRIM",                        "AQL_TRIM", "s|n", true, false, true) },
-  { "FIND_FIRST",                  Function("FIND_FIRST",                  "AQL_FIND_FIRST", "s,s|zn,zn", true, true, true) },
+  { "FIND_FIRST",                  Function("FIND_FIRST",                  "AQL_FIND_FIRST", "s,s|zn,zn", true, false, true) },
-  { "FIND_LAST",                   Function("FIND_LAST",                   "AQL_FIND_LAST", "s,s|zn,zn", true, true, true) },
+  { "FIND_LAST",                   Function("FIND_LAST",                   "AQL_FIND_LAST", "s,s|zn,zn", true, false, true) },
 
   // numeric functions
-  { "FLOOR",                       Function("FLOOR",                       "AQL_FLOOR", "n", true, true, true) },
+  { "FLOOR",                       Function("FLOOR",                       "AQL_FLOOR", "n", true, false, true) },
-  { "CEIL",                        Function("CEIL",                        "AQL_CEIL", "n", true, true, true) },
+  { "CEIL",                        Function("CEIL",                        "AQL_CEIL", "n", true, false, true) },
-  { "ROUND",                       Function("ROUND",                       "AQL_ROUND", "n", true, true, true) },
+  { "ROUND",                       Function("ROUND",                       "AQL_ROUND", "n", true, false, true) },
-  { "ABS",                         Function("ABS",                         "AQL_ABS", "n", true, true, true) },
+  { "ABS",                         Function("ABS",                         "AQL_ABS", "n", true, false, true) },
   { "RAND",                        Function("RAND",                        "AQL_RAND", "", false, false, true) },
-  { "SQRT",                        Function("SQRT",                        "AQL_SQRT", "n", true, true, true) },
+  { "SQRT",                        Function("SQRT",                        "AQL_SQRT", "n", true, false, true) },
   
   // list functions
-  { "RANGE",                       Function("RANGE",                       "AQL_RANGE", "n,n|n", true, true, true) },
+  { "RANGE",                       Function("RANGE",                       "AQL_RANGE", "n,n|n", true, false, true) },
-  { "UNION",                       Function("UNION",                       "AQL_UNION", "l,l|+",true, true, true) },
+  { "UNION",                       Function("UNION",                       "AQL_UNION", "l,l|+",true, false, true) },
-  { "UNION_DISTINCT",              Function("UNION_DISTINCT",              "AQL_UNION_DISTINCT", "l,l|+", true, true, true) },
+  { "UNION_DISTINCT",              Function("UNION_DISTINCT",              "AQL_UNION_DISTINCT", "l,l|+", true, false, true) },
-  { "MINUS",                       Function("MINUS",                       "AQL_MINUS", "l,l|+", true, true, true) },
+  { "MINUS",                       Function("MINUS",                       "AQL_MINUS", "l,l|+", true, false, true) },
-  { "INTERSECTION",                Function("INTERSECTION",                "AQL_INTERSECTION", "l,l|+", true, true, true) },
+  { "INTERSECTION",                Function("INTERSECTION",                "AQL_INTERSECTION", "l,l|+", true, false, true) },
-  { "FLATTEN",                     Function("FLATTEN",                     "AQL_FLATTEN", "l|n", true, true, true) },
+  { "FLATTEN",                     Function("FLATTEN",                     "AQL_FLATTEN", "l|n", true, false, true) },
-  { "LENGTH",                      Function("LENGTH",                      "AQL_LENGTH", "las", true, true, true) },
+  { "LENGTH",                      Function("LENGTH",                      "AQL_LENGTH", "las", true, false, true) },
-  { "MIN",                         Function("MIN",                         "AQL_MIN", "l", true, true, true) },
+  { "MIN",                         Function("MIN",                         "AQL_MIN", "l", true, false, true) },
-  { "MAX",                         Function("MAX",                         "AQL_MAX", "l", true, true, true) },
+  { "MAX",                         Function("MAX",                         "AQL_MAX", "l", true, false, true) },
-  { "SUM",                         Function("SUM",                         "AQL_SUM", "l", true, true, true) },
+  { "SUM",                         Function("SUM",                         "AQL_SUM", "l", true, false, true) },
-  { "MEDIAN",                      Function("MEDIAN",                      "AQL_MEDIAN", "l", true, true, true) }, 
+  { "MEDIAN",                      Function("MEDIAN",                      "AQL_MEDIAN", "l", true, false, true) }, 
-  { "AVERAGE",                     Function("AVERAGE",                     "AQL_AVERAGE", "l", true, true, true) },
+  { "AVERAGE",                     Function("AVERAGE",                     "AQL_AVERAGE", "l", true, false, true) },
-  { "VARIANCE_SAMPLE",             Function("VARIANCE_SAMPLE",             "AQL_VARIANCE_SAMPLE", "l", true, true, true) },
+  { "VARIANCE_SAMPLE",             Function("VARIANCE_SAMPLE",             "AQL_VARIANCE_SAMPLE", "l", true, false, true) },
-  { "VARIANCE_POPULATION",         Function("VARIANCE_POPULATION",         "AQL_VARIANCE_POPULATION", "l", true, true, true) },
+  { "VARIANCE_POPULATION",         Function("VARIANCE_POPULATION",         "AQL_VARIANCE_POPULATION", "l", true, false, true) },
-  { "STDDEV_SAMPLE",               Function("STDDEV_SAMPLE",               "AQL_STDDEV_SAMPLE", "l", true, true, true) },
+  { "STDDEV_SAMPLE",               Function("STDDEV_SAMPLE",               "AQL_STDDEV_SAMPLE", "l", true, false, true) },
-  { "STDDEV_POPULATION",           Function("STDDEV_POPULATION",           "AQL_STDDEV_POPULATION", "l", true, true, true) },
+  { "STDDEV_POPULATION",           Function("STDDEV_POPULATION",           "AQL_STDDEV_POPULATION", "l", true, false, true) },
-  { "UNIQUE",                      Function("UNIQUE",                      "AQL_UNIQUE", "l", true, true, true) },
+  { "UNIQUE",                      Function("UNIQUE",                      "AQL_UNIQUE", "l", true, false, true) },
-  { "SLICE",                       Function("SLICE",                       "AQL_SLICE", "l,n|n", true, true, true) },
+  { "SLICE",                       Function("SLICE",                       "AQL_SLICE", "l,n|n", true, false, true) },
-  { "REVERSE",                     Function("REVERSE",                     "AQL_REVERSE", "ls", true, true, true) },    // note: REVERSE() can be applied on strings, too
+  { "REVERSE",                     Function("REVERSE",                     "AQL_REVERSE", "ls", true, false, true) },    // note: REVERSE() can be applied on strings, too
-  { "FIRST",                       Function("FIRST",                       "AQL_FIRST", "l", true, true, true) },
+  { "FIRST",                       Function("FIRST",                       "AQL_FIRST", "l", true, false, true) },
-  { "LAST",                        Function("LAST",                        "AQL_LAST", "l", true, true, true) },
+  { "LAST",                        Function("LAST",                        "AQL_LAST", "l", true, false, true) },
-  { "NTH",                         Function("NTH",                         "AQL_NTH", "l,n", true, true, true) },
+  { "NTH",                         Function("NTH",                         "AQL_NTH", "l,n", true, false, true) },
-  { "POSITION",                    Function("POSITION",                    "AQL_POSITION", "l,.|b", true, true, true) },
+  { "POSITION",                    Function("POSITION",                    "AQL_POSITION", "l,.|b", true, false, true) },
 
   // document functions
-  { "HAS",                         Function("HAS",                         "AQL_HAS", "az,s", true, true, true) },
+  { "HAS",                         Function("HAS",                         "AQL_HAS", "az,s", true, false, true) },
-  { "ATTRIBUTES",                  Function("ATTRIBUTES",                  "AQL_ATTRIBUTES", "a|b,b", true, true, true) },
+  { "ATTRIBUTES",                  Function("ATTRIBUTES",                  "AQL_ATTRIBUTES", "a|b,b", true, false, true) },
-  { "MERGE",                       Function("MERGE",                       "AQL_MERGE", "a,a|+", true, true, true) },
+  { "MERGE",                       Function("MERGE",                       "AQL_MERGE", "a,a|+", true, false, true) },
-  { "MERGE_RECURSIVE",             Function("MERGE_RECURSIVE",             "AQL_MERGE_RECURSIVE", "a,a|+", true, true, true) },
+  { "MERGE_RECURSIVE",             Function("MERGE_RECURSIVE",             "AQL_MERGE_RECURSIVE", "a,a|+", true, false, true) },
   { "DOCUMENT",                    Function("DOCUMENT",                    "AQL_DOCUMENT", "h.|.", false, true, false) },
-  { "MATCHES",                     Function("MATCHES",                     "AQL_MATCHES", ".,l|b", true, true, true) },
+  { "MATCHES",                     Function("MATCHES",                     "AQL_MATCHES", ".,l|b", true, false, true) },
-  { "UNSET",                       Function("UNSET",                       "AQL_UNSET", "a,sl|+", true, true, true) },
+  { "UNSET",                       Function("UNSET",                       "AQL_UNSET", "a,sl|+", true, false, true) },
-  { "KEEP",                        Function("KEEP",                        "AQL_KEEP", "a,sl|+", true, true, true) },
+  { "KEEP",                        Function("KEEP",                        "AQL_KEEP", "a,sl|+", true, false, true) },
-  { "TRANSLATE",                   Function("TRANSLATE",                   "AQL_TRANSLATE", ".,a|.", true, true, true) },
+  { "TRANSLATE",                   Function("TRANSLATE",                   "AQL_TRANSLATE", ".,a|.", true, false, true) },
 
   // geo functions
   { "NEAR",                        Function("NEAR",                        "AQL_NEAR", "h,n,n|nz,s", false, true, false) },
@@ -199,26 +199,26 @@ std::unordered_map<std::string, Function const> const Executor::FunctionNames{
 
   // date functions
   { "DATE_NOW",                    Function("DATE_NOW",                    "AQL_DATE_NOW", "", false, false, true) },
-  { "DATE_TIMESTAMP",              Function("DATE_TIMESTAMP",              "AQL_DATE_TIMESTAMP", "ns|ns,ns,ns,ns,ns,ns", true, true, true) },
+  { "DATE_TIMESTAMP",              Function("DATE_TIMESTAMP",              "AQL_DATE_TIMESTAMP", "ns|ns,ns,ns,ns,ns,ns", true, false, true) },
-  { "DATE_ISO8601",                Function("DATE_ISO8601",                "AQL_DATE_ISO8601", "ns|ns,ns,ns,ns,ns,ns", true, true, true) },
+  { "DATE_ISO8601",                Function("DATE_ISO8601",                "AQL_DATE_ISO8601", "ns|ns,ns,ns,ns,ns,ns", true, false, true) },
-  { "DATE_DAYOFWEEK",              Function("DATE_DAYOFWEEK",              "AQL_DATE_DAYOFWEEK", "ns", true, true, true) },
+  { "DATE_DAYOFWEEK",              Function("DATE_DAYOFWEEK",              "AQL_DATE_DAYOFWEEK", "ns", true, false, true) },
-  { "DATE_YEAR",                   Function("DATE_YEAR",                   "AQL_DATE_YEAR", "ns", true, true, true) },
+  { "DATE_YEAR",                   Function("DATE_YEAR",                   "AQL_DATE_YEAR", "ns", true, false, true) },
-  { "DATE_MONTH",                  Function("DATE_MONTH",                  "AQL_DATE_MONTH", "ns", true, true, true) },
+  { "DATE_MONTH",                  Function("DATE_MONTH",                  "AQL_DATE_MONTH", "ns", true, false, true) },
-  { "DATE_DAY",                    Function("DATE_DAY",                    "AQL_DATE_DAY", "ns", true, true, true) },
+  { "DATE_DAY",                    Function("DATE_DAY",                    "AQL_DATE_DAY", "ns", true, false, true) },
-  { "DATE_HOUR",                   Function("DATE_HOUR",                   "AQL_DATE_HOUR", "ns", true, true, true) },
+  { "DATE_HOUR",                   Function("DATE_HOUR",                   "AQL_DATE_HOUR", "ns", true, false, true) },
-  { "DATE_MINUTE",                 Function("DATE_MINUTE",                 "AQL_DATE_MINUTE", "ns", true, true, true) },
+  { "DATE_MINUTE",                 Function("DATE_MINUTE",                 "AQL_DATE_MINUTE", "ns", true, false, true) },
-  { "DATE_SECOND",                 Function("DATE_SECOND",                 "AQL_DATE_SECOND", "ns", true, true, true) },
+  { "DATE_SECOND",                 Function("DATE_SECOND",                 "AQL_DATE_SECOND", "ns", true, false, true) },
-  { "DATE_MILLISECOND",            Function("DATE_MILLISECOND",            "AQL_DATE_MILLISECOND", "ns", true, true, true) },
+  { "DATE_MILLISECOND",            Function("DATE_MILLISECOND",            "AQL_DATE_MILLISECOND", "ns", true, false, true) },
 
   // misc functions
   { "FAIL",                        Function("FAIL",                        "AQL_FAIL", "|s", false, true, true) },
-  { "PASSTHRU",                    Function("PASSTHRU",                    "AQL_PASSTHRU", ".", false, true, true) },
+  { "PASSTHRU",                    Function("PASSTHRU",                    "AQL_PASSTHRU", ".", false, false, true) },
-  { "SLEEP",                       Function("SLEEP",                       "AQL_SLEEP", "n", false, false, true) },
+  { "SLEEP",                       Function("SLEEP",                       "AQL_SLEEP", "n", false, true, true) },
   { "COLLECTIONS",                 Function("COLLECTIONS",                 "AQL_COLLECTIONS", "", false, true, false) },
-  { "NOT_NULL",                    Function("NOT_NULL",                    "AQL_NOT_NULL", ".|+", true, true, true) },
+  { "NOT_NULL",                    Function("NOT_NULL",                    "AQL_NOT_NULL", ".|+", true, false, true) },
   { "FIRST_LIST",                  Function("FIRST_LIST",                  "AQL_FIRST_LIST", ".|+", true, false, true) },
   { "FIRST_DOCUMENT",              Function("FIRST_DOCUMENT",              "AQL_FIRST_DOCUMENT", ".|+", true, false, true) },
-  { "PARSE_IDENTIFIER",            Function("PARSE_IDENTIFIER",            "AQL_PARSE_IDENTIFIER", ".", true, true, true) },
+  { "PARSE_IDENTIFIER",            Function("PARSE_IDENTIFIER",            "AQL_PARSE_IDENTIFIER", ".", true, false, true) },
   { "SKIPLIST",                    Function("SKIPLIST",                    "AQL_SKIPLIST", "h,a|n,n", false, true, false) },
   { "CURRENT_USER",                Function("CURRENT_USER",                "AQL_CURRENT_USER", "", false, false, false) },
   { "CURRENT_DATABASE",            Function("CURRENT_DATABASE",            "AQL_CURRENT_DATABASE", "", false, false, false) }

arangod/Aql/Optimizer.cpp

@@ -420,7 +420,7 @@ void Optimizer::setupRules () {
                moveFiltersUpRule,
                moveFiltersUpRule_pass4,
                true);
-
+  
   //////////////////////////////////////////////////////////////////////////////
   /// "Pass 5": try to remove redundant or unnecessary nodes (second try)
   ///           use levels between 601 and 699 for this
@@ -450,6 +450,12 @@ void Optimizer::setupRules () {
   /// "Pass 6": use indexes if possible for FILTER and/or SORT nodes
   ///           use levels between 701 and 799 for this
   //////////////////////////////////////////////////////////////////////////////
+  
+  // try to replace simple OR conditions with IN
+  registerRule("replace-OR-with-IN",
+               replaceORwithIN,
+               replaceORwithIN_pass6,
+               true);
 
   // try to find a filter after an enumerate collection and find an index . . . 
   registerRule("use-index-range",

arangod/Aql/Optimizer.h

@@ -103,6 +103,7 @@ namespace triagens {
         // move filters up the dependency chain (to make result sets as small
         // as possible as early as possible)
         moveFiltersUpRule_pass4                   = 620,
+        
 
 //////////////////////////////////////////////////////////////////////////////
 /// "Pass 5": try to remove redundant or unnecessary nodes (second try)
@@ -125,11 +126,15 @@ namespace triagens {
 //////////////////////////////////////////////////////////////////////////////
 
         pass6                                     = 800,
+        
+        // replace simple OR conditions with IN
+        replaceORwithIN_pass6                     = 810,
+        
         // try to find a filter after an enumerate collection and find an index . . . 
-        useIndexRange_pass6                       = 810,
+        useIndexRange_pass6                       = 820,
 
         // try to find sort blocks which are superseeded by indexes
-        useIndexForSort_pass6                     = 820,
+        useIndexForSort_pass6                     = 830,
   
 //////////////////////////////////////////////////////////////////////////////
 /// "Pass 10": final transformations for the cluster

arangod/Aql/OptimizerRules.cpp

@@ -35,6 +35,14 @@ using namespace triagens::aql;
 using Json = triagens::basics::Json;
 using EN   = triagens::aql::ExecutionNode;
 
+//#if 0
+#define ENTER_BLOCK try { (void) 0;
+#define LEAVE_BLOCK } catch (...) { std::cout << "caught an exception in " << __FUNCTION__ << ", " << __FILE__ << ":" << __LINE__ << "!\n"; throw; }
+//#else
+//#define ENTER_BLOCK
+//#define LEAVE_BLOCK
+//#endif
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                           rules for the optimizer
 // -----------------------------------------------------------------------------
@@ -48,7 +56,7 @@ using EN   = triagens::aql::ExecutionNode;
 int triagens::aql::removeRedundantSorts (Optimizer* opt, 
                                          ExecutionPlan* plan,
                                          Optimizer::Rule const* rule) { 
-  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::SORT, true);
+  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(EN::SORT, true);
   std::unordered_set<ExecutionNode*> toUnlink;
 
   triagens::basics::StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE);
@@ -77,7 +85,7 @@ int triagens::aql::removeRedundantSorts (Optimizer* opt,
         auto current = stack.back();
         stack.pop_back();
 
-        if (current->getType() == triagens::aql::ExecutionNode::SORT) {
+        if (current->getType() == EN::SORT) {
           // we found another sort. now check if they are compatible!
     
           auto other = static_cast<SortNode*>(current)->getSortInformation(plan, &buffer);
@@ -88,8 +96,8 @@ int triagens::aql::removeRedundantSorts (Optimizer* opt,
               if (nodesRelyingOnSort == 0) {
                 // a sort directly followed by another sort: now remove one of them
 
-                if (other.canThrow) {
+                if (other.canThrow || ! other.isDeterministic) {
-                  // if the sort can throw, we must not remove it
+                  // if the sort can throw or is non-deterministic, we must not remove it
                   break;
                 }
 
@@ -126,17 +134,17 @@ int triagens::aql::removeRedundantSorts (Optimizer* opt,
             }
           }
         }
-        else if (current->getType() == triagens::aql::ExecutionNode::FILTER) {
+        else if (current->getType() == EN::FILTER) {
           // ok: a filter does not depend on sort order
         }
-        else if (current->getType() == triagens::aql::ExecutionNode::CALCULATION) {
+        else if (current->getType() == EN::CALCULATION) {
           // ok: a filter does not depend on sort order only if it does not throw
           if (current->canThrow()) {
             ++nodesRelyingOnSort;
           }
         }
-        else if (current->getType() == triagens::aql::ExecutionNode::ENUMERATE_LIST ||
+        else if (current->getType() == EN::ENUMERATE_LIST ||
-                 current->getType() == triagens::aql::ExecutionNode::ENUMERATE_COLLECTION) {
+                 current->getType() == EN::ENUMERATE_COLLECTION) {
           // ok, but we cannot remove two different sorts if one of these node types is between them
           // example: in the following query, the one sort will be optimized away:
           //   FOR i IN [ { a: 1 }, { a: 2 } , { a: 3 } ] SORT i.a ASC SORT i.a DESC RETURN i
@@ -188,7 +196,7 @@ int triagens::aql::removeUnnecessaryFiltersRule (Optimizer* opt,
   bool modified = false;
   std::unordered_set<ExecutionNode*> toUnlink;
   // should we enter subqueries??
-  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::FILTER, true);
+  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(EN::FILTER, true);
   
   for (auto n : nodes) {
     // filter nodes always have one input variable
@@ -200,7 +208,7 @@ int triagens::aql::removeUnnecessaryFiltersRule (Optimizer* opt,
     auto setter = plan->getVarSetBy(variable->id);
 
     if (setter == nullptr || 
-        setter->getType() != triagens::aql::ExecutionNode::CALCULATION) {
+        setter->getType() != EN::CALCULATION) {
       // filter variable was not introduced by a calculation. 
       continue;
     }
@@ -254,13 +262,14 @@ int triagens::aql::removeUnnecessaryFiltersRule (Optimizer* opt,
 int triagens::aql::moveCalculationsUpRule (Optimizer* opt, 
                                            ExecutionPlan* plan, 
                                            Optimizer::Rule const* rule) {
-  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::CALCULATION, true);
+  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(EN::CALCULATION, true);
   bool modified = false;
   
   for (auto n : nodes) {
     auto nn = static_cast<CalculationNode*>(n);
-    if (nn->expression()->canThrow()) {
+    if (nn->expression()->canThrow() || 
-      // we will only move expressions up that cannot throw
+        ! nn->expression()->isDeterministic()) {
+      // we will only move expressions up that cannot throw and that are deterministic
       continue;
     }
 
@@ -333,7 +342,7 @@ int triagens::aql::moveCalculationsUpRule (Optimizer* opt,
 int triagens::aql::moveFiltersUpRule (Optimizer* opt, 
                                       ExecutionPlan* plan,
                                       Optimizer::Rule const* rule) {
-  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::FILTER, true);
+  std::vector<ExecutionNode*> nodes = plan->findNodesOfType(EN::FILTER, true);
   bool modified = false;
   
   for (auto n : nodes) {
@@ -349,7 +358,7 @@ int triagens::aql::moveFiltersUpRule (Optimizer* opt,
       auto current = stack.back();
       stack.pop_back();
 
-      if (current->getType() == triagens::aql::ExecutionNode::LIMIT) {
+      if (current->getType() == EN::LIMIT) {
         // cannot push a filter beyond a LIMIT node
         break;
       }
@@ -359,6 +368,14 @@ int triagens::aql::moveFiltersUpRule (Optimizer* opt,
         break;
       }
 
+      if (current->getType() == EN::CALCULATION) {
+        // must not move a filter beyond a node with a non-deterministic result
+        auto calculation = static_cast<CalculationNode const*>(current);
+        if (! calculation->expression()->isDeterministic()) {
+          break;
+        }
+      }
+
       bool found = false;
 
       auto&& varsSet = current->getVariablesSetHere();
@@ -500,7 +517,7 @@ int triagens::aql::removeRedundantCalculationsRule (Optimizer* opt,
   std::unordered_map<VariableId, Variable const*> replacements;
 
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::CALCULATION, true);
+    = plan->findNodesOfType(EN::CALCULATION, true);
 
   for (auto n : nodes) {
     auto nn = static_cast<CalculationNode*>(n);
@@ -535,7 +552,7 @@ int triagens::aql::removeRedundantCalculationsRule (Optimizer* opt,
       auto current = stack.back();
       stack.pop_back();
 
-      if (current->getType() == triagens::aql::ExecutionNode::CALCULATION) {
+      if (current->getType() == EN::CALCULATION) {
         try {
           static_cast<CalculationNode*>(current)->expression()->stringify(&buffer);
         }
@@ -584,7 +601,7 @@ int triagens::aql::removeRedundantCalculationsRule (Optimizer* opt,
         }
       }
 
-      if (current->getType() == triagens::aql::ExecutionNode::AGGREGATE) {
+      if (current->getType() == EN::AGGREGATE) {
         if (static_cast<AggregateNode*>(current)->hasOutVariable()) {
           // COLLECT ... INTO is evil (tm): it needs to keep all already defined variables
           // we need to abort optimization here
@@ -632,19 +649,18 @@ int triagens::aql::removeUnnecessaryCalculationsRule (Optimizer* opt,
                                                       ExecutionPlan* plan,
                                                       Optimizer::Rule const* rule) {
   std::vector<ExecutionNode::NodeType> const types = {
-    triagens::aql::ExecutionNode::CALCULATION,
+    EN::CALCULATION,
-    triagens::aql::ExecutionNode::SUBQUERY
+    EN::SUBQUERY
   };
 
   std::vector<ExecutionNode*> nodes = plan->findNodesOfType(types, true);
   std::unordered_set<ExecutionNode*> toUnlink;
   for (auto n : nodes) {
-    if (n->getType() == triagens::aql::ExecutionNode::CALCULATION) {
+    if (n->getType() == EN::CALCULATION) {
       auto nn = static_cast<CalculationNode*>(n);
 
-      if (nn->canThrow() ||
+      if (nn->canThrow()) {
-          ! nn->expression()->isDeterministic()) {
+        // If this node can throw, we must not optimize it away!
-        // If this node can throw or is non-deterministic, we must not optimize it away!
         continue;
       }
     }
@@ -983,7 +999,7 @@ class FilterToEnumCollFinder : public WalkerWorker<ExecutionNode> {
         auto x = static_cast<Variable*>(node->getData());
         auto setter = _plan->getVarSetBy(x->id);
         if (setter != nullptr && 
-            setter->getType() == triagens::aql::ExecutionNode::ENUMERATE_COLLECTION) {
+            setter->getType() == EN::ENUMERATE_COLLECTION) {
           enumCollVar = x;
         }
         return;
@@ -1132,7 +1148,7 @@ int triagens::aql::useIndexRange (Optimizer* opt,
                                   Optimizer::Rule const* rule) {
   
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::FILTER, true);
+    = plan->findNodesOfType(EN::FILTER, true);
   
   for (auto n : nodes) {
     auto nn = static_cast<FilterNode*>(n);
@@ -1441,7 +1457,7 @@ int triagens::aql::useIndexForSort (Optimizer* opt,
                                     Optimizer::Rule const* rule) {
   bool planModified = false;
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::SORT, true);
+    = plan->findNodesOfType(EN::SORT, true);
   for (auto n : nodes) {
     auto thisSortNode = static_cast<SortNode*>(n);
     SortAnalysis node(thisSortNode);
@@ -1503,7 +1519,7 @@ int triagens::aql::interchangeAdjacentEnumerations (Optimizer* opt,
                                                     ExecutionPlan* plan,
                                                     Optimizer::Rule const* rule) {
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::ENUMERATE_COLLECTION, 
+    = plan->findNodesOfType(EN::ENUMERATE_COLLECTION, 
                             true);
   std::unordered_set<ExecutionNode*> nodesSet;
   for (auto n : nodes) {
@@ -1532,7 +1548,7 @@ int triagens::aql::interchangeAdjacentEnumerations (Optimizer* opt,
           break;
         }
         if (deps[0]->getType() != 
-            triagens::aql::ExecutionNode::ENUMERATE_COLLECTION) {
+            EN::ENUMERATE_COLLECTION) {
           break;
         }
         nwalker = deps[0];
@@ -1818,7 +1834,7 @@ int triagens::aql::distributeFilternCalcToCluster (Optimizer* opt,
   bool modified = false;
 
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::GATHER, true);
+    = plan->findNodesOfType(EN::GATHER, true);
 
   
   for (auto n : nodes) {
@@ -1911,7 +1927,7 @@ int triagens::aql::distributeSortToCluster (Optimizer* opt,
   bool modified = false;
 
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::GATHER, true);
+    = plan->findNodesOfType(EN::GATHER, true);
   
   for (auto n : nodes) {
     auto remoteNodeList = n->getDependencies();
@@ -1993,7 +2009,7 @@ int triagens::aql::removeUnnecessaryRemoteScatter (Optimizer* opt,
                                                    ExecutionPlan* plan, 
                                                    Optimizer::Rule const* rule) {
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::REMOTE, true);
+    = plan->findNodesOfType(EN::REMOTE, true);
   std::unordered_set<ExecutionNode*> toUnlink;
 
   for (auto n : nodes) {
@@ -2237,7 +2253,7 @@ int triagens::aql::undistributeRemoveAfterEnumColl (Optimizer* opt,
                                                     ExecutionPlan* plan, 
                                                     Optimizer::Rule const* rule) {
   std::vector<ExecutionNode*> nodes
-    = plan->findNodesOfType(triagens::aql::ExecutionNode::REMOVE, true);
+    = plan->findNodesOfType(EN::REMOVE, true);
   std::unordered_set<ExecutionNode*> toUnlink;
 
   for (auto n : nodes) {
@@ -2257,6 +2273,224 @@ int triagens::aql::undistributeRemoveAfterEnumColl (Optimizer* opt,
   return TRI_ERROR_NO_ERROR;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief
+////////////////////////////////////////////////////////////////////////////////
+
+struct OrToInConverter {
+  AstNode const* variableNode;
+  std::string variableName; 
+  std::vector<AstNode const*> valueNodes;
+  std::vector<AstNode const*> possibleNodes;
+  std::vector<AstNode const*> orConditions;
+
+  std::string getString (AstNode const* node) {
+    triagens::basics::StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE);
+    node->append(&buffer, false);
+    return std::string(buffer.c_str(), buffer.length());
+  }
+
+  AstNode* buildInExpression (Ast* ast) {
+    // the list of comparison values
+    auto list = ast->createNodeList();
+    for (auto x : valueNodes) {
+      list->addMember(x);
+    }
+
+    // return a new IN operator node
+    return ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_IN,
+                                         variableNode->clone(ast),
+                                         list);
+  }
+
+  bool flattenOr (AstNode const* node) {
+    if (node->type == NODE_TYPE_OPERATOR_BINARY_OR) {
+      return (flattenOr(node->getMember(0)) && flattenOr(node->getMember(1)));
+    }
+    if (node->type == NODE_TYPE_OPERATOR_BINARY_EQ) {
+      orConditions.push_back(node);
+      return true;
+    }
+    if (node->type == NODE_TYPE_VALUE) {
+      return true;
+    }
+    return false;
+  }
+
+  bool findCommonNode (AstNode const* node) {
+    
+    if (! flattenOr(node)) {
+      return false;
+    }
+    TRI_ASSERT(orConditions.size() > 1);
+
+    for (AstNode const* n: orConditions) {
+      
+      auto lhs = n->getMember(0);
+      auto rhs = n->getMember(1);
+      
+      if (lhs->isConstant()) {
+        variableNode = rhs;
+        return true;
+      }
+
+      if (rhs->isConstant()) {
+        variableNode = lhs;
+        return true;
+      }
+
+      if (rhs->type == NODE_TYPE_FCALL || 
+          rhs->type == NODE_TYPE_FCALL_USER ||
+          rhs->type == NODE_TYPE_REFERENCE) {
+        variableNode = lhs;
+        return true;
+      }
+      
+      if (lhs->type == NODE_TYPE_FCALL || 
+          lhs->type == NODE_TYPE_FCALL_USER ||
+          lhs->type == NODE_TYPE_REFERENCE) {
+        variableNode = rhs;
+        return true;
+      }
+
+      if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
+          lhs->type == NODE_TYPE_INDEXED_ACCESS) {
+        if (possibleNodes.size() == 2) {
+          for (size_t i = 0; i < 2; i++) {
+            if (getString(lhs) == getString(possibleNodes[i])) {
+              variableNode = possibleNodes[i];
+              variableName = getString(variableNode);
+              return true;
+            }
+          }
+        } 
+        else {
+          possibleNodes.push_back(lhs);
+        }
+      }
+      if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
+          rhs->type == NODE_TYPE_INDEXED_ACCESS) {
+        if (possibleNodes.size() == 2) {
+          for (size_t i = 0; i < 2; i++) {
+            if (getString(rhs) == getString(possibleNodes[i])) {
+              variableNode = possibleNodes[i];
+              variableName = getString(variableNode);
+              return true;
+            }
+          }
+          return false;
+        } 
+        else {
+          possibleNodes.push_back(rhs);
+        }
+      }
+    }
+    return false;
+  }
+
+  bool canConvertExpression (AstNode const* node) {
+    if (node->type == NODE_TYPE_OPERATOR_BINARY_OR) {
+      return (canConvertExpression(node->getMember(0)) &&
+              canConvertExpression(node->getMember(1)));
+    }
+
+    if (node->type == NODE_TYPE_OPERATOR_BINARY_EQ) {
+      auto lhs = node->getMember(0);
+      auto rhs = node->getMember(1);
+
+      if (canConvertExpression(rhs) && ! canConvertExpression(lhs)) {
+        valueNodes.push_back(lhs);
+        return true;
+      } 
+      
+      if (canConvertExpression(lhs) && ! canConvertExpression(rhs)) {
+        valueNodes.push_back(rhs);
+        return true;
+      } 
+      // if canConvertExpression(lhs) and canConvertExpression(rhs), then one of
+      // the equalities in the OR statement is of the form x == x
+      // fall-through intentional
+    }
+    else if (node->type == NODE_TYPE_REFERENCE ||
+             node->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
+             node->type == NODE_TYPE_INDEXED_ACCESS) {
+      // get a string representation of the node for comparisons 
+      std::string nodeString = getString(node);
+      return nodeString == getString(variableNode);
+    } else if (node->isBoolValue()) {
+      return true;
+    }
+
+    return false;
+  }
+};
+
+int triagens::aql::replaceORwithIN (Optimizer* opt, 
+                                    ExecutionPlan* plan, 
+                                    Optimizer::Rule const* rule) {
+  ENTER_BLOCK;
+  std::vector<ExecutionNode*> nodes
+    = plan->findNodesOfType(EN::FILTER, true);
+
+  bool modified = false;
+  for (auto n : nodes) {
+    auto deps = n->getDependencies();
+    TRI_ASSERT(deps.size() == 1);
+    if (deps[0]->getType() != EN::CALCULATION) {
+      continue;
+    }
+
+    auto fn = static_cast<FilterNode*>(n);
+    auto cn = static_cast<CalculationNode*>(deps[0]);
+
+    auto inVar  = fn->getVariablesUsedHere();
+    auto outVar = cn->getVariablesSetHere();
+
+    if (outVar.size() != 1 || outVar[0]->id != inVar[0]->id) {
+      continue;
+    }
+    if (cn->expression()->node()->type != NODE_TYPE_OPERATOR_BINARY_OR) {
+      continue;
+    }
+
+    OrToInConverter converter;
+    if (converter.findCommonNode(cn->expression()->node())
+        && converter.canConvertExpression(cn->expression()->node())) {
+      Expression* expr = nullptr;
+      ExecutionNode* newNode = nullptr;
+      auto inNode = converter.buildInExpression(plan->getAst());
+
+      try {
+        expr = new Expression(plan->getAst(), inNode);
+      }
+      catch (...) {
+        delete inNode;
+        throw;
+      }
+
+      try {
+        newNode = new CalculationNode(plan, plan->nextId(), expr, outVar[0]);
+      }
+      catch (...) {
+        delete expr;
+        throw;
+      }
+
+      plan->registerNode(newNode);
+      plan->replaceNode(cn, newNode);
+      modified = true;
+    }
+  }
+ 
+  if (modified) {
+    plan->findVarUsage();
+  }
+  opt->addPlan(plan, rule->level, modified);
+
+  return TRI_ERROR_NO_ERROR;
+  LEAVE_BLOCK;
+}
+
 // Local Variables:
 // mode: outline-minor
 // outline-regexp: "^\\(/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|// --SECTION--\\|/// @\\}\\)"

arangod/Aql/OptimizerRules.h

@@ -164,6 +164,17 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
     int undistributeRemoveAfterEnumColl (Optimizer*, ExecutionPlan*, Optimizer::Rule const*);
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief this rule replaces expressions of the type: 
+///   x.val == 1 || x.val == 2 || x.val == 3
+//  with
+//    x.val IN [1,2,3]
+//  when the OR conditions are present in the same FILTER node, and refer to the
+//  same (single) attribute.
+////////////////////////////////////////////////////////////////////////////////
+
+    int replaceORwithIN (Optimizer*, ExecutionPlan*, Optimizer::Rule const*);
     
   }  // namespace aql
 }  // namespace triagens

arangod/Aql/Query.cpp

@@ -583,7 +583,7 @@ QueryResult Query::execute (QueryRegistry* registry) {
       return res;
     }
 
-    triagens::basics::Json json(triagens::basics::Json::List, 16);
+    triagens::basics::Json jsonResult(triagens::basics::Json::List, 16);
     triagens::basics::Json stats;
 
     AqlItemBlock* value;
@@ -592,13 +592,13 @@ QueryResult Query::execute (QueryRegistry* registry) {
       auto doc = value->getDocumentCollection(0);
       size_t const n = value->size();
       // reserve space for n additional results at once
-      json.reserve(n);
+      jsonResult.reserve(n);
 
       for (size_t i = 0; i < n; ++i) {
         AqlValue val = value->getValue(i, 0);
 
         if (! val.isEmpty()) {
-          json.add(val.toJson(_trx, doc)); 
+          jsonResult.add(val.toJson(_trx, doc)); 
         }
       }
       delete value;
@@ -614,7 +614,7 @@ QueryResult Query::execute (QueryRegistry* registry) {
 
     QueryResult result(TRI_ERROR_NO_ERROR);
     result.warnings = warningsToJson();
-    result.json     = json.steal();
+    result.json     = jsonResult.steal();
     result.stats    = stats.steal(); 
 
     if (_profile != nullptr) {
@@ -641,6 +641,79 @@ QueryResult Query::execute (QueryRegistry* registry) {
   }
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief execute an AQL query 
+/// may only be called with an active V8 handle scope
+////////////////////////////////////////////////////////////////////////////////
+
+QueryResultV8 Query::executeV8 (QueryRegistry* registry) {
+
+  // Now start the execution:
+  try {
+    QueryResultV8 res = prepare(registry);
+    if (res.code != TRI_ERROR_NO_ERROR) {
+      return res;
+    }
+
+    uint32_t j = 0;
+    QueryResultV8 result(TRI_ERROR_NO_ERROR);
+    result.result  = v8::Array::New();
+    triagens::basics::Json stats;
+
+    AqlItemBlock* value;
+
+    while (nullptr != (value = _engine->getSome(1, ExecutionBlock::DefaultBatchSize))) {
+      auto doc = value->getDocumentCollection(0);
+      size_t const n = value->size();
+      // reserve space for n additional results at once
+      /// json.reserve(n);
+      
+      for (size_t i = 0; i < n; ++i) {
+        AqlValue val = value->getValue(i, 0);
+
+        if (! val.isEmpty()) {
+          result.result->Set(j++, val.toV8(_trx, doc)); 
+        }
+      }
+      delete value;
+    }
+
+    stats = _engine->_stats.toJson();
+
+    _trx->commit();
+    
+    cleanupPlanAndEngine(TRI_ERROR_NO_ERROR);
+
+    enterState(FINALIZATION); 
+
+    result.warnings = warningsToJson();
+    result.stats    = stats.steal(); 
+
+    if (_profile != nullptr) {
+      result.profile = _profile->toJson(TRI_UNKNOWN_MEM_ZONE);
+    }
+
+    return result;
+  }
+  catch (triagens::arango::Exception const& ex) {
+    cleanupPlanAndEngine(ex.code());
+    return QueryResultV8(ex.code(), getStateString() + ex.message());
+  }
+  catch (std::bad_alloc const&) {
+    cleanupPlanAndEngine(TRI_ERROR_OUT_OF_MEMORY);
+    return QueryResultV8(TRI_ERROR_OUT_OF_MEMORY, getStateString() + TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY));
+  }
+  catch (std::exception const& ex) {
+    cleanupPlanAndEngine(TRI_ERROR_INTERNAL);
+    return QueryResultV8(TRI_ERROR_INTERNAL, getStateString() + ex.what());
+  }
+  catch (...) {
+    cleanupPlanAndEngine(TRI_ERROR_INTERNAL);
+    return QueryResult(TRI_ERROR_INTERNAL, getStateString() + TRI_errno_string(TRI_ERROR_INTERNAL));
+  }
+}
+
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief parse an AQL query
 ////////////////////////////////////////////////////////////////////////////////

arangod/Aql/Query.h

@@ -34,7 +34,7 @@
 #include "Basics/JsonHelper.h"
 #include "Aql/BindParameters.h"
 #include "Aql/Collections.h"
-#include "Aql/QueryResult.h"
+#include "Aql/QueryResultV8.h"
 #include "Aql/types.h"
 #include "Utils/AqlTransaction.h"
 #include "Utils/V8TransactionContext.h"
@@ -299,6 +299,14 @@ namespace triagens {
 
         QueryResult execute (QueryRegistry*);
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief execute an AQL query 
+/// may only be called with an active V8 handle scope
+////////////////////////////////////////////////////////////////////////////////
+
+        QueryResultV8 executeV8 (QueryRegistry*);
+
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief parse an AQL query
 ////////////////////////////////////////////////////////////////////////////////

arangod/Aql/QueryResult.h

@@ -80,7 +80,7 @@ namespace triagens {
         : QueryResult(TRI_ERROR_NO_ERROR) {
       }
 
-      ~QueryResult () {
+      virtual ~QueryResult () {
         if (warnings != nullptr) {
           TRI_FreeJson(zone, warnings);
         }

arangod/Aql/QueryResultV8.h

@@ -0,0 +1,84 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Aql, query results
+///
+/// @file
+///
+/// DISCLAIMER
+///
+/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
+/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGODB_AQL_QUERY_RESULTV8_H
+#define ARANGODB_AQL_QUERY_RESULTV8_H 1
+
+#include "Basics/Common.h"
+#include "Basics/json.h"
+#include "Aql/QueryResult.h"
+#include <v8.h>
+
+namespace triagens {
+  namespace aql {
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                struct QueryResult
+// -----------------------------------------------------------------------------
+
+    struct QueryResultV8 : public QueryResult {
+      QueryResultV8& operator= (QueryResultV8 const& other) = delete;
+            
+      QueryResultV8 (QueryResultV8&& other) 
+        : QueryResult ( (QueryResult&&) other),
+          result(other.result) {
+      }
+      
+      QueryResultV8 (QueryResult&& other) 
+        : QueryResult((QueryResult&&)other),
+          result(nullptr) {
+      }
+      
+      QueryResultV8 (int code,
+                     std::string const& details) 
+        : QueryResult(code, details),
+          result(nullptr) {
+      }
+
+      explicit QueryResultV8 (int code)
+        : QueryResult(code, ""),
+          result(nullptr) {
+      }
+
+      v8::Handle<v8::Array>          result;
+    };
+
+  }
+}
+
+#endif
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                       END-OF-FILE
+// -----------------------------------------------------------------------------
+
+// Local Variables:
+// mode: outline-minor
+// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
+// End:

arangod/V8Server/v8-vocbase.cpp

@@ -1200,7 +1200,7 @@ static v8::Handle<v8::Value> JS_ExecuteAql (v8::Arguments const& argv) {
   TRI_v8_global_t* v8g = static_cast<TRI_v8_global_t*>(v8::Isolate::GetCurrent()->GetData());
   triagens::aql::Query query(v8g->_applicationV8, true, vocbase, queryString.c_str(), queryString.size(), parameters, options, triagens::aql::PART_MAIN);
 
-  auto queryResult = query.execute(static_cast<triagens::aql::QueryRegistry*>(v8g->_queryRegistry));
+  auto queryResult = query.executeV8(static_cast<triagens::aql::QueryRegistry*>(v8g->_queryRegistry));
   
   if (queryResult.code != TRI_ERROR_NO_ERROR) {
     if (queryResult.code == TRI_ERROR_REQUEST_CANCELED) {
@@ -1212,12 +1212,12 @@ static v8::Handle<v8::Value> JS_ExecuteAql (v8::Arguments const& argv) {
     TRI_V8_EXCEPTION_FULL(scope, queryResult.code, queryResult.details);
   }
   
-  if (TRI_LengthListJson(queryResult.json) <= batchSize) {
+  if (queryResult.result->Length() <= batchSize) {
     // return the array value as it is. this is a performance optimisation
     v8::Handle<v8::Object> result = v8::Object::New();
-    if (queryResult.json != nullptr) {
+
-      result->Set(TRI_V8_STRING("json"), TRI_ObjectJson(queryResult.json));
+    result->Set(TRI_V8_STRING("json"), queryResult.result);
-    }
+
     if (queryResult.stats != nullptr) {
       result->Set(TRI_V8_STRING("stats"), TRI_ObjectJson(queryResult.stats));
     }
@@ -1251,7 +1251,7 @@ static v8::Handle<v8::Value> JS_ExecuteAql (v8::Arguments const& argv) {
     queryResult.profile = nullptr;
   }
 
-  TRI_general_cursor_result_t* cursorResult = TRI_CreateResultGeneralCursor(queryResult.json);
+  TRI_general_cursor_result_t* cursorResult = TRI_CreateResultGeneralCursor(queryResult.result);
   
   if (cursorResult == nullptr) {
     if (extra != nullptr) {
@@ -1260,8 +1260,6 @@ static v8::Handle<v8::Value> JS_ExecuteAql (v8::Arguments const& argv) {
     TRI_V8_EXCEPTION_MEMORY(scope);
   }
   
-  queryResult.json = nullptr;
-
   TRI_general_cursor_t* cursor = TRI_CreateGeneralCursor(vocbase, cursorResult, doCount,
       static_cast<TRI_general_cursor_length_t>(batchSize), ttl, extra);
 

arangod/VocBase/general-cursor.cpp

@@ -32,6 +32,7 @@
 #include "Basics/json.h"
 #include "Basics/logging.h"
 #include "Basics/vector.h"
+#include "V8/v8-conv.h"
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                     cursor result
@@ -48,7 +49,7 @@
 static void FreeData (TRI_general_cursor_result_t* result) {
   TRI_json_t* json = (TRI_json_t*) result->_data;
 
-  TRI_ASSERT(json);
+  TRI_ASSERT(json != nullptr);
 
   TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
 }
@@ -94,6 +95,20 @@ TRI_general_cursor_result_t* TRI_CreateResultGeneralCursor (TRI_json_t* data) {
   return TRI_CreateCursorResult((void*) data, &FreeData, &GetAt, &GetLength);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create a result set
+////////////////////////////////////////////////////////////////////////////////
+
+TRI_general_cursor_result_t* TRI_CreateResultGeneralCursor (v8::Handle<v8::Array> const data) {
+  TRI_json_t* json = TRI_ObjectToJson(data);
+
+  if (! TRI_IsListJson(json)) {
+    return nullptr;
+  }
+
+  return TRI_CreateCursorResult((void*) json, &FreeData, &GetAt, &GetLength);
+}
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                      cursor store
 // -----------------------------------------------------------------------------

arangod/VocBase/general-cursor.h

@@ -36,6 +36,8 @@
 #include "VocBase/server.h"
 #include "VocBase/vocbase.h"
 
+#include <v8.h>
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                              forward declarations
 // -----------------------------------------------------------------------------
@@ -90,6 +92,12 @@ TRI_general_cursor_result_t* TRI_CreateCursorResult (void*,
 
 TRI_general_cursor_result_t* TRI_CreateResultGeneralCursor (TRI_json_t*);
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create a result set
+////////////////////////////////////////////////////////////////////////////////
+
+TRI_general_cursor_result_t* TRI_CreateResultGeneralCursor (v8::Handle<v8::Array> const);
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                      cursor store
 // -----------------------------------------------------------------------------

js/server/tests/aql-optimizer-rule-move-calculations-up.js

@@ -104,13 +104,16 @@ function optimizerRuleTestSuite () {
         "FOR i IN 1..10 LET a = 1 FILTER i == 1 RETURN a",
         "FOR i IN 1..10 FILTER i == 1 LET a = 1 RETURN i",
         "FOR i IN 1..10 LET a = 25 + 7 RETURN i",
+        "FOR i IN 1..10 LET a = MIN([ i, 1 ]) LET b = i + 1 RETURN [ a, b ]",
+        "FOR i IN 1..10 LET a = RAND() LET b = 25 + i RETURN i",
+        "FOR i IN 1..10 LET a = SLEEP(0.1) LET b = i + 1 RETURN b",
         "FOR i IN 1..10 FOR j IN 1..10 LET a = i + 2 RETURN i",
         "FOR i IN 1..10 FILTER i == 7 LET a = i COLLECT x = i RETURN x"
       ];
 
       queries.forEach(function(query) {
         var result = AQL_EXPLAIN(query, { }, paramEnabled);
-        assertEqual([ ruleName ], result.plan.rules);
+        assertEqual([ ruleName ], result.plan.rules, query);
       });
     },
 

js/server/tests/aql-optimizer-rule-move-filters-up.js

@@ -88,7 +88,7 @@ function optimizerRuleTestSuite () {
         "FOR i IN 1..10 LET a = i LET b = i FILTER a == b RETURN i",
         "FOR i IN 1..10 FOR j IN 1..10 FILTER i > j RETURN i",
         "FOR i IN 1..10 LET a = 2 * i FILTER a == 1 RETURN i",
-        "FOR i IN 1..10 LIMIT 1 FILTER i == 1 RETURN i"
+        "FOR i IN 1..10 LET a = SLEEP(1) FILTER i == 1 RETURN i" // SLEEP is non-deterministic
       ];
 
       queries.forEach(function(query) {
@@ -106,7 +106,8 @@ function optimizerRuleTestSuite () {
         "FOR i IN 1..10 FOR j IN 1..10 FILTER i > 1 RETURN i",
         "FOR i IN 1..10 LET x = (FOR j IN [i] RETURN j) FILTER i > 1 RETURN i",
         "FOR i IN 1..10 FOR l IN 1..10 LET a = 2 * i FILTER i == 1 RETURN a",
-        "FOR i IN 1..10 FOR l IN 1..10 LET a = 2 * i FILTER i == 1 LIMIT 1 RETURN a"
+        "FOR i IN 1..10 FOR l IN 1..10 LET a = 2 * i FILTER i == 1 LIMIT 1 RETURN a",
+        "FOR i IN 1..10 FOR l IN 1..10 LET a = MIN([1, l]) FILTER i == 1 LIMIT 1 RETURN a",
       ];
 
       queries.forEach(function(query) {

js/server/tests/aql-optimizer-rule-replace-or-with-in.js

@@ -0,0 +1,459 @@
+/*jshint strict: false, maxlen: 500 */
+/*global require, assertEqual, assertTrue, AQL_EXPLAIN, AQL_EXECUTE */
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief tests for Ahuacatl, skiplist index queries
+///
+/// @file
+///
+/// DISCLAIMER
+///
+/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is triAGENS GmbH, Cologne, Germany
+///
+/// @author 
+/// @author Copyright 2014, triAGENS GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+var internal = require("internal");
+var jsunity = require("jsunity");
+var helper = require("org/arangodb/aql-helper");
+var getQueryResults = helper.getQueryResults;
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test suite
+////////////////////////////////////////////////////////////////////////////////
+
+function NewAqlReplaceORWithINTestSuite () {
+  var replace;
+  var ruleName = "replace-OR-with-IN";
+  
+  var isRuleUsed = function (query, params) {
+   var result = AQL_EXPLAIN(query, params, { optimizer: { rules: [ "-all", "+" + ruleName ] } });
+   assertTrue(result.plan.rules.indexOf(ruleName) !== -1, query);
+   result = AQL_EXPLAIN(query, params, { optimizer: { rules: [ "-all" ] } });
+   assertTrue(result.plan.rules.indexOf(ruleName) === -1, query);
+  };
+  
+  var ruleIsNotUsed = function (query, params) {
+   var result = AQL_EXPLAIN(query, params, { optimizer: { rules: [ "-all", "+" + ruleName ] } });
+   assertTrue(result.plan.rules.indexOf(ruleName) === -1, query);
+  };
+
+  var executeWithRule = function (query, params) {
+    return AQL_EXECUTE(query, params, { optimizer: { rules: [ "-all", "+" + ruleName ] } }).json;
+  };
+
+  var executeWithoutRule = function (query, params) {
+    return AQL_EXECUTE(query, params, { optimizer: { rules: [ "-all" ] } }).json;
+  };
+
+  return {
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set up
+////////////////////////////////////////////////////////////////////////////////
+
+    setUp : function () {
+      internal.db._drop("UnitTestsNewAqlReplaceORWithINTestSuite");
+      replace = internal.db._create("UnitTestsNewAqlReplaceORWithINTestSuite");
+
+      for (var i = 1; i <= 10; ++i) {
+        replace.save({ "value" : i, x: [i]});
+        replace.save({"a" : {"b" : i}});
+        replace.save({"value": i + 10, "bb": i, "cc": 10 - i });
+
+      }
+
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief tear down
+////////////////////////////////////////////////////////////////////////////////
+
+    tearDown : function () {
+      internal.db._drop("UnitTestsNewAqlReplaceORWithINTestSuite");
+      replace = null;
+    },
+    
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test the rule fires for actual values
+////////////////////////////////////////////////////////////////////////////////
+
+    testFires2Values1 : function () {
+      var query = "FOR x IN " + replace.name() + 
+        " FILTER x.value == 1 || x.value == 2 SORT x.value RETURN x.value";
+      
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFires2Values2 : function () {
+      var query = "FOR x IN " + replace.name() + 
+        " FILTER x.a.b == 1 || x.a.b == 2 SORT x.a.b RETURN x.a.b";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFires4Values1 : function () {
+      var query = "FOR x IN " + replace.name() + " FILTER x.value == 1 " +
+        "|| x.value == 2 || x.value == 3 || x.value == 4 SORT x.value RETURN x.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2, 3, 4];
+      var actual = getQueryResults(query); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresNoAttributeAccess : function () {
+      var query = 
+        "FOR x IN " + replace.name() + " FILTER x == 1 || x == 2 RETURN x";
+
+      isRuleUsed(query, {});
+
+      var expected = [ ];
+      var actual = getQueryResults(query); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+
+    testFiresNoCollection : function () {
+      var query = 
+        "FOR x in 1..10 FILTER x == 1 || x == 2 SORT x RETURN x";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresBind : function () {
+      var query = 
+        "FOR v IN " + replace.name() 
+        + " FILTER v.value == @a || v.value == @b SORT v.value RETURN v.value";
+      var params = {"a": 1, "b": 2};
+
+      isRuleUsed(query, params);
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, params); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, params), executeWithoutRule(query, params));
+    },
+  
+    testFiresVariables : function () {
+      var query = 
+        "LET x = 1 LET y = 2 FOR v IN " + replace.name() 
+        + " FILTER v.value == x || v.value == y SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFires2AttributeAccesses1 : function () {
+      var query = 
+        "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+        + " FILTER v.value == x.a || v.value == x.b SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFires2AttributeAccesses2 : function () {
+      var query = 
+        "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+        + " FILTER x.a == v.value || v.value == x.b SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFires2AttributeAccesses3 : function () {
+      var query = 
+        "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+        + " FILTER x.a == v.value || x.b == v.value SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+
+    
+    testFiresMixed1 : function () {
+      var query = 
+        "LET x = [1,2] FOR v IN " + replace.name() 
+        + " FILTER x[0] == v.value || x[1] == v.value SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresMixed2 : function () {
+      var query = 
+        "LET x = [1,2] LET y = {b:2} FOR v IN " + replace.name() 
+        + " FILTER x[0] == v.value || y.b == v.value SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 2 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresSelfReference1 : function () {
+      var query = 
+        "FOR v IN " + replace.name() 
+        + " FILTER v.value == v.a.b  || v.value == 10 || v.value == 7 SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 7, 10 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresSelfReference2 : function () {
+      var query = 
+        "FOR v IN " + replace.name() 
+        + " FILTER v.a.b == v.value || v.value == 10 || 7 == v.value SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 7, 10 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+
+    testFiresAttributeIsList1 : function () {
+      var query = 
+       "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+       + " FILTER v.x[0] == x.a || v.x[0] == 3 SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresAttributeIsList2 : function () {
+      var query = 
+       "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+       + " FILTER x.a == v.x[0] || v.x[0] == 3 SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+
+    testFiresAttributeIsList3 : function () {
+      var query = 
+       "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+       + " FILTER x.a == v.x[0] || 3 == v.x[0] SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFiresAttributeIsList4 : function () {
+      var query = 
+       "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+       + " FILTER v.x[0] == x.a || 3 == v.x[0] SORT v.value RETURN v.value";
+
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+    },
+    
+    testFires2Loops: function () {
+      var query = 
+       "FOR x IN " + replace.name() 
+       + " FOR y IN "  + replace.name() 
+       + " FILTER x.value == y.bb || x.value == y.cc" 
+       + " FILTER x.value != null SORT x.value RETURN x.value";
+     
+      isRuleUsed(query, {});
+
+      var expected = [ 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10  ]; 
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+      assertEqual(expected, executeWithoutRule(query, {}));
+    },
+    
+    testFiresNonsense1: function () {
+      var query =
+        "FOR v in " + replace.name() 
+        + " FILTER 1 == 2 || v.value == 2 || v.value == 3 SORT v.value RETURN v.value" ;
+     
+      isRuleUsed(query, {});
+
+      var expected = [ 2, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+      assertEqual(expected, executeWithoutRule(query, {}));
+    },
+    
+    testFiresNonsense2: function () {
+      var query = "FOR v in " + replace.name() + 
+        " FILTER 1 == 2 || 2 == v.value || v.value == 3 SORT v.value RETURN v.value";
+     
+      isRuleUsed(query, {});
+
+      var expected = [ 2, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+      assertEqual(expected, executeWithoutRule(query, {}));
+    },
+    
+    testFiresNonsense3: function () {
+      var query = 
+      "FOR v in " + replace.name() 
+      + " FILTER v.value == 2 || 3 == v.value || 1 == 2 SORT v.value RETURN v.value";
+     
+      isRuleUsed(query, {});
+
+      var expected = [ 2, 3 ];
+      var actual = getQueryResults(query, {}); 
+      assertEqual(expected, actual);
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+      assertEqual(expected, executeWithoutRule(query, {}));
+    },
+    
+    testDudAlwaysTrue: function () {
+      var query = 
+      "FOR x IN " + replace.name() 
+      + " FILTER x.value == x.value || x.value == 2 || x.value == 3 SORT x.value RETURN x.value";
+     
+      ruleIsNotUsed(query, {});
+      assertEqual(executeWithRule(query, {}), executeWithoutRule(query, {}));
+
+    },
+
+    testDudDifferentAttributes1 : function () {
+      var query = 
+        "FOR x IN " + replace.name() + " FILTER x.val1 == 1 || x.val2 == 2 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+
+    testDudDifferentVariables : function () {
+      var query = 
+        "FOR y IN " + replace.name() + " FOR x IN " + replace.name() 
+        + " FILTER x.val1 == 1 || y.val1 == 2 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+    
+    testDudDifferentAttributes2: function () {
+      var query = 
+        "FOR x IN " + replace.name() + " FILTER x.val1 == 1 || x == 2 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+
+    testDudNoOR1: function () {
+      var query = 
+        "FOR x IN " + replace.name() + " FILTER x.val1 == 1 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+    
+    testDudNoOR2: function () {
+      var query = 
+        "FOR x IN " + replace.name() 
+        + " FILTER x.val1 == 1 && x.val2 == 2 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+
+    testDudNonEquality1: function () {
+      var query = 
+        "FOR x IN " + replace.name() 
+        + " FILTER x.val1 > 1 || x.val1 == 2 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+
+    testDudNonEquality2: function () {
+      var query = 
+        "FOR x IN " + replace.name() 
+        + " FILTER x.val1 == 1 ||  2 < x.val1 RETURN x";
+
+      ruleIsNotUsed(query, {});
+    },
+
+    testDudAttributeIsList: function () {
+      var query = 
+        "LET x = {a:1,b:2} FOR v IN " + replace.name() 
+        + " FILTER v.x[0] == x.a || v.x[1] == 3 SORT v.value RETURN v.value";
+      ruleIsNotUsed(query, {});
+    }
+  };
+}
+
+jsunity.run(NewAqlReplaceORWithINTestSuite);
+
+return jsunity.done();
+

js/server/tests/aql-skiplist-noncluster.js

@@ -1147,6 +1147,38 @@ function ahuacatlSkiplistTestSuite () {
       assertEqual(expected, actual);
         
       assertEqual([ "SingletonNode", "IndexRangeNode", "CalculationNode", "FilterNode", "CalculationNode", "CalculationNode", "SortNode", "CalculationNode", "ReturnNode" ], explain(query));
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test reverse iteration
+////////////////////////////////////////////////////////////////////////////////
+
+    testReverseIteration : function () {
+      var expectedOne = [ ], expectedTwo = [ ];
+      for (var i = 5; i >= 1; --i) {
+        for (var j = 5; j >= 1; --j) {
+          expectedOne.push(i);
+          expectedTwo.push([ i, j ]);
+        }
+      }
+
+      var query = "FOR a IN " + skiplist.name() + " SORT a.a DESC RETURN a.a";
+      var actual = getQueryResults(query);
+      assertEqual(expectedOne, actual);
+      
+      // produces an empty range
+      query = "FOR a IN " + skiplist.name() + " FILTER a.a >= 100 SORT a.a DESC RETURN a.a";
+      actual = getQueryResults(query);
+      assertEqual([ ], actual);
+      
+      query = "FOR a IN " + skiplist.name() + " SORT a.a DESC, a.b DESC RETURN [ a.a, a.b ]";
+      actual = getQueryResults(query);
+      assertEqual(expectedTwo, actual);
+      
+      // produces an empty range
+      query = "FOR a IN " + skiplist.name() + " FILTER a.a >= 100 SORT a.a DESC, a.b DESC RETURN [ a.a, a.b ]";
+      actual = getQueryResults(query);
+      assertEqual([ ], actual);
     }
 
   };