Feature/aql native call apply (#5100)

Documentation/Books/Manual/ReleaseNotes/UpgradingChanges34.md

@@ -34,3 +34,10 @@ REST API
 
 Miscellaneous
 -------------
+
+AQL Functions
+-------------
+- CALL / APPLY
+  - may emmit `ERROR_QUERY_FUNCTION_NAME_UNKNOWN` or `ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH`
+    instead of `ERROR_QUERY_FUNCTION_NOT_FOUND` in some situations.
+  - are now able to be invoked recursive

Installation/Jenkins/build.sh

@@ -495,6 +495,8 @@ fi
 
 if [ "${USE_JEMALLOC}" = 1 ]; then
     CONFIGURE_OPTIONS+=(-DUSE_JEMALLOC=On)
+else
+    CONFIGURE_OPTIONS+=(-DUSE_JEMALLOC=Off)
 fi
 
 if [ "$SANITIZE" == 1 ]; then

arangod/Aql/AqlFunctionFeature.cpp

@@ -365,8 +365,10 @@ void AqlFunctionFeature::addListFunctions() {
        &Functions::Nth});
   add({"POSITION", ".,.|.", true, false, true, true,
        &Functions::Position});
-  add({"CALL", ".|.+", false, true, false, true});
+  add({"CALL", ".|.+", false, true, false, true,
-  add({"APPLY", ".|.", false, true, false, false});
+        &Functions::Call});
+  add({"APPLY", ".|.", false, true, false, false,
+        &Functions::Apply});
   add({"PUSH", ".,.|.", true, false, true, false,
        &Functions::Push});
   add({"APPEND", ".,.|.", true, false, true, true,

arangod/Aql/Expression.cpp

@@ -95,7 +95,6 @@ Expression::Expression(ExecutionPlan* plan, Ast* ast, AstNode* node)
       _canRunOnDBServer(false),
       _isDeterministic(false),
       _willUseV8(false),
-      _preparedV8Context(false),
       _attributes(),
       _expressionContext(nullptr) {
   TRI_ASSERT(_ast != nullptr);
@@ -234,8 +233,6 @@ void Expression::invalidateAfterReplacements() {
 /// used and destroyed in the same context. when a V8 function is used across
 /// multiple V8 contexts, it must be invalidated in between
 void Expression::invalidate() {
-  // context may change next time, so "prepare for re-preparation"
-  _preparedV8Context = false;
 
   // V8 expressions need a special handling
   freeInternals();
@@ -909,28 +906,88 @@ AqlValue Expression::executeSimpleExpressionFCallCxx(
   }
 }
 
+AqlValue Expression::invokeV8Function(arangodb::aql::Query* query,
+                                      transaction::Methods* trx,
+                                      std::string const& jsName,
+                                      std::string const& ucInvokeFN,
+                                      char const* AFN,
+                                      bool rethrowV8Exception,
+                                      int callArgs,
+                                      v8::Handle<v8::Value>* args,
+                                      bool &mustDestroy
+                                      ){
+  ISOLATE;
+  auto current = isolate->GetCurrentContext()->Global();
+
+  v8::Handle<v8::Value> module = current->Get(TRI_V8_ASCII_STRING(isolate, "_AQL")); 
+  if (module.IsEmpty() || !module->IsObject()) {
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unable to find global _AQL module");
+  }
+
+  v8::Handle<v8::Value> function = v8::Handle<v8::Object>::Cast(module)->Get(TRI_V8_STD_STRING(isolate, jsName));
+  if (function.IsEmpty() || !function->IsFunction()) {
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, std::string("unable to find AQL function '") + jsName + "'");
+  }
+
+  // actually call the V8 function
+  v8::TryCatch tryCatch;
+  v8::Handle<v8::Value> result = v8::Handle<v8::Function>::Cast(function)->Call(current, callArgs, args); 
+
+  try {
+    V8Executor::HandleV8Error(tryCatch, result, nullptr, false);
+  }
+  catch (arangodb::basics::Exception const& ex) {
+    if (rethrowV8Exception) {
+      throw ex;
+    }
+    if (ex.code() == TRI_ERROR_QUERY_FUNCTION_NOT_FOUND) {
+      throw ex;
+    }
+    std::string message("While invoking '");
+    message +=  ucInvokeFN + "' via '" + AFN + "': " + ex.message();
+    query->registerWarning(ex.code(), message.c_str());
+    return AqlValue(AqlValueHintNull());
+  }
+  if (result.IsEmpty() || result->IsUndefined()) {
+    return AqlValue(AqlValueHintNull());
+  }
+
+  transaction::BuilderLeaser builder(trx);
+    
+  int res = TRI_V8ToVPack(isolate, *builder.get(), result, false);
+    
+  if (res != TRI_ERROR_NO_ERROR) {
+    THROW_ARANGO_EXCEPTION(res);
+  }
+  
+  mustDestroy = true; // builder = dynamic data       
+  return AqlValue(builder.get());
+}
+
 /// @brief execute an expression of type SIMPLE, JavaScript variant
 AqlValue Expression::executeSimpleExpressionFCallJS(
     AstNode const* node, transaction::Methods* trx, bool& mustDestroy) {
 
-  prepareV8Context();
-  
   auto member = node->getMemberUnchecked(0);
   TRI_ASSERT(member->type == NODE_TYPE_ARRAY);
     
   mustDestroy = false;
 
-  v8::Isolate* isolate = v8::Isolate::GetCurrent();
-  TRI_ASSERT(isolate != nullptr);
-
   {
 
-    v8::HandleScope scope(isolate); 
+    ISOLATE;
+    TRI_ASSERT(isolate != nullptr);
+    TRI_V8_CURRENT_GLOBALS_AND_SCOPE;
+    _ast->query()->prepareV8Context();
+    
+    auto old = v8g->_query;
+    v8g->_query = static_cast<void*>(_ast->query());
+    TRI_DEFER(v8g->_query = old);
 
     std::string jsName;
     size_t const n = member->numMembers();
-    size_t const callArgs = (node->type == NODE_TYPE_FCALL_USER ? 2 : n);
+    int callArgs = (node->type == NODE_TYPE_FCALL_USER ? 2 : n);
-    auto args = std::make_unique<v8::Handle<v8::Value>[]>(callArgs); 
+    v8::Handle<v8::Value> args[callArgs];
 
     if (node->type == NODE_TYPE_FCALL_USER) {
       // a call to a user-defined function
@@ -951,6 +1008,7 @@ AqlValue Expression::executeSimpleExpressionFCallJS(
       args[0] = TRI_V8_STD_STRING(isolate, node->getString());
       // call parameters
       args[1] = params;
+      // args[2] will be null
     } else {
       // a call to a built-in V8 function
       auto func = static_cast<Function*>(node->getData());
@@ -972,43 +1030,7 @@ AqlValue Expression::executeSimpleExpressionFCallJS(
       }
     }
 
-    TRI_v8_global_t* v8g = static_cast<TRI_v8_global_t*>(isolate->GetData(arangodb::V8PlatformFeature::V8_DATA_SLOT));
+    return invokeV8Function(_ast->query(), trx, jsName, "", "", true, callArgs, args, mustDestroy);
-    auto old = v8g->_query;
-    v8g->_query = static_cast<void*>(_ast->query());
-    TRI_DEFER(v8g->_query = old);
-
-    auto current = isolate->GetCurrentContext()->Global();
-
-    v8::Handle<v8::Value> module = current->Get(TRI_V8_ASCII_STRING(isolate, "_AQL")); 
-    if (module.IsEmpty() || !module->IsObject()) {
-      THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unable to find global _AQL module");
-    }
-
-    v8::Handle<v8::Value> function = v8::Handle<v8::Object>::Cast(module)->Get(TRI_V8_STD_STRING(isolate, jsName));
-    if (function.IsEmpty() || !function->IsFunction()) {
-      THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, std::string("unable to find AQL function '") + jsName + "'");
-    }
-
-    // actually call the V8 function
-    v8::TryCatch tryCatch;
-    v8::Handle<v8::Value> result = v8::Handle<v8::Function>::Cast(function)->Call(current, callArgs, args.get()); 
-    
-    V8Executor::HandleV8Error(tryCatch, result, nullptr, false);
-
-    if (result.IsEmpty() || result->IsUndefined()) {
-      return AqlValue(AqlValueHintNull());
-    }
-
-    transaction::BuilderLeaser builder(trx);
-    
-    int res = TRI_V8ToVPack(isolate, *builder.get(), result, false);
-    
-    if (res != TRI_ERROR_NO_ERROR) {
-      THROW_ARANGO_EXCEPTION(res);
-    }
-  
-    mustDestroy = true; // builder = dynamic data
-    return AqlValue(builder.get());
   }
 }
 
@@ -1654,31 +1676,3 @@ AqlValue Expression::executeSimpleExpressionArithmetic(
   // this will convert NaN, +inf & -inf to null
   return AqlValue(AqlValueHintDouble(result));
 }
-
-/// @brief prepare a V8 context for execution for this expression
-/// this needs to be called once before executing any V8 function in this
-/// expression
-void Expression::prepareV8Context() {
-  if (_preparedV8Context) {
-    // already done
-    return;
-  }
-  
-  TRI_ASSERT(_ast->query()->trx() != nullptr);
-
-  v8::Isolate* isolate = v8::Isolate::GetCurrent();
-  TRI_ASSERT(isolate != nullptr);
-
-  std::string body("if (_AQL === undefined) { _AQL = require(\"@arangodb/aql\"); _AQL.clearCaches(); }");
-  
-  {
-    v8::HandleScope scope(isolate); 
-    v8::Handle<v8::Script> compiled = v8::Script::Compile(
-        TRI_V8_STD_STRING(isolate, body), TRI_V8_ASCII_STRING(isolate, "--script--"));
-  
-    if (!compiled.IsEmpty()) {
-      v8::Handle<v8::Value> func(compiled->Run());
-      _preparedV8Context = true;
-    }
-  }
-}

arangod/Aql/Expression.h

@@ -32,6 +32,7 @@
 
 #include <velocypack/Builder.h>
 #include <velocypack/Slice.h>
+#include <v8.h>
 
 namespace arangodb {
 namespace transaction {
@@ -50,6 +51,7 @@ class Ast;
 class AttributeAccessor;
 class ExecutionPlan;
 class ExpressionContext;
+class Query;
 
 /// @brief AqlExpression, used in execution plans and execution blocks
 class Expression {
@@ -160,6 +162,17 @@ class Expression {
     return "unknown";
   }
 
+  // @brief invoke javascript aql functions with args as param.
+  static AqlValue invokeV8Function(arangodb::aql::Query* query,
+                                   transaction::Methods* trx,
+                                   std::string const& jsName,
+                                   std::string const& ucInvokeFN,
+                                   char const* AFN,
+                                   bool rethrowV8Exception,
+                                   int callArgs,
+                                   v8::Handle<v8::Value>* args,
+                                   bool &mustDestroy);
+
   /// @brief check whether this is an attribute access of any degree (e.g. a.b,
   /// a.b.c, ...)
   bool isAttributeAccess() const;
@@ -269,7 +282,6 @@ class Expression {
   AqlValue executeSimpleExpressionFCallCxx(AstNode const*,
                                            transaction::Methods*,
                                            bool& mustDestroy);
-  
   /// @brief execute an expression of type SIMPLE with FCALL, JavaScript variant
   AqlValue executeSimpleExpressionFCallJS(AstNode const*,
                                           transaction::Methods*,
@@ -337,11 +349,6 @@ class Expression {
       AstNode const*, transaction::Methods*, 
       bool& mustDestroy);
 
-  /// @brief prepare a V8 context for execution for this expression
-  /// this needs to be called once before executing any V8 function in this
-  /// expression
-  void prepareV8Context();
-
  private:
   /// @brief the query execution plan. note: this may be a nullptr for expressions
   /// created in the early optimization stage!
@@ -374,11 +381,6 @@ class Expression {
   /// @brief whether or not the expression will make use of V8
   bool _willUseV8;
 
-  /// @brief whether or not the preparation routine for V8 contexts was run
-  /// once for this expression
-  /// it needs to be run once before any V8-based function is called
-  bool _preparedV8Context;
-
   /// @brief the top-level attributes used in the expression, grouped
   /// by variable name
   std::unordered_map<Variable const*, std::unordered_set<std::string>>

arangod/Aql/Functions.cpp

@@ -25,9 +25,12 @@
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "ApplicationFeatures/LanguageFeature.h"
+#include "Aql/AqlFunctionFeature.h"
+#include "Aql/Expression.h"
 #include "Aql/Function.h"
 #include "Aql/Query.h"
 #include "Aql/RegexCache.h"
+#include "Aql/V8Executor.h"
 #include "Basics/Exceptions.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/StringRef.h"
@@ -37,6 +40,7 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/fpconv.h"
 #include "Basics/tri-strings.h"
+#include "V8/v8-vpack.h"
 #include "GeneralServer/AuthenticationFeature.h"
 #include "Indexes/Index.h"
 #include "Logger/Logger.h"
@@ -112,7 +116,7 @@ static_assert(DateSelectionModifier::MONTH < DateSelectionModifier::YEAR,
     - ICU related errors: if (U_FAILURE(status)) { RegisterICUWarning(query, "MYFUNC", status); }
     - close with: return AqlValue(AqlValueHintNull());
 - specify the number of parameters you expect at least and at max using: 
-  ValidateParameters(parameters, "MYFUNC", 1, 3); (min: 1, max: 3)
+  ValidateParameters(parameters, "MYFUNC", 1, 3); (min: 1, max: 3); Max is optional.
 - if you support optional parameters, first check whether the count is sufficient
   using parameters.size()
 - fetch the values using:
@@ -483,6 +487,24 @@ void Functions::RegisterWarning(arangodb::aql::Query* query, char const* fName,
   query->registerWarning(code, msg.c_str());
 }
 
+/// @brief register warning
+void Functions::RegisterError(arangodb::aql::Query* query,
+                              char const* fName,
+                              int code) {
+  std::string msg;
+
+  if (code == TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH) {
+    msg = arangodb::basics::Exception::FillExceptionString(code, fName);
+  } else {
+    msg.append("in function '");
+    msg.append(fName);
+    msg.append("()': ");
+    msg.append(TRI_errno_string(code));
+  }
+
+  query->registerError(code, msg.c_str());
+}
+
 /// @brief register usage of an invalid function argument
 void Functions::RegisterInvalidArgumentWarning(arangodb::aql::Query* query,
                                                char const* functionName) {
@@ -1681,13 +1703,13 @@ AqlValue Functions::Substitute(arangodb::aql::Query* query,
     for (auto const& it : VPackObjectIterator(slice)) {
       arangodb::velocypack::ValueLength length;
       const char *str = it.key.getString(length);
-      matchPatterns.push_back(UnicodeString(str, length));
+      matchPatterns.push_back(UnicodeString(str, static_cast<int32_t>(length)));
       if (!it.value.isString()) {
         RegisterInvalidArgumentWarning(query, AFN);
         return AqlValue(AqlValueHintNull());
       }
       str = it.value.getString(length);
-      replacePatterns.push_back(UnicodeString(str, length));
+      replacePatterns.push_back(UnicodeString(str, static_cast<int32_t>(length)));
     }
   }
   else {
@@ -1708,7 +1730,7 @@ AqlValue Functions::Substitute(arangodb::aql::Query* query,
         }
         arangodb::velocypack::ValueLength length;
         const char *str = it.getString(length);
-        matchPatterns.push_back(UnicodeString(str, length));
+        matchPatterns.push_back(UnicodeString(str, static_cast<int32_t>(length)));
       }
     }
     else {
@@ -1718,7 +1740,7 @@ AqlValue Functions::Substitute(arangodb::aql::Query* query,
       }
       arangodb::velocypack::ValueLength length;
       const char *str = slice.getString(length);
-      matchPatterns.push_back(UnicodeString(str, length));
+      matchPatterns.push_back(UnicodeString(str, static_cast<int32_t>(length)));
     }
     if (parameters.size() > 2) {
       AqlValue replace = ExtractFunctionParameterValue(parameters, 2);
@@ -1731,7 +1753,7 @@ AqlValue Functions::Substitute(arangodb::aql::Query* query,
           }
           arangodb::velocypack::ValueLength length;
           const char *str = it.getString(length);
-          replacePatterns.push_back(UnicodeString(str, length));
+          replacePatterns.push_back(UnicodeString(str, static_cast<int32_t>(length)));
         }
       }
       else if (replace.isString()) {
@@ -1740,7 +1762,7 @@ AqlValue Functions::Substitute(arangodb::aql::Query* query,
         replaceWasPlainString = true;
         arangodb::velocypack::ValueLength length;
         const char *str = rslice.getString(length);
-        replacePatterns.push_back(UnicodeString(str, length));
+        replacePatterns.push_back(UnicodeString(str, static_cast<int32_t>(length)));
       }
       else {
         RegisterInvalidArgumentWarning(query, AFN);
@@ -5628,6 +5650,159 @@ AqlValue Functions::Position(arangodb::aql::Query* query,
   return AqlValue(builder.get());
 }
 
+
+AqlValue Functions::CallApplyBackend(arangodb::aql::Query* query,
+                                     transaction::Methods* trx,
+                                     VPackFunctionParameters const& parameters,
+                                     char const* AFN,
+                                     AqlValue const& invokeFN,
+                                     VPackFunctionParameters const& invokeParams) {
+  std::string ucInvokeFN;
+  transaction::StringBufferLeaser buffer(trx);
+  arangodb::basics::VPackStringBufferAdapter adapter(buffer->stringBuffer());
+
+  AppendAsString(trx, adapter, invokeFN);
+
+  UnicodeString unicodeStr(buffer->c_str(),
+                           static_cast<int32_t>(buffer->length()));
+  unicodeStr.toUpper(NULL);
+  unicodeStr.toUTF8String(ucInvokeFN);
+
+  arangodb::aql::Function const* func = nullptr;
+  if (ucInvokeFN.find("::") == std::string::npos) {
+    func = AqlFunctionFeature::getFunctionByName(ucInvokeFN);
+    if (func->implementation != nullptr) {
+      return func->implementation(query, trx, invokeParams);
+    }
+  }
+
+  {
+    ISOLATE;
+    TRI_V8_CURRENT_GLOBALS_AND_SCOPE;
+    query->prepareV8Context();
+    
+    auto old = v8g->_query;
+    v8g->_query = query;
+    TRI_DEFER(v8g->_query = old);
+
+    std::string jsName;
+    size_t const n = invokeParams.size();
+    int const callArgs = (func == nullptr ? 3 : n);
+    v8::Handle<v8::Value> args[callArgs]; 
+
+    if (func == nullptr) {
+      // a call to a user-defined function
+      jsName = "FCALL_USER";
+
+      // function name
+      args[0] = TRI_V8_STD_STRING(isolate, ucInvokeFN);
+      // call parameters
+      v8::Handle<v8::Array> params = v8::Array::New(isolate, static_cast<int>(n));
+      
+      for (size_t i = 0; i < n; ++i) {
+        params->Set(static_cast<uint32_t>(i), invokeParams[i].toV8(isolate, trx));
+      }
+      args[1] = params;
+      args[2] = TRI_V8_ASCII_STRING(isolate, AFN);
+    } else {
+      // a call to a built-in V8 function
+      jsName = "AQL_" + func->nonAliasedName;
+      for (size_t i = 0; i < n; ++i) {
+        args[i] = invokeParams[i].toV8(isolate, trx);
+      }
+    }
+
+    bool dummy;
+    return Expression::invokeV8Function(query, trx, jsName, ucInvokeFN, AFN, false, callArgs, args, dummy);
+  }
+}
+
+
+
+/// @brief function CALL
+AqlValue Functions::Call(arangodb::aql::Query* query,
+                         transaction::Methods* trx,
+                         VPackFunctionParameters const& parameters) {
+  static char const* AFN = "CALL";
+  ValidateParameters(parameters, AFN, 1);
+
+  AqlValue invokeFN = ExtractFunctionParameterValue(parameters, 0);
+  if (!invokeFN.isString()) { 
+    RegisterError(query, AFN, TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH);
+    return AqlValue(AqlValueHintNull());
+  }
+
+  SmallVector<AqlValue>::allocator_type::arena_type arena;
+  VPackFunctionParameters invokeParams{arena};
+  if (parameters.size() >= 2) {
+    // we have a list of parameters, need to copy them over except the functionname:
+    invokeParams.reserve(parameters.size() -1);
+
+    for (uint64_t i = 1; i < parameters.size(); i++) {
+      invokeParams.push_back(ExtractFunctionParameterValue(parameters, i));
+    }
+  }
+
+  return CallApplyBackend(query, trx, parameters, AFN, invokeFN, invokeParams);
+}
+
+/// @brief function APPLY
+AqlValue Functions::Apply(
+    arangodb::aql::Query* query, transaction::Methods* trx,
+    VPackFunctionParameters const& parameters) {
+  static char const* AFN = "APPLY";
+  ValidateParameters(parameters, AFN, 1, 2);
+
+  AqlValue invokeFN = ExtractFunctionParameterValue(parameters, 0);
+  if (!invokeFN.isString()) { 
+    RegisterError(query, AFN, TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH);
+    return AqlValue(AqlValueHintNull());
+  }
+
+  SmallVector<AqlValue>::allocator_type::arena_type arena;
+  VPackFunctionParameters invokeParams{arena};
+  VPackSlice paramArray;
+  AqlValue rawParamArray;
+  std::vector<bool> mustFree;
+  if (parameters.size() == 2) {
+    // We have a parameter that should be an array, whichs content we need to make
+    // the sub functions parameters.
+    rawParamArray = ExtractFunctionParameterValue(parameters, 1);
+
+    if (!rawParamArray.isArray()) {
+      RegisterWarning(query, AFN, TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH);
+      return AqlValue(AqlValueHintNull());
+    }
+    uint64_t len = rawParamArray.length();
+    invokeParams.reserve(len);
+    mustFree.reserve(len);
+    for (uint64_t i = 0; i < len; i++) {
+      bool f;
+      invokeParams.push_back(rawParamArray.at(trx, i, f, false));
+      mustFree[i] = f;
+    }
+  }
+
+  try {
+    auto rc = CallApplyBackend(query, trx, parameters, AFN, invokeFN, invokeParams);
+    for (size_t i = 0; i < mustFree.size(); ++i) {
+      if (mustFree[i]) {
+        invokeParams[i].destroy();
+      }
+    }
+
+    return rc;
+  }
+  catch (...) {
+    for (size_t i = 0; i < mustFree.size(); ++i) {
+      if (mustFree[i]) {
+        invokeParams[i].destroy();
+      }
+    }
+    throw;
+  }
+}
+
 /// @brief function IS_SAME_COLLECTION
 AqlValue Functions::IsSameCollection(
     arangodb::aql::Query* query, transaction::Methods* trx,

arangod/Aql/Functions.h

@@ -75,6 +75,9 @@ struct Functions {
    /// @brief register warning
    static void RegisterWarning(arangodb::aql::Query* query,
                        char const* functionName, int code);
+   /// @brief register error
+   static void RegisterError(arangodb::aql::Query* query,
+                             char const* functionName, int code);
    /// @brief register usage of an invalid function argument
    static void RegisterInvalidArgumentWarning(arangodb::aql::Query* query,
                                               char const* functionName);
@@ -456,6 +459,16 @@ struct Functions {
                           VPackFunctionParameters const&);
     static AqlValue Position(arangodb::aql::Query*, transaction::Methods*,
                              VPackFunctionParameters const&);
+    static AqlValue CallApplyBackend(arangodb::aql::Query* query,
+                                     transaction::Methods* trx,
+                                     VPackFunctionParameters const& parameters,
+                                     char const* AFN,
+                                     AqlValue const& invokeFN,
+                                     VPackFunctionParameters const& invokeParams);
+    static AqlValue Call(arangodb::aql::Query*, transaction::Methods*,
+                         VPackFunctionParameters const&);
+    static AqlValue Apply(arangodb::aql::Query*, transaction::Methods*,
+                          VPackFunctionParameters const&);  
     static AqlValue IsSameCollection(arangodb::aql::Query*,
                                      transaction::Methods*,
                                      VPackFunctionParameters const&);

arangod/Aql/Query.cpp

@@ -90,7 +90,8 @@ Query::Query(bool contextOwnedByExterior, TRI_vocbase_t* vocbase,
       _part(part),
       _contextOwnedByExterior(contextOwnedByExterior),
       _killed(false),
-      _isModificationQuery(false) {
+      _isModificationQuery(false),
+      _preparedV8Context(false) {
     
   AqlFeature* aql = AqlFeature::lease();
   if (aql == nullptr) {
@@ -168,7 +169,8 @@ Query::Query(bool contextOwnedByExterior, TRI_vocbase_t* vocbase,
       _part(part),
       _contextOwnedByExterior(contextOwnedByExterior),
       _killed(false),
-      _isModificationQuery(false) {
+      _isModificationQuery(false),
+      _preparedV8Context(false) {
   
   AqlFeature* aql = AqlFeature::lease();
   if (aql == nullptr) {
@@ -1000,6 +1002,34 @@ void Query::releaseEngine() {
   _engine.release();
 }
 
+/// @brief prepare a V8 context for execution for this expression
+/// this needs to be called once before executing any V8 function in this
+/// expression
+void Query::prepareV8Context() {
+  if (_preparedV8Context) {
+    // already done
+    return;
+  }
+
+  TRI_ASSERT(_trx != nullptr);
+
+  ISOLATE;
+  TRI_ASSERT(isolate != nullptr);
+
+  std::string body("if (_AQL === undefined) { _AQL = require(\"@arangodb/aql\"); _AQL.clearCaches(); }");
+  
+  {
+    v8::HandleScope scope(isolate); 
+    v8::Handle<v8::Script> compiled = v8::Script::Compile(
+        TRI_V8_STD_STRING(isolate, body), TRI_V8_ASCII_STRING(isolate, "--script--"));
+  
+    if (!compiled.IsEmpty()) {
+      v8::Handle<v8::Value> func(compiled->Run());
+      _preparedV8Context = true;
+    }
+  }
+}
+
 /// @brief enter a V8 context
 void Query::enterContext() {
   if (!_contextOwnedByExterior) {
@@ -1016,12 +1046,14 @@ void Query::enterContext() {
 
       ISOLATE;
       TRI_GET_GLOBALS();
+      _preparedV8Context = false;
       auto ctx = static_cast<arangodb::transaction::V8Context*>(
           v8g->_transactionContext);
       if (ctx != nullptr) {
         ctx->registerTransaction(_trx->state());
       }
     }
+    _preparedV8Context = false;
 
     TRI_ASSERT(_context != nullptr);
   }
@@ -1043,6 +1075,7 @@ void Query::exitContext() {
       V8DealerFeature::DEALER->exitContext(_context);
       _context = nullptr;
     }
+    _preparedV8Context = false;
   }
 }
 

arangod/Aql/Query.h

@@ -219,6 +219,11 @@ class Query {
   /// @brief mark a query as modification query
   void setIsModificationQuery() { _isModificationQuery = true; }
 
+  /// @brief prepare a V8 context for execution for this expression
+  /// this needs to be called once before executing any V8 function in this
+  /// expression
+  void prepareV8Context();
+
   /// @brief enter a V8 context
   void enterContext();
 
@@ -375,6 +380,12 @@ class Query {
 
   /// @brief whether or not the query is a data modification query
   bool _isModificationQuery;
+
+  /// @brief whether or not the preparation routine for V8 contexts was run
+  /// once for this expression
+  /// it needs to be run once before any V8-based function is called
+  bool _preparedV8Context;
+
 };
 }
 }

js/server/modules/@arangodb/aql.js

@@ -819,7 +819,7 @@ function COMPILE_LIKE (regex, modifiers) {
 // / @brief call a user function
 // //////////////////////////////////////////////////////////////////////////////
 
-function FCALL_USER (name, parameters) {
+function FCALL_USER (name, parameters, func) {
   'use strict';
 
   var prefix = DB_PREFIX(), reloaded = false;
@@ -834,7 +834,7 @@ function FCALL_USER (name, parameters) {
   }
 
   if (!UserFunctions[prefix].hasOwnProperty(name)) {
-    THROW(null, INTERNAL.errors.ERROR_QUERY_FUNCTION_NOT_FOUND, name);
+    THROW(func, INTERNAL.errors.ERROR_QUERY_FUNCTION_NOT_FOUND, name);
   }
 
   try {
@@ -5835,7 +5835,6 @@ exports.AQL_WARN = AQL_WARN;
 
 exports.reload = reloadUserFunctions;
 exports.clearCaches = clearCaches;
-exports.lookupFunction = GET_USERFUNCTION;
 exports.throwFromFunction = THROW;
 exports.fixValue = FIX_VALUE;
 

js/server/tests/aql/aql-call-apply.js

@@ -68,7 +68,11 @@ function ahuacatlCallApplyTestSuite () {
 
       data.forEach(function (d) {
         var actual = getQueryResults("RETURN CALL(" + d[1].map(function (v) { return JSON.stringify(v); }).join(", ") + ")");
+        if (Array.isArray(d[0])) {
+          assertEqual(d[0].sort(), actual[0].sort(), d);
+        } else {
           assertEqual(d[0], actual[0], d);
+        }
       });
     },
 
@@ -97,23 +101,35 @@ function ahuacatlCallApplyTestSuite () {
     testCallNonExisting : function () {
       assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN CALL()"); 
 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN CALL('nono-existing', [ 'baz' ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code, "RETURN CALL('nono-existing', [ 'baz' ])"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN CALL('foobar', 'baz')"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code, "RETURN CALL('foobar', 'baz')"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN CALL(' trim', 'baz')"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code, "RETURN CALL(' trim', 'baz')"); 
       assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN CALL('foo::bar::baz', 'baz')"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN CALL(123, 'baz')"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN CALL(123, 'baz')"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN CALL([ ], 'baz')"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN CALL([ ], 'baz')"); 
     },
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test call function
 ////////////////////////////////////////////////////////////////////////////////
+    testCallRecursive : function () {
+      var actual = getQueryResults("RETURN CALL('CALL', 'TRIM', '  foo bar  ')");
+      assertEqual(actual, [ 'foo bar' ]);
+      actual = getQueryResults("RETURN CALL('APPLY', 'TRIM', ['  foo bar  '])");
+      assertEqual(actual, [ 'foo bar' ]);
+
+      let recursion = [];
+      for (let i = 0; i < 100; i++) {
+        recursion.push('CALL');
+      }
+      recursion.push('TRIM');
+      recursion.push('  foo bar  ');
+      let query = "RETURN CALL('" + recursion.join('\',\'') + "')";
+      actual = getQueryResults(query);
+      
+      assertEqual(actual, [ 'foo bar' ]);
 
-    testCallDisallowed : function () {
-      assertQueryError(errors.ERROR_QUERY_DISALLOWED_DYNAMIC_CALL.code, "RETURN CALL('CALL')"); 
-      assertQueryError(errors.ERROR_QUERY_DISALLOWED_DYNAMIC_CALL.code, "RETURN CALL('APPLY')"); 
     },
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test apply function
 ////////////////////////////////////////////////////////////////////////////////
@@ -144,7 +160,11 @@ function ahuacatlCallApplyTestSuite () {
           args.push(d[1][i]);
         }
         var actual = getQueryResults("RETURN APPLY(" + JSON.stringify(d[1][0]) + ", " + JSON.stringify(args) + ")");
+        if (Array.isArray(d[0])) {
+          assertEqual(d[0].sort(), actual[0].sort(), d);
+        } else {
           assertEqual(d[0], actual[0], d);
+        }
       });
     },
 
@@ -174,21 +194,42 @@ function ahuacatlCallApplyTestSuite () {
       assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN APPLY()"); 
       assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN APPLY('TRIM', 1, 2)"); 
 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN APPLY('nono-existing', [ 'baz' ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code, "RETURN APPLY('nono-existing', [ 'baz' ])"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN APPLY('foobar', [ 'baz' ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code, "RETURN APPLY('foobar', [ 'baz' ])"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN APPLY(' trim', [ 'baz' ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code, "RETURN APPLY(' trim', [ 'baz' ])"); 
       assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN APPLY('foo::bar::baz', [ 'baz' ])"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN APPLY(123, [ 'baz' ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN APPLY(123, [ 'baz' ])"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code, "RETURN APPLY([ ], [ 'baz' ])"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN APPLY([ ], [ 'baz' ])"); 
     },
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test apply function
 ////////////////////////////////////////////////////////////////////////////////
 
-    testApplyDisallowed : function () {
+    testApplyRecursive : function () {
-      assertQueryError(errors.ERROR_QUERY_DISALLOWED_DYNAMIC_CALL.code, "RETURN APPLY('CALL')"); 
+      var actual = getQueryResults("RETURN APPLY('CALL', ['TRIM', '  foo bar  '])");
-      assertQueryError(errors.ERROR_QUERY_DISALLOWED_DYNAMIC_CALL.code, "RETURN APPLY('APPLY')"); 
+      assertEqual(actual, [ 'foo bar' ]);
+      actual = getQueryResults("RETURN APPLY('APPLY', ['TRIM', ['  foo bar  ']])");
+      assertEqual(actual, [ 'foo bar' ]);
+
+      let recursion = '';
+      let close = '';
+      let rDepth = 20;
+      for (let i = 0; i < rDepth; i++) {
+        if (i > 0) {
+          recursion += '[';
+        }
+        recursion += '\'APPLY\'';
+        if (i < rDepth) {
+          recursion += ',';
+        }
+        close += ']';
+      }
+      recursion += '[\'TRIM\', [ \'  foo bar  \'] ' + close;
+      let query = "RETURN APPLY(" + recursion + ")";
+      actual = getQueryResults(query);
+      
+      assertEqual(actual, [ 'foo bar' ]);
     }
 
   };

js/server/tests/aql/aql-functions-brute.js

@@ -203,6 +203,7 @@ function ahuacatlFunctionsBruteTestSuite () {
       errors.ERROR_ARANGO_CROSS_COLLECTION_REQUEST.code,
       errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code,
       errors.ERROR_QUERY_FUNCTION_NOT_FOUND.code,
+      errors.ERROR_QUERY_FUNCTION_NAME_UNKNOWN.code,
       errors.ERROR_GRAPH_NOT_FOUND.code,
       errors.ERROR_GRAPH_INVALID_GRAPH.code,
     ].indexOf(err.errorNum) !== -1;

scripts/build-dbg-deb.sh

@@ -30,7 +30,6 @@ fi
     --package DEB \
     --buildDir build-${EP}deb-dbg \
     --targetDir /var/tmp/ \
-    --jemalloc \
     --noopt \
     ${MOREOPTS} \
     $@