added internal.download()

html/admin/js/bootstrap/module-internal.js

@@ -1,6 +1,6 @@
 /*jslint indent: 2, nomen: true, maxlen: 100, sloppy: true, vars: true, white: true, plusplus: true, nonpropdel: true, proto: true */
 /*global require, module, Module, FS_MOVE, FS_REMOVE, FS_EXISTS, FS_IS_DIRECTORY, FS_IS_FILE, 
-  FS_LIST_TREE, FS_UNZIP_FILE, FS_ZIP_FILE,
+  FS_LIST_TREE, FS_UNZIP_FILE, FS_ZIP_FILE, SYS_DOWNLOAD,
   SYS_EXECUTE, SYS_LOAD, SYS_LOG, SYS_LOG_LEVEL, SYS_MD5, SYS_OUTPUT, SYS_PROCESS_STAT, SYS_RAND,
   SYS_READ, SYS_SPRINTF, SYS_TIME, SYS_START_PAGER, SYS_STOP_PAGER, SYS_SHA256, SYS_WAIT,
   SYS_GETLINE, SYS_PARSE, SYS_SAVE, SYS_IMPORT_CSV_FILE, SYS_IMPORT_JSON_FILE, PACKAGE_PATH,
@@ -58,6 +58,11 @@
 ////////////////////////////////////////////////////////////////////////////////
 
   // system functions
+  if (typeof SYS_DOWNLOAD !== "undefined") {
+    internal.download = SYS_DOWNLOAD;
+    delete SYS_DOWNLOAD;
+  }
+
   if (typeof SYS_EXECUTE !== "undefined") {
     internal.execute = SYS_EXECUTE;
     delete SYS_EXECUTE;

js/common/bootstrap/module-internal.js

@@ -1,6 +1,6 @@
 /*jslint indent: 2, nomen: true, maxlen: 100, sloppy: true, vars: true, white: true, plusplus: true, nonpropdel: true, proto: true */
 /*global require, module, Module, FS_MOVE, FS_REMOVE, FS_EXISTS, FS_IS_DIRECTORY, FS_IS_FILE, 
-  FS_LIST_TREE, FS_UNZIP_FILE, FS_ZIP_FILE,
+  FS_LIST_TREE, FS_UNZIP_FILE, FS_ZIP_FILE, SYS_DOWNLOAD,
   SYS_EXECUTE, SYS_LOAD, SYS_LOG, SYS_LOG_LEVEL, SYS_MD5, SYS_OUTPUT, SYS_PROCESS_STAT, SYS_RAND,
   SYS_READ, SYS_SPRINTF, SYS_TIME, SYS_START_PAGER, SYS_STOP_PAGER, SYS_SHA256, SYS_WAIT,
   SYS_GETLINE, SYS_PARSE, SYS_SAVE, SYS_IMPORT_CSV_FILE, SYS_IMPORT_JSON_FILE, PACKAGE_PATH,
@@ -58,6 +58,11 @@
 ////////////////////////////////////////////////////////////////////////////////
 
   // system functions
+  if (typeof SYS_DOWNLOAD !== "undefined") {
+    internal.download = SYS_DOWNLOAD;
+    delete SYS_DOWNLOAD;
+  }
+
   if (typeof SYS_EXECUTE !== "undefined") {
     internal.execute = SYS_EXECUTE;
     delete SYS_EXECUTE;

lib/Makefile.files

@@ -149,7 +149,7 @@ lib_libarango_fe_a_SOURCES = \
 ################################################################################
 ### @brief library "libarango.a", JavaScript part
 ################################################################################
-
+	
 lib_libarango_v8_a_SOURCES = \
 	lib/V8/JSLoader.cpp \
 	lib/V8/V8LineEditor.cpp \
@@ -158,7 +158,14 @@ lib_libarango_v8_a_SOURCES = \
 	lib/V8/v8-globals.cpp \
 	lib/V8/v8-json.cpp \
 	lib/V8/v8-shell.cpp \
-	lib/V8/v8-utils.cpp
+	lib/V8/v8-utils.cpp \
+	lib/SimpleHttpClient/GeneralClientConnection.cpp \
+	lib/SimpleHttpClient/ClientConnection.cpp \
+	lib/SimpleHttpClient/SslClientConnection.cpp \
+	lib/SimpleHttpClient/SimpleClient.cpp \
+	lib/SimpleHttpClient/SimpleHttpClient.cpp \
+	lib/SimpleHttpClient/SimpleHttpResult.cpp
+
 
 ################################################################################
 ### @brief library "libarango.a", Ruby part

lib/SimpleHttpClient/ClientConnection.cpp

@@ -72,6 +72,7 @@ ClientConnection::ClientConnection (Endpoint* endpoint,
 ////////////////////////////////////////////////////////////////////////////////
 
 ClientConnection::~ClientConnection () {
+  disconnect();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

lib/SimpleHttpClient/GeneralClientConnection.cpp

@@ -65,7 +65,6 @@ GeneralClientConnection::GeneralClientConnection (Endpoint* endpoint,
 ////////////////////////////////////////////////////////////////////////////////
 
 GeneralClientConnection::~GeneralClientConnection () {
-  disconnect();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

lib/SimpleHttpClient/SimpleHttpResult.cpp

@@ -125,6 +125,17 @@ namespace triagens {
       _headerFields[k] = value;
     }
 
+    string SimpleHttpResult::getHeaderField (const string& name, bool& found) const {
+      map<string, string>::const_iterator find = _headerFields.find(name);
+      if (find == _headerFields.end()) {
+        found = false;
+        return string("");
+      }
+
+      found = true;
+      return (*find).second;
+    }
+    
     const string SimpleHttpResult::getContentType (const bool partial) {
       map<string, string>::const_iterator find = _headerFields.find("content-type");
       if (find != _headerFields.end()) {

lib/SimpleHttpClient/SimpleHttpResult.h

@@ -107,6 +107,7 @@ namespace triagens {
       ~SimpleHttpResult ();
 
     public:
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief clear result values
 ////////////////////////////////////////////////////////////////////////////////
@@ -217,6 +218,12 @@ namespace triagens {
 
       void addHeaderField (std::string const& key, std::string const& value);
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief return the value of a single header
+////////////////////////////////////////////////////////////////////////////////
+    
+      string getHeaderField (const string&, bool&) const;
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief get X-VOC-* header fields
 ////////////////////////////////////////////////////////////////////////////////

lib/SimpleHttpClient/SslClientConnection.cpp

@@ -87,6 +87,8 @@ SslClientConnection::SslClientConnection (Endpoint* endpoint,
 ////////////////////////////////////////////////////////////////////////////////
 
 SslClientConnection::~SslClientConnection () {
+  disconnect();
+
   if (_ssl) {
     SSL_free(_ssl);
   }

lib/V8/v8-utils.cpp

@@ -50,7 +50,11 @@
 #include "BasicsC/utf8-helper.h"
 #include "BasicsC/zip.h"
 #include "Basics/FileUtils.h"
+#include "Rest/HttpRequest.h"
 #include "Rest/SslInterface.h"
+#include "SimpleHttpClient/GeneralClientConnection.h"
+#include "SimpleHttpClient/SimpleHttpClient.h"
+#include "SimpleHttpClient/SimpleHttpResult.h"
 #include "Statistics/statistics.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-globals.h"
@@ -61,6 +65,7 @@
 
 using namespace std;
 using namespace triagens::basics;
+using namespace triagens::httpclient;
 using namespace triagens::rest;
 
 // -----------------------------------------------------------------------------
@@ -348,6 +353,158 @@ static v8::Handle<v8::Value> JS_Parse (v8::Arguments const& argv) {
   return scope.Close(v8::True());
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief downloads data from a URL
+///
+/// @FUN{internal.download(@FA{url})}
+///
+/// Downloads the data from the URL specified by @FA{url}.
+////////////////////////////////////////////////////////////////////////////////
+
+static v8::Handle<v8::Value> JS_Download (v8::Arguments const& argv) {
+  v8::HandleScope scope;
+  
+  if (argv.Length() < 3) {
+    return scope.Close(v8::ThrowException(v8::String::New("usage: download(<url>, <method>, <outfile>, <timeout>)")));
+  }
+
+  string url = TRI_ObjectToString(argv[0]);
+
+  HttpRequest::HttpRequestType method = HttpRequest::HTTP_REQUEST_GET;
+  const string methodString = TRI_ObjectToString(argv[1]);
+  if (methodString == "head") {
+    method = HttpRequest::HTTP_REQUEST_HEAD;
+  }
+  else if (methodString == "delete") {
+    method = HttpRequest::HTTP_REQUEST_DELETE;
+  }
+
+  const string outfile = TRI_ObjectToString(argv[2]);
+
+  double timeout = 10.0;
+  if (argv.Length() > 3) {
+    timeout = TRI_ObjectToDouble(argv[3]);
+  }
+
+  if (TRI_ExistsFile(outfile.c_str())) {
+    return scope.Close(v8::ThrowException(TRI_CreateErrorObject(TRI_ERROR_CANNOT_OVERWRITE_FILE)));
+  }
+
+  int numRedirects = 0;
+
+  while (numRedirects++ < 5) {
+    string endpoint;
+    string relative;
+
+    if (url.substr(0, 7) == "http://") {
+      size_t found = url.find('/', 7);
+
+      relative = "/";
+      if (found != string::npos) {
+        relative.append(url.substr(found + 1));
+        endpoint = "tcp://" + url.substr(7, found - 7) + ":80";
+      }
+      else {
+        endpoint = "tcp://" + url.substr(7) + ":80";
+      }
+    }
+    else if (url.substr(0, 8) == "https://") {
+      size_t found = url.find('/', 8);
+
+      relative = "/";
+      if (found != string::npos) {
+        relative.append(url.substr(found + 1));
+        endpoint = "ssl://" + url.substr(8, found - 8) + ":443";
+      }
+      else {
+        endpoint = "ssl://" + url.substr(8) + ":443";
+      }
+    }
+    else {
+      return scope.Close(v8::ThrowException(v8::String::New("unsupported URL specified")));
+    }
+    
+    LOGGER_TRACE("downloading file. endpoint: " << endpoint << ", relative URL: " << url);
+
+    GeneralClientConnection* connection = GeneralClientConnection::factory(Endpoint::clientFactory(endpoint), timeout, timeout, 3);
+
+    if (connection == 0) {
+      return scope.Close(v8::ThrowException(TRI_CreateErrorObject(TRI_ERROR_OUT_OF_MEMORY)));
+    }
+
+    SimpleHttpClient client(connection, timeout, false);
+
+    v8::Handle<v8::Object> result = v8::Object::New();
+
+    // connect to server and get version number
+    map<string, string> headerFields;
+    SimpleHttpResult* response = client.request(method, relative, 0, 0, headerFields);
+
+    int returnCode;
+    string returnMessage;
+
+    if (! response || ! response->isComplete()) {
+      // save error message
+      returnMessage = client.getErrorMessage();
+      returnCode = 500;
+
+      if (response && response->getHttpReturnCode() > 0) {
+        returnCode = response->getHttpReturnCode();
+      }
+    }
+    else {
+      returnMessage = response->getHttpReturnMessage();
+      returnCode = response->getHttpReturnCode();
+
+      if (returnCode == 301 || returnCode == 302) {
+        bool found;
+        url = response->getHeaderField(string("location"), found);
+
+        delete response;
+        delete connection;
+
+        if (! found) {
+          return scope.Close(v8::ThrowException(TRI_CreateErrorObject(TRI_ERROR_INTERNAL, "caught invalid redirect URL")));
+        }
+        continue;
+      }
+
+      result->Set(v8::String::New("code"),    v8::Number::New(returnCode));
+      result->Set(v8::String::New("message"), v8::String::New(returnMessage.c_str()));
+
+      const map<string, string> responseHeaders = response->getHeaderFields();
+      map<string, string>::const_iterator it;
+
+      v8::Handle<v8::Object> headers = v8::Object::New();
+      for (it = responseHeaders.begin(); it != responseHeaders.end(); ++it) {
+        headers->Set(v8::String::New((*it).first.c_str()), v8::String::New((*it).second.c_str()));
+      }
+      result->Set(v8::String::New("headers"), headers);
+
+      if (returnCode == SimpleHttpResult::HTTP_STATUS_OK) {
+        try {
+          FileUtils::spit(outfile, response->getBody().str());
+        }
+        catch (...) {
+
+        }
+      }
+    }
+      
+    result->Set(v8::String::New("code"),    v8::Number::New(returnCode));
+    result->Set(v8::String::New("message"), v8::String::New(returnMessage.c_str()));
+
+    if (response) {
+      delete response;
+    }
+
+    delete connection;
+    return scope.Close(result);
+  }
+  
+  return scope.Close(v8::ThrowException(TRI_CreateErrorObject(TRI_ERROR_INTERNAL, "too many redirects")));
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief executes a script
 ///
@@ -1922,6 +2079,7 @@ void TRI_InitV8Utils (v8::Handle<v8::Context> context,
   TRI_AddGlobalFunctionVocbase(context, "FS_UNZIP_FILE", JS_UnzipFile);
   TRI_AddGlobalFunctionVocbase(context, "FS_ZIP_FILE", JS_ZipFile);
 
+  TRI_AddGlobalFunctionVocbase(context, "SYS_DOWNLOAD", JS_Download);
   TRI_AddGlobalFunctionVocbase(context, "SYS_EXECUTE", JS_Execute);
   TRI_AddGlobalFunctionVocbase(context, "SYS_GETLINE", JS_Getline);
   TRI_AddGlobalFunctionVocbase(context, "SYS_LOAD", JS_Load);