made internal.download() function more flexible

required by @mchacki
can now send custom headers and use non-GET method

js/client/modules/org/arangodb/aal.js

@@ -329,7 +329,7 @@ function processGithubRepository (source) {
   var tempFile = fs.getTempFile("downloads", false); 
 
   try {
-    var result = internal.download(url, "get", tempFile);
+    var result = internal.download(url, "", { method: "get", followRedirects: true, timeout: 30 }, tempFile);
 
     if (result.code >= 200 && result.code <= 299) {
       source.filename = tempFile;
@@ -471,7 +471,7 @@ function updateFishbowl () {
   var path = fs.getTempFile("zip", false); 
 
   try {
-    var result = internal.download(url, "get", filename);
+    var result = internal.download(url, "", { method: "get", followRedirects: true, timeout: 30 }, filename);
 
     if (result.code < 200 || result.code > 299) {
       throw "github download failed";

lib/Rest/HttpRequest.cpp

@@ -144,39 +144,10 @@ char const* HttpRequest::requestPath () const {
 ////////////////////////////////////////////////////////////////////////////////
 
 void HttpRequest::write (TRI_string_buffer_t* buffer) const {
-  switch (_type) {
+  const string& method = translateMethod(_type);
-    case HTTP_REQUEST_GET:
-      TRI_AppendString2StringBuffer(buffer, "GET ", 4);
-      break;
 
-    case HTTP_REQUEST_POST:
+  TRI_AppendString2StringBuffer(buffer, method.c_str(), method.size());
-      TRI_AppendString2StringBuffer(buffer, "POST ", 5);
+  TRI_AppendCharStringBuffer(buffer, ' ');
-      break;
-
-    case HTTP_REQUEST_PUT:
-      TRI_AppendString2StringBuffer(buffer, "PUT ", 4);
-      break;
-
-    case HTTP_REQUEST_DELETE:
-      TRI_AppendString2StringBuffer(buffer, "DELETE ", 7);
-      break;
-
-    case HTTP_REQUEST_HEAD:
-      TRI_AppendString2StringBuffer(buffer, "HEAD ", 5);
-      break;
-
-    case HTTP_REQUEST_OPTIONS:
-      TRI_AppendString2StringBuffer(buffer, "OPTIONS ", 8);
-      break;
-
-    case HTTP_REQUEST_PATCH:
-      TRI_AppendString2StringBuffer(buffer, "PATCH ", 6);
-      break;
-
-    default:
-      TRI_AppendString2StringBuffer(buffer, "UNKNOWN ", 8);
-      break;
-  }
 
   // do NOT url-encode the path, we need to distingush between
   // "/document/a/b" and "/document/a%2fb"
@@ -1194,38 +1165,76 @@ void HttpRequest::addSuffix (char const* part) {
 // --SECTION--                                             public static methods
 // -----------------------------------------------------------------------------
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief translate an enum value into an HTTP method string
+////////////////////////////////////////////////////////////////////////////////
+
+string HttpRequest::translateMethod (const HttpRequestType method) {
+  if (method == HTTP_REQUEST_DELETE) {
+    return "DELETE";
+  }
+  else if (method == HTTP_REQUEST_GET) {
+    return "GET";
+  }
+  else if (method == HTTP_REQUEST_HEAD) {
+    return "HEAD";
+  }
+  else if (method == HTTP_REQUEST_OPTIONS) {
+    return "OPTIONS";
+  }
+  else if (method == HTTP_REQUEST_PATCH) {
+    return "PATCH";
+  }
+  else if (method == HTTP_REQUEST_POST) {
+    return "POST";
+  }
+  else if (method == HTTP_REQUEST_PUT) {
+    return "PUT";
+  }
+
+  LOGGER_WARNING("illegal http request method encountered in switch");
+  return "UNKNOWN";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief translate an HTTP method string into an enum value
+////////////////////////////////////////////////////////////////////////////////
+
+HttpRequest::HttpRequestType HttpRequest::translateMethod (const string& method) {
+  const string methodString = StringUtils::toupper(method);
+
+  if (methodString == "DELETE") {
+    return HTTP_REQUEST_DELETE;
+  }
+  else if (methodString == "GET") {
+    return HTTP_REQUEST_GET;
+  }
+  else if (methodString == "HEAD") {
+    return HTTP_REQUEST_HEAD;
+  }
+  else if (methodString == "OPTIONS") {
+    return HTTP_REQUEST_OPTIONS;
+  }
+  else if (methodString == "PATCH") {
+    return HTTP_REQUEST_PATCH;
+  }
+  else if (methodString == "POST") {
+    return HTTP_REQUEST_POST;
+  }
+  else if (methodString == "PUT") {
+    return HTTP_REQUEST_PUT;
+  }
+
+  return HTTP_REQUEST_ILLEGAL;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief append the request method string to a string buffer
 ////////////////////////////////////////////////////////////////////////////////
 
 void HttpRequest::appendMethod (HttpRequestType method, StringBuffer* buffer) {
-  switch (method) {
+  buffer->appendText(translateMethod(method));
-    case HTTP_REQUEST_GET:
+  buffer->appendChar(' ');
-      buffer->appendText("GET ");
-      break;
-    case HTTP_REQUEST_POST:
-      buffer->appendText("POST ");
-      break;
-    case HTTP_REQUEST_PUT:
-      buffer->appendText("PUT ");
-      break;
-    case HTTP_REQUEST_DELETE:
-      buffer->appendText("DELETE ");
-      break;
-    case HTTP_REQUEST_OPTIONS:
-      buffer->appendText("OPTIONS ");
-      break;
-    case HTTP_REQUEST_PATCH:
-      buffer->appendText("PATCH ");
-      break;
-    case HTTP_REQUEST_HEAD:
-      buffer->appendText("HEAD ");
-      break;
-    case HTTP_REQUEST_ILLEGAL:
-      buffer->appendText("UNKNOWN ");
-      LOGGER_WARNING("illegal http request method encountered in switch");
-      break;
-  }
 }
 
 ////////////////////////////////////////////////////////////////////////////////

lib/Rest/HttpRequest.h

@@ -417,6 +417,18 @@ namespace triagens {
 // --SECTION--                                             public static methods
 // -----------------------------------------------------------------------------
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief translate an enum value into an HTTP method string
+////////////////////////////////////////////////////////////////////////////////
+
+        static string translateMethod (const HttpRequestType);
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief translate an HTTP method string into an enum value
+////////////////////////////////////////////////////////////////////////////////
+
+        static HttpRequestType translateMethod (const string&);
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief append the request method string to a string buffer
 ////////////////////////////////////////////////////////////////////////////////

lib/V8/v8-utils.cpp

@@ -332,45 +332,118 @@ static v8::Handle<v8::Value> JS_Parse (v8::Arguments const& argv) {
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief downloads data from a URL
 ///
-/// @FUN{internal.download(@FA{url}, @FA{method}, @FA{outfile}, @FA{timeout})}
+/// @FUN{internal.download(@FA{url}, @FA{body}, @FA{options}, @FA{outfile})}
 ///
 /// Downloads the data from the URL specified by @FA{url} and saves the
-/// response body to @FA{outfile}.
+/// response body to @FA{outfile}. The following @FA{options} are supported:
+///
+/// - @LIT{method}: the HTTP method to be used. The supported HTTP methods are 
+///   @LIT{DELETE}, @LIT{GET}, @LIT{HEAD}, @LIT{POST}, @LIT{PUT}, @LIT{PATCH}
+///
+/// - @LIT{timeout}: a timeout value for the connection
+///
+/// - @LIT{followRedirects}: whether or not to follow redirects
+///
+/// - @LIT{headers}: an optional array of headers to be sent for the first
+///   (non-redirect) request.
+///
+/// Up to 5 redirects will be followed. Any user-defined headers will only be
+/// sent for the first request. If no timeout is given, a default timeout of
 ////////////////////////////////////////////////////////////////////////////////
 
 static v8::Handle<v8::Value> JS_Download (v8::Arguments const& argv) {
   v8::HandleScope scope;
 
+  const string signature = "download(<url>, <body>, <options>, <outfile>)";
+
   if (argv.Length() < 3) {
-    TRI_V8_EXCEPTION_USAGE(scope, "download(<url>, <method>, <outfile>, <timeout>)");
+    TRI_V8_EXCEPTION_USAGE(scope, signature);
   }
 
   string url = TRI_ObjectToString(argv[0]);
 
+  string body;
+  if (argv[1]->IsString() || argv[1]->IsStringObject()) {
+    body = TRI_ObjectToString(argv[1]);
+  }
+
+  // options
+  // ------------------------------------------------------------------------
+
+  if (! argv[2]->IsObject()) {
+    TRI_V8_EXCEPTION_USAGE(scope, signature);
+  }
+
+  v8::Handle<v8::Array> options = v8::Handle<v8::Array>::Cast(argv[2]);
+  if (options.IsEmpty()) {
+    TRI_V8_EXCEPTION_USAGE(scope, signature);
+  }
+
+  // method
   HttpRequest::HttpRequestType method = HttpRequest::HTTP_REQUEST_GET;
-  const string methodString = TRI_ObjectToString(argv[1]);
+  if (options->Has(TRI_V8_SYMBOL("method"))) {
-
+    string methodString = TRI_ObjectToString(options->Get(TRI_V8_SYMBOL("method")));
-  if (methodString == "head") {
+  
-    method = HttpRequest::HTTP_REQUEST_HEAD;
+    method = HttpRequest::translateMethod(methodString);
-  }
-  else if (methodString == "delete") {
-    method = HttpRequest::HTTP_REQUEST_DELETE;
   }
 
-  const string outfile = TRI_ObjectToString(argv[2]);
+  // headers
+  map<string, string> headerFields;
+  if (options->Has(TRI_V8_SYMBOL("headers"))) {
+    v8::Handle<v8::Object> v8Headers = options->Get(TRI_V8_SYMBOL("headers")).As<v8::Object> ();
+    if (v8Headers->IsObject()) {
+      v8::Handle<v8::Array> props = v8Headers->GetPropertyNames();
 
+      for (uint32_t i = 0; i < props->Length(); i++) {
+        v8::Handle<v8::Value> key = props->Get(v8::Integer::New(i));
+        headerFields[TRI_ObjectToString(key)] = TRI_ObjectToString(v8Headers->Get(key));
+      }
+    }
+  }
+
+  // timeout
   double timeout = 10.0;
-  if (argv.Length() > 3) {
+  if (options->Has(TRI_V8_SYMBOL("timeout"))) {
-    timeout = TRI_ObjectToDouble(argv[3]);
+    if (! options->Get(TRI_V8_SYMBOL("timeout"))->IsNumber()) {
+      TRI_V8_EXCEPTION_MESSAGE(scope, TRI_ERROR_BAD_PARAMETER, "invalid option value for timeout");
+    }
+
+    timeout = TRI_ObjectToDouble(options->Get(TRI_V8_SYMBOL("timeout")));
   }
 
-  if (TRI_ExistsFile(outfile.c_str())) {
+  // follow redirects
-    TRI_V8_EXCEPTION(scope, TRI_ERROR_CANNOT_OVERWRITE_FILE);
+  bool followRedirects = true;
+  if (options->Has(TRI_V8_SYMBOL("followRedirects"))) {
+    followRedirects = TRI_ObjectToBoolean(options->Get(TRI_V8_SYMBOL("followRedirects")));
   }
 
+  if (body.size() > 0 &&
+      (method == HttpRequest::HTTP_REQUEST_GET ||
+       method == HttpRequest::HTTP_REQUEST_HEAD)) {
+    TRI_V8_EXCEPTION_MESSAGE(scope, TRI_ERROR_BAD_PARAMETER, "should not provide a body value for this request method");
+  }
+
+
+  // outfile
+  string outfile;
+  if (argv.Length() == 4) {
+    if (argv[3]->IsString() || argv[3]->IsStringObject()) {
+      outfile = TRI_ObjectToString(argv[3]);
+    }
+
+    if (outfile == "") {
+      TRI_V8_EXCEPTION_MESSAGE(scope, TRI_ERROR_BAD_PARAMETER, "invalid value provided for outfile");
+    }
+
+    if (TRI_ExistsFile(outfile.c_str())) {
+      TRI_V8_EXCEPTION(scope, TRI_ERROR_CANNOT_OVERWRITE_FILE);
+    }
+  }
+
+
   int numRedirects = 0;
 
-  while (numRedirects++ < 5) {
+  while (numRedirects < 5) {
     string endpoint;
     string relative;
 
@@ -413,9 +486,13 @@ static v8::Handle<v8::Value> JS_Download (v8::Arguments const& argv) {
     SimpleHttpClient client(connection, timeout, false);
 
     v8::Handle<v8::Object> result = v8::Object::New();
+    
+    if (numRedirects > 0) {
+      // do not send extra headers now
+      headerFields.clear();
+    }
 
-    // connect to server and get version number
+    // send the actual request
-    map<string, string> headerFields;
     SimpleHttpResult* response = client.request(method, relative, 0, 0, headerFields);
 
     int returnCode;
@@ -434,7 +511,9 @@ static v8::Handle<v8::Value> JS_Download (v8::Arguments const& argv) {
       returnMessage = response->getHttpReturnMessage();
       returnCode = response->getHttpReturnCode();
 
-      if (returnCode == 301 || returnCode == 302) {
+      // follow redirects?
+      if (followRedirects && 
+          (returnCode == 301 || returnCode == 302)) {
         bool found;
         url = response->getHeaderField(string("location"), found);
 
@@ -445,12 +524,14 @@ static v8::Handle<v8::Value> JS_Download (v8::Arguments const& argv) {
           TRI_V8_EXCEPTION_INTERNAL(scope, "caught invalid redirect URL");
         }
 
+        numRedirects++;
         continue;
       }
 
       result->Set(v8::String::New("code"),    v8::Number::New(returnCode));
       result->Set(v8::String::New("message"), v8::String::New(returnMessage.c_str()));
 
+      // process response headers
       const map<string, string> responseHeaders = response->getHeaderFields();
       map<string, string>::const_iterator it;
 
@@ -460,9 +541,17 @@ static v8::Handle<v8::Value> JS_Download (v8::Arguments const& argv) {
       }
       result->Set(v8::String::New("headers"), headers);
 
+
       if (returnCode >= 200 && returnCode <= 299) {
         try {
-          FileUtils::spit(outfile, response->getBody().str());
+          if (outfile.size() > 0) {
+            // save outfile
+            FileUtils::spit(outfile, response->getBody().str());
+          }
+          else {
+            // set "body" attribute in result
+            result->Set(v8::String::New("body"), v8::String::New(response->getBody().str().c_str(), response->getBody().str().length()));
+          }
         }
         catch (...) {