////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Max Neunhoeffer
////////////////////////////////////////////////////////////////////////////////

#include "ClusterMethods.h"
#include "Basics/conversions.h"
#include "Basics/json.h"
#include "Basics/json-utilities.h"
#include "Basics/StringUtils.h"
#include "Basics/tri-strings.h"
#include "Basics/VelocyPackHelper.h"
#include "Cluster/ClusterComm.h"
#include "Cluster/ClusterInfo.h"
#include "Indexes/Index.h"
#include "VocBase/Traverser.h"
#include "VocBase/server.h"

#include <velocypack/Buffer.h>
#include <velocypack/Helpers.h>
#include <velocypack/Iterator.h>
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>

using namespace arangodb::basics;
using namespace arangodb::rest;

namespace arangodb {

static int handleGeneralCommErrors(ClusterCommResult const* res) {
  if (res->status == CL_COMM_TIMEOUT) {
    // No reply, we give up:
    return TRI_ERROR_CLUSTER_TIMEOUT;
  } else if (res->status == CL_COMM_ERROR) {
    // This could be a broken connection or an Http error:
    if (res->result == nullptr || !res->result->isComplete()) {
      // there is not result
      return TRI_ERROR_CLUSTER_CONNECTION_LOST;
    }
  } else if (res->status == CL_COMM_BACKEND_UNAVAILABLE) {
    return TRI_ERROR_CLUSTER_BACKEND_UNAVAILABLE;
  }

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief extracts a numeric value from an hierarchical VelocyPack
////////////////////////////////////////////////////////////////////////////////

template <typename T>
static T ExtractFigure(VPackSlice const& slice, char const* group,
                       char const* name) {
  TRI_ASSERT(slice.isObject());
  VPackSlice g = slice.get(group);

  if (!g.isObject()) {
    return static_cast<T>(0);
  }
  return arangodb::basics::VelocyPackHelper::getNumericValue<T>(g, name, 0);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief extracts answer from response into a VPackBuilder.
///        If there was an error extracting the answer the builder will be
///        empty.
///        No Error can be thrown.
////////////////////////////////////////////////////////////////////////////////

static std::shared_ptr<VPackBuilder> ExtractAnswer(
    ClusterCommResult const& res) {
  try {
    return VPackParser::fromJson(res.answer->body());
  } catch (...) {
    // Return an empty Builder
    return std::make_shared<VPackBuilder>();
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief merge the baby-object results.
///        The shard map contains the ordering of elements, the vector in this
///        Map is expected to be sorted from front to back.
///        The second map contains the answers for each shard.
///        The builder in the third parameter will be cleared and will contain
///        the resulting array. It is guaranteed that the resulting array indexes
///        are equal to the original request ordering before it was destructured
///        for babies.
////////////////////////////////////////////////////////////////////////////////

static void mergeResults(
    std::vector<std::pair<ShardID, VPackValueLength>> const& reverseMapping,
    std::unordered_map<ShardID, std::shared_ptr<VPackBuilder>> const& resultMap,
    std::shared_ptr<VPackBuilder>& resultBody) {
  resultBody->clear();
  resultBody->openArray();
  for (auto const& pair : reverseMapping) {
    VPackSlice arr = resultMap.find(pair.first)->second->slice();
    resultBody->add(arr.at(pair.second));
  }
  resultBody->close();
}

////////////////////////////////////////////////////////////////////////////////
/// @brief merge headers of a DB server response into the current response
////////////////////////////////////////////////////////////////////////////////

void mergeResponseHeaders(GeneralResponse* response,
                          std::map<std::string, std::string> const& headers) {
  std::map<std::string, std::string>::const_iterator it = headers.begin();

  while (it != headers.end()) {
    // skip first header line (which is the HTTP response code)
    std::string const& key = (*it).first;

    // the following headers are ignored
    if (key != "http/1.1" && key != "connection" && key != "content-length" &&
        key != "server") {
      response->setHeader(key, (*it).second);
    }
    ++it;
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief creates a copy of all HTTP headers to forward
////////////////////////////////////////////////////////////////////////////////

std::map<std::string, std::string> getForwardableRequestHeaders(
    arangodb::HttpRequest* request) {
  std::map<std::string, std::string> const& headers = request->headers();
  std::map<std::string, std::string>::const_iterator it = headers.begin();

  std::map<std::string, std::string> result;

  while (it != headers.end()) {
    std::string const& key = (*it).first;

    // ignore the following headers
    if (key != "x-arango-async" && key != "authorization" &&
        key != "content-length" && key != "connection" && key != "expect" &&
        key != "host" && key != "origin" &&
        key.substr(0, 14) != "access-control") {
      result.emplace(key, (*it).second);
    }
    ++it;
  }

  result["content-length"] = StringUtils::itoa(request->contentLength());

  return result;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief check if a list of attributes have the same values in two vpack
/// documents
////////////////////////////////////////////////////////////////////////////////

bool shardKeysChanged(std::string const& dbname, std::string const& collname,
                      VPackSlice const& oldValue, VPackSlice const& newValue,
                      bool isPatch) {
  if (!oldValue.isObject() || !newValue.isObject()) {
    // expecting two objects. everything else is an error
    return true;
  }

  ClusterInfo* ci = ClusterInfo::instance();
  std::shared_ptr<CollectionInfo> c = ci->getCollection(dbname, collname);
  std::vector<std::string> const& shardKeys = c->shardKeys();

  for (size_t i = 0; i < shardKeys.size(); ++i) {
    if (shardKeys[i] == TRI_VOC_ATTRIBUTE_KEY) {
      continue;
    }

    VPackSlice n = newValue.get(shardKeys[i]);

    if (n.isNone() && isPatch) {
      // attribute not set in patch document. this means no update
      continue;
    }
   
    // a temporary buffer to hold a null value 
    char buffer[1];
    VPackSlice nullValue = arangodb::velocypack::buildNullValue(&buffer[0], sizeof(buffer));

    VPackSlice o = oldValue.get(shardKeys[i]);

    if (o.isNone()) {
      // if attribute is undefined, use "null" instead
      o = nullValue;
    }

    if (n.isNone()) {
      // if attribute is undefined, use "null" instead
      n = nullValue;
    }

    if (arangodb::basics::VelocyPackHelper::compare(n, o, false) != 0) {
      return true;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief returns users
////////////////////////////////////////////////////////////////////////////////

int usersOnCoordinator(std::string const& dbname, VPackBuilder& result,
                       double timeout) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, TRI_COL_NAME_USERS);

  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  // If we get here, the sharding attributes are not only _key, therefore
  // we have to contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();

  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());

    // set collection name (shard id)
    auto body = std::make_shared<std::string>();
    body->append("{\"collection\":\"");
    body->append(p.first);
    body->append("\"}");

    cc->asyncRequest(
        "", coordTransactionID, "shard:" + p.first,
        arangodb::GeneralRequest::RequestType::PUT,
        "/_db/" + StringUtils::urlEncode(dbname) + "/_api/simple/all", body,
        headers, nullptr, 10.0);
  }

  try {
    // Now listen to the results:
    int count;
    int nrok = 0;
    for (count = (int)shards->size(); count > 0; count--) {
      auto res = cc->wait("", coordTransactionID, 0, "", timeout);
      if (res.status == CL_COMM_RECEIVED) {
        if (res.answer_code == arangodb::GeneralResponse::ResponseCode::OK ||
            res.answer_code == arangodb::GeneralResponse::ResponseCode::CREATED) {
          std::shared_ptr<VPackBuilder> answerBuilder = ExtractAnswer(res);
          VPackSlice answer = answerBuilder->slice();

          if (answer.isObject()) {
            VPackSlice r = answer.get("result");
            if (r.isArray()) {
              for (auto const& p : VPackArrayIterator(r)) {
                if (p.isObject()) {
                  result.add(p);
                }
              }
            }
            nrok++;
          }
        }
      }
    }
    if (nrok != (int)shards->size()) {
      return TRI_ERROR_INTERNAL;
    }
  } catch (...) {
    return TRI_ERROR_OUT_OF_MEMORY;
  }
  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

////////////////////////////////////////////////////////////////////////////////
/// @brief returns revision for a sharded collection
////////////////////////////////////////////////////////////////////////////////

int revisionOnCoordinator(std::string const& dbname,
                          std::string const& collname, TRI_voc_rid_t& rid) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);

  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  rid = 0;

  // If we get here, the sharding attributes are not only _key, therefore
  // we have to contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();

  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    cc->asyncRequest(
        "", coordTransactionID, "shard:" + p.first,
        arangodb::GeneralRequest::RequestType::GET,
        "/_db/" + StringUtils::urlEncode(dbname) + "/_api/collection/" +
            StringUtils::urlEncode(p.first) + "/revision",
        std::shared_ptr<std::string const>(), headers, nullptr, 300.0);
  }

  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code == arangodb::GeneralResponse::ResponseCode::OK) {
        std::shared_ptr<VPackBuilder> answerBuilder = ExtractAnswer(res);
        VPackSlice answer = answerBuilder->slice();

        if (answer.isObject()) {
          VPackSlice r = answer.get("revision");

          if (r.isString()) {
            TRI_voc_rid_t cmp = StringUtils::uint64(r.copyString());

            if (cmp > rid) {
              // get the maximum value
              rid = cmp;
            }
          }
          nrok++;
        }
      }
    }
  }

  if (nrok != (int)shards->size()) {
    return TRI_ERROR_INTERNAL;
  }

  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

////////////////////////////////////////////////////////////////////////////////
/// @brief returns figures for a sharded collection
////////////////////////////////////////////////////////////////////////////////

int figuresOnCoordinator(std::string const& dbname, std::string const& collname,
                         TRI_doc_collection_info_t*& result) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);

  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  // prefill with 0s
  result = (TRI_doc_collection_info_t*)TRI_Allocate(
      TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_doc_collection_info_t), true);

  if (result == nullptr) {
    return TRI_ERROR_OUT_OF_MEMORY;
  }

  // If we get here, the sharding attributes are not only _key, therefore
  // we have to contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();

  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    cc->asyncRequest(
        "", coordTransactionID, "shard:" + p.first,
        arangodb::GeneralRequest::RequestType::GET,
        "/_db/" + StringUtils::urlEncode(dbname) + "/_api/collection/" +
            StringUtils::urlEncode(p.first) + "/figures",
        std::shared_ptr<std::string const>(), headers, nullptr, 300.0);
  }

  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code == arangodb::GeneralResponse::ResponseCode::OK) {
        std::shared_ptr<VPackBuilder> answerBuilder = ExtractAnswer(res);
        VPackSlice answer = answerBuilder->slice();

        if (answer.isObject()) {
          VPackSlice figures = answer.get("figures");
          if (figures.isObject()) {
            // add to the total
            result->_numberAlive +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "alive", "count");
            result->_numberDead +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "dead", "count");
            result->_numberDeletions +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "dead", "deletion");
            result->_numberIndexes +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "indexes", "count");

            result->_sizeAlive +=
                ExtractFigure<int64_t>(figures, "alive", "size");
            result->_sizeDead +=
                ExtractFigure<int64_t>(figures, "dead", "size");
            result->_sizeIndexes +=
                ExtractFigure<int64_t>(figures, "indexes", "size");

            result->_numberDatafiles +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "datafiles", "count");
            result->_numberJournalfiles +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "journals", "count");
            result->_numberCompactorfiles +=
                ExtractFigure<TRI_voc_ssize_t>(figures, "compactors", "count");

            result->_datafileSize +=
                ExtractFigure<int64_t>(figures, "datafiles", "fileSize");
            result->_journalfileSize +=
                ExtractFigure<int64_t>(figures, "journals", "fileSize");
            result->_compactorfileSize +=
                ExtractFigure<int64_t>(figures, "compactors", "fileSize");

            result->_numberDocumentDitches +=
                arangodb::basics::VelocyPackHelper::getNumericValue<uint64_t>(
                    figures, "documentReferences", 0);
          }
          nrok++;
        }
      }
    }
  }

  if (nrok != (int)shards->size()) {
    TRI_Free(TRI_UNKNOWN_MEM_ZONE, result);
    result = 0;
    return TRI_ERROR_INTERNAL;
  }

  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

////////////////////////////////////////////////////////////////////////////////
/// @brief counts number of documents in a coordinator
////////////////////////////////////////////////////////////////////////////////

int countOnCoordinator(std::string const& dbname, std::string const& collname,
                       uint64_t& result) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  result = 0;

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);

  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    cc->asyncRequest(
        "", coordTransactionID, "shard:" + p.first,
        arangodb::GeneralRequest::RequestType::GET,
        "/_db/" + StringUtils::urlEncode(dbname) + "/_api/collection/" +
            StringUtils::urlEncode(p.first) + "/count",
        std::shared_ptr<std::string>(nullptr), headers, nullptr, 300.0);
  }
  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code == arangodb::GeneralResponse::ResponseCode::OK) {
        std::shared_ptr<VPackBuilder> answerBuilder = ExtractAnswer(res);
        VPackSlice answer = answerBuilder->slice();

        if (answer.isObject()) {
          // add to the total
          result +=
              arangodb::basics::VelocyPackHelper::getNumericValue<uint64_t>(
                  answer, "count", 0);
          nrok++;
        }
      }
    }
  }

  if (nrok != (int)shards->size()) {
    return TRI_ERROR_INTERNAL;
  }

  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

////////////////////////////////////////////////////////////////////////////////
/// @brief creates one or many documents in a coordinator
///
/// In case of many documents (slice is a VPackArray) it will send to each
/// shard all the relevant documents for this shard only.
/// If one of them fails, this error is reported.
/// There is NO guarantee for the stored documents of all other shards, they may
/// be stored or not. All answers of these shards are dropped.
/// If we return with NO_ERROR it is guaranteed that all shards reported success
/// for their documents.
////////////////////////////////////////////////////////////////////////////////

int createDocumentOnCoordinator(
    std::string const& dbname, std::string const& collname,
    arangodb::OperationOptions const& options, VPackSlice const& slice,
    std::map<std::string, std::string> const& headers,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::unordered_map<int, size_t>& errorCounter,
    std::shared_ptr<VPackBuilder>& resultBody) {
   // Set a few variables needed for our work:
   ClusterInfo* ci = ClusterInfo::instance();
   ClusterComm* cc = ClusterComm::instance();
 
   // First determine the collection ID from the name:
   std::shared_ptr<CollectionInfo> collinfo =
       ci->getCollection(dbname, collname);
 
   if (collinfo->empty()) {
     return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
   }
 
   std::string const collid = StringUtils::itoa(collinfo->id());
   std::unordered_map<ShardID, std::vector<VPackValueLength>> shardMap;
  std::vector<std::pair<ShardID, VPackValueLength>> reverseMapping;
   bool useMultiple = slice.isArray();

   auto workOnOneNode = [&shardMap, &ci, &collid, &collinfo, &reverseMapping](
       VPackSlice const node, VPackValueLength const index) -> int {
     // Sort out the _key attribute:
     // The user is allowed to specify _key, provided that _key is the one
     // and only sharding attribute, because in this case we can delegate
     // the responsibility to make _key attributes unique to the responsible
     // shard. Otherwise, we ensure uniqueness here and now by taking a
     // cluster-wide unique number. Note that we only know the sharding
     // attributes a bit further down the line when we have determined
     // the responsible shard.
 
     bool userSpecifiedKey = false;
     std::string _key;
     VPackSlice keySlice = node.get(TRI_VOC_ATTRIBUTE_KEY);
     if (keySlice.isNone()) {
       // The user did not specify a key, let's create one:
       uint64_t uid = ci->uniqid();
       _key = arangodb::basics::StringUtils::itoa(uid);
     } else {
       userSpecifiedKey = true;
     }
 
     // Now find the responsible shard:
     bool usesDefaultShardingAttributes;
     ShardID shardID;
     int error = ci->getResponsibleShard(collid, node, true, shardID,
                                         usesDefaultShardingAttributes);
     if (error == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND) {
       return TRI_ERROR_CLUSTER_SHARD_GONE;
     }
 
     // Now perform the above mentioned check:
     if (userSpecifiedKey && (!usesDefaultShardingAttributes || !collinfo->allowUserKeys())) {
       return TRI_ERROR_CLUSTER_MUST_NOT_SPECIFY_KEY;
     }
     // We found the responsible shard. Add it to the list.
     auto it = shardMap.find(shardID);
     if (it == shardMap.end()) {
       std::vector<VPackValueLength> counter({index});
       shardMap.emplace(shardID, counter);
     } else {
       it->second.emplace_back(index);
     }
     reverseMapping.emplace_back(shardID, index);
     return TRI_ERROR_NO_ERROR;
   };

   if (useMultiple) {
     VPackValueLength length = slice.length();
     for (VPackValueLength idx = 0; idx < length; ++idx) {
       workOnOneNode(slice.at(idx), idx);
     }
   } else {
     workOnOneNode(slice, 0);
   }
 
   std::string const baseUrl =
       "/_db/" + StringUtils::urlEncode(dbname) + "/_api/document?collection=";
 
   std::string const optsUrlPart =
       std::string("&waitForSync=") + (options.waitForSync ? "true" : "false") +
       "&returnNew=" + (options.returnNew ? "true" : "false") + "&returnOld=" +
       (options.returnOld ? "true" : "false");
 
   VPackBuilder reqBuilder;
   CoordTransactionID coordTransactionID = TRI_NewTickServer();
 
   auto body = std::make_shared<std::string>();
   for (auto const& it : shardMap) {
     if (!useMultiple) {
       body->assign(slice.toJson());
     } else {
       reqBuilder.clear();
       reqBuilder.openArray();
       for (auto idx : it.second) {
         reqBuilder.add(slice.at(idx));
       }
       reqBuilder.close();
       body->assign(reqBuilder.slice().toJson());
     }
     auto headersCopy = std::make_unique<std::map<std::string, std::string>>(headers);
     cc->asyncRequest("", coordTransactionID, "shard:" + it.first,
                      arangodb::GeneralRequest::RequestType::POST,
                      baseUrl + StringUtils::urlEncode(it.first) + optsUrlPart,
                      body, headersCopy, nullptr,
                      60.0);
   }
 
   // Now listen to the results:
   if (!useMultiple) {
     auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
 
     int commError = handleGeneralCommErrors(&res);
     if (commError != TRI_ERROR_NO_ERROR) {
       return commError;
     }
 
     responseCode = res.answer_code;
     TRI_ASSERT(res.answer != nullptr);
     auto parsedResult = res.answer->toVelocyPack(&VPackOptions::Defaults);
     resultBody.swap(parsedResult);
     return TRI_ERROR_NO_ERROR;
   }
 
   size_t count;
   std::unordered_map<ShardID, std::shared_ptr<VPackBuilder>> resultMap;
 
   for (count = shardMap.size(); count > 0; count--) {
     auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
     if (res.status == CL_COMM_RECEIVED) {
       // In babies case we only have good response codes, even in Error cases
       TRI_ASSERT(res.answer_code ==
                  (options.waitForSync
                       ? GeneralResponse::ResponseCode::CREATED
                       : GeneralResponse::ResponseCode::ACCEPTED));
 
       int commError = handleGeneralCommErrors(&res);
       if (commError != TRI_ERROR_NO_ERROR) {
         auto tmpBuilder = std::make_shared<VPackBuilder>();
         auto weSend = shardMap.find(res.shardID);
         TRI_ASSERT(weSend != shardMap.end()); // We send sth there earlier.
         size_t count = weSend->second.size();
         for (size_t i = 0; i < count; ++i) {
           tmpBuilder->openObject();
           tmpBuilder->add("error", VPackValue(true));
           tmpBuilder->add("errorNum", VPackValue(commError));
           tmpBuilder->close();
         }
         resultMap.emplace(res.shardID, tmpBuilder);
       } else {
         TRI_ASSERT(res.answer != nullptr);
         resultMap.emplace(res.shardID,
                           res.answer->toVelocyPack(&VPackOptions::Defaults));
         auto resultHeaders = res.answer->headers();
         auto codes = resultHeaders.find("X-Arango-Error-Codes");
         if (codes != resultHeaders.end()) {
           auto parsedCodes = VPackParser::fromJson(codes->second);
           VPackSlice codesSlice = parsedCodes->slice();
           TRI_ASSERT(codesSlice.isObject());
           for (auto const& code : VPackObjectIterator(codesSlice)) {
             VPackValueLength codeLength;
             char const* codeString = code.key.getString(codeLength);
             int codeNr = static_cast<int>(
                 arangodb::basics::StringUtils::int64(codeString, codeLength));
             errorCounter[codeNr] += code.value.getNumericValue<size_t>();
           }
         }
       }
     }
   }
   responseCode = (options.waitForSync ? GeneralResponse::ResponseCode::CREATED
                                       : GeneralResponse::ResponseCode::ACCEPTED);
   mergeResults(reverseMapping, resultMap, resultBody);
   return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                               // the DBserver could have reported an error.
}


////////////////////////////////////////////////////////////////////////////////
/// @brief creates a document in a coordinator
////////////////////////////////////////////////////////////////////////////////

int createDocumentOnCoordinator(
    std::string const& dbname, std::string const& collname, arangodb::OperationOptions const& options,
    std::unique_ptr<TRI_json_t>& json,
    std::map<std::string, std::string> const& headers,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::map<std::string, std::string>& resultHeaders,
    std::string& resultBody) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);

  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  std::string const collid = StringUtils::itoa(collinfo->id());

  // Sort out the _key attribute:
  // The user is allowed to specify _key, provided that _key is the one
  // and only sharding attribute, because in this case we can delegate
  // the responsibility to make _key attributes unique to the responsible
  // shard. Otherwise, we ensure uniqueness here and now by taking a
  // cluster-wide unique number. Note that we only know the sharding
  // attributes a bit further down the line when we have determined
  // the responsible shard.
  TRI_json_t* subjson = TRI_LookupObjectJson(json.get(), TRI_VOC_ATTRIBUTE_KEY);
  bool userSpecifiedKey = false;
  std::string _key;
  if (subjson == nullptr) {
    // The user did not specify a key, let's create one:
    uint64_t uid = ci->uniqid();
    _key = arangodb::basics::StringUtils::itoa(uid);
    TRI_Insert3ObjectJson(TRI_UNKNOWN_MEM_ZONE, json.get(),
                          TRI_VOC_ATTRIBUTE_KEY,
                          TRI_CreateStringReferenceJson(
                              TRI_UNKNOWN_MEM_ZONE, _key.c_str(), _key.size()));
  } else {
    userSpecifiedKey = true;
  }

  // Now find the responsible shard:
  bool usesDefaultShardingAttributes;
  ShardID shardID;
  int error = ci->getResponsibleShard(collid, json.get(), true, shardID,
                                      usesDefaultShardingAttributes);
  if (error == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND) {
    return TRI_ERROR_CLUSTER_SHARD_GONE;
  }

  // Now perform the above mentioned check:
  if (userSpecifiedKey && !usesDefaultShardingAttributes) {
    return TRI_ERROR_CLUSTER_MUST_NOT_SPECIFY_KEY;
  }

  if (userSpecifiedKey && !collinfo->allowUserKeys()) {
    return TRI_ERROR_CLUSTER_MUST_NOT_SPECIFY_KEY;
  }

  std::string const body = JsonHelper::toString(json.get());

  // Send a synchronous request to that shard using ClusterComm:
  auto res = cc->syncRequest(
      "", TRI_NewTickServer(), "shard:" + shardID,
      arangodb::GeneralRequest::RequestType::POST,
      "/_db/" + StringUtils::urlEncode(dbname) + "/_api/document?collection=" +
          StringUtils::urlEncode(shardID) + "&waitForSync=" +
          (options.waitForSync ? "true" : "false") + "&returnNew=" +
          (options.returnNew ? "true" : "false") + "&returnOld=" +
            (options.returnOld ? "true" : "false"),
      body, headers, 60.0);

  int commError = handleGeneralCommErrors(res.get());
  if (commError != TRI_ERROR_NO_ERROR) {
    return commError;
  }
  responseCode = static_cast<arangodb::GeneralResponse::ResponseCode>(
      res->result->getHttpReturnCode());
  resultHeaders = res->result->getHeaderFields();
  resultBody.assign(res->result->getBody().c_str(),
                    res->result->getBody().length());
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief deletes a document in a coordinator
////////////////////////////////////////////////////////////////////////////////

int deleteDocumentOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& key, TRI_voc_rid_t const rev,
    arangodb::OperationOptions const& options,
    std::unique_ptr<std::map<std::string, std::string>>& headers,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::map<std::string, std::string>& resultHeaders,
    std::string& resultBody) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);
  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }
  std::string collid = StringUtils::itoa(collinfo->id());

  // If _key is the one and only sharding attribute, we can do this quickly,
  // because we can easily determine which shard is responsible for the
  // document. Otherwise we have to contact all shards and ask them to
  // delete the document. All but one will not know it.
  // Now find the responsible shard:
  TRI_json_t* json = TRI_CreateObjectJson(TRI_UNKNOWN_MEM_ZONE);
  if (json == nullptr) {
    return TRI_ERROR_OUT_OF_MEMORY;
  }
  TRI_Insert3ObjectJson(TRI_UNKNOWN_MEM_ZONE, json, TRI_VOC_ATTRIBUTE_KEY,
                        TRI_CreateStringReferenceJson(TRI_UNKNOWN_MEM_ZONE,
                                                      key.c_str(), key.size()));
  bool usesDefaultShardingAttributes;
  ShardID shardID;
  int error = ci->getResponsibleShard(collid, json, true, shardID,
                                      usesDefaultShardingAttributes);
  TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);

  // Some stuff to prepare cluster-intern requests:
  if (rev != 0) {
    headers->emplace("if-match", StringUtils::itoa(rev));
  }

  if (usesDefaultShardingAttributes) {
    // OK, this is the fast method, we only have to ask one shard:
    if (error == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND) {
      return TRI_ERROR_CLUSTER_SHARD_GONE;
    }

    // Send a synchronous request to that shard using ClusterComm:
    auto res = cc->syncRequest(
        "", TRI_NewTickServer(), "shard:" + shardID,
        arangodb::GeneralRequest::RequestType::DELETE_REQ,
        "/_db/" + dbname + "/_api/document/" + StringUtils::urlEncode(shardID) +
            "/" + StringUtils::urlEncode(key) + "?waitForSync=" +
            (options.waitForSync ? "true" : "false") + "&returnOld=" +
            (options.returnOld ? "true" : "false"),
        "", *headers, 60.0);

    int error = handleGeneralCommErrors(res.get());
    if (error != TRI_ERROR_NO_ERROR) {
      return error;
    }
    responseCode = static_cast<arangodb::GeneralResponse::ResponseCode>(
        res->result->getHttpReturnCode());
    resultHeaders = res->result->getHeaderFields();
    resultBody.assign(res->result->getBody().c_str(),
                      res->result->getBody().length());
    return TRI_ERROR_NO_ERROR;
  }

  // If we get here, the sharding attributes are not only _key, therefore
  // we have to contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headersCopy(
        new std::map<std::string, std::string>(*headers));
    cc->asyncRequest("", coordTransactionID, "shard:" + p.first,
                     arangodb::GeneralRequest::RequestType::DELETE_REQ,
                     "/_db/" + StringUtils::urlEncode(dbname) +
                         "/_api/document/" + StringUtils::urlEncode(p.first) +
                         "/" + StringUtils::urlEncode(key) + "?waitForSync=" +
                         (options.waitForSync ? "true" : "false") +
                         "&returnOld=" + (options.returnOld ? "true" : "false"),
                     std::shared_ptr<std::string const>(), headersCopy, nullptr,
                     60.0);
  }
  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code != arangodb::GeneralResponse::ResponseCode::NOT_FOUND ||
          (nrok == 0 && count == 1)) {
        nrok++;
        responseCode = res.answer_code;
        resultHeaders = res.answer->headers();
        resultHeaders["content-length"] =
            StringUtils::itoa(res.answer->contentLength());
        resultBody = res.answer->body();
      }
    }
  }

  // Note that nrok is always at least 1!
  if (nrok > 1) {
    return TRI_ERROR_CLUSTER_GOT_CONTRADICTING_ANSWERS;
  }
  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

////////////////////////////////////////////////////////////////////////////////
/// @brief truncate a cluster collection on a coordinator
////////////////////////////////////////////////////////////////////////////////

int truncateCollectionOnCoordinator(std::string const& dbname,
                                    std::string const& collname) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);

  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  // Some stuff to prepare cluster-intern requests:
  // We have to contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    cc->asyncRequest("", coordTransactionID, "shard:" + p.first,
                     arangodb::GeneralRequest::RequestType::PUT,
                     "/_db/" + StringUtils::urlEncode(dbname) +
                         "/_api/collection/" + p.first + "/truncate",
                     std::shared_ptr<std::string>(nullptr), headers, nullptr,
                     60.0);
  }
  // Now listen to the results:
  unsigned int count;
  unsigned int nrok = 0;
  for (count = (unsigned int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code == arangodb::GeneralResponse::ResponseCode::OK) {
        nrok++;
      }
    }
  }

  // Note that nrok is always at least 1!
  if (nrok < shards->size()) {
    return TRI_ERROR_CLUSTER_COULD_NOT_TRUNCATE_COLLECTION;
  }
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get a document in a coordinator
////////////////////////////////////////////////////////////////////////////////

int getDocumentOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& key, TRI_voc_rid_t const rev,
    std::unique_ptr<std::map<std::string, std::string>>& headers,
    bool generateDocument,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::map<std::string, std::string>& resultHeaders,
    std::string& resultBody) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);
  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }
  std::string collid = StringUtils::itoa(collinfo->id());

  // If _key is the one and only sharding attribute, we can do this quickly,
  // because we can easily determine which shard is responsible for the
  // document. Otherwise we have to contact all shards and ask them to
  // delete the document. All but one will not know it.
  // Now find the responsible shard:
  TRI_json_t* json = TRI_CreateObjectJson(TRI_UNKNOWN_MEM_ZONE);
  if (json == nullptr) {
    return TRI_ERROR_OUT_OF_MEMORY;
  }
  TRI_Insert3ObjectJson(TRI_UNKNOWN_MEM_ZONE, json, "_key",
                        TRI_CreateStringReferenceJson(TRI_UNKNOWN_MEM_ZONE,
                                                      key.c_str(), key.size()));
  bool usesDefaultShardingAttributes;
  ShardID shardID;
  int error = ci->getResponsibleShard(collid, json, true, shardID,
                                      usesDefaultShardingAttributes);
  TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);

  // Some stuff to prepare cluster-intern requests:
  std::string revstr;
  if (rev != 0) {
    headers->emplace("if-match", StringUtils::itoa(rev));
  }
  arangodb::GeneralRequest::RequestType reqType;
  if (generateDocument) {
    reqType = arangodb::GeneralRequest::RequestType::GET;
  } else {
    reqType = arangodb::GeneralRequest::RequestType::HEAD;
  }

  if (usesDefaultShardingAttributes) {
    // OK, this is the fast method, we only have to ask one shard:
    if (error == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND) {
      return TRI_ERROR_CLUSTER_SHARD_GONE;
    }

    // Send a synchronous request to that shard using ClusterComm:
    auto res = cc->syncRequest(
        "", TRI_NewTickServer(), "shard:" + shardID, reqType,
        "/_db/" + dbname + "/_api/document/" + StringUtils::urlEncode(shardID) +
            "/" + StringUtils::urlEncode(key) + revstr,
        "", *headers, 60.0);

    int error = handleGeneralCommErrors(res.get());
    if (error != TRI_ERROR_NO_ERROR) {
      return error;
    }
    responseCode = static_cast<arangodb::GeneralResponse::ResponseCode>(
        res->result->getHttpReturnCode());
    resultHeaders = res->result->getHeaderFields();
    resultBody.assign(res->result->getBody().c_str(),
                      res->result->getBody().length());
    return TRI_ERROR_NO_ERROR;
  }

  // If we get here, the sharding attributes are not only _key, therefore
  // we have to contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headersCopy(
        new std::map<std::string, std::string>(*headers));
    cc->asyncRequest("", coordTransactionID, "shard:" + p.first, reqType,
                     "/_db/" + StringUtils::urlEncode(dbname) +
                         "/_api/document/" + StringUtils::urlEncode(p.first) +
                         "/" + StringUtils::urlEncode(key) + revstr,
                     std::shared_ptr<std::string const>(), headersCopy, nullptr,
                     60.0);
  }
  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code != arangodb::GeneralResponse::ResponseCode::NOT_FOUND ||
          (nrok == 0 && count == 1)) {
        nrok++;
        responseCode = res.answer_code;
        resultHeaders = res.answer->headers();
        resultHeaders["content-length"] =
            StringUtils::itoa(res.answer->contentLength());
        resultBody = res.answer->body();
      }
    }
  }

  // Note that nrok is always at least 1!
  if (nrok > 1) {
    return TRI_ERROR_CLUSTER_GOT_CONTRADICTING_ANSWERS;
  }
  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

static void insertIntoShardMap(
    ClusterInfo* ci, std::string const& dbname, std::string const& documentId,
    std::unordered_map<ShardID, std::vector<std::string>>& shardMap) {
  std::vector<std::string> splitId =
      arangodb::basics::StringUtils::split(documentId, '/');
  TRI_ASSERT(splitId.size() == 2);

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, splitId[0]);
  if (collinfo->empty()) {
    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND,
                                   "Collection not found: " + splitId[0]);
  }
  std::string collid = StringUtils::itoa(collinfo->id());
  if (collinfo->usesDefaultShardKeys()) {
    // We only need add one resp. shard
    arangodb::basics::Json partial(arangodb::basics::Json::Object, 1);
    partial.set("_key", arangodb::basics::Json(splitId[1]));
    bool usesDefaultShardingAttributes;
    ShardID shardID;

    int error = ci->getResponsibleShard(collid, partial.json(), true, shardID,
                                        usesDefaultShardingAttributes);
    if (error != TRI_ERROR_NO_ERROR) {
      THROW_ARANGO_EXCEPTION(error);
    }
    TRI_ASSERT(usesDefaultShardingAttributes);  // If this is false the if
                                                // condition should be false in
                                                // the first place
    auto it = shardMap.find(shardID);
    if (it == shardMap.end()) {
      shardMap.emplace(shardID, std::vector<std::string>({splitId[1]}));
    } else {
      it->second.push_back(splitId[1]);
    }
  } else {
    // Sorry we do not know the responsible shard yet
    // Ask all of them
    auto shardList = ci->getShardList(collid);
    for (auto const& shard : *shardList) {
      auto it = shardMap.find(shard);
      if (it == shardMap.end()) {
        shardMap.emplace(shard, std::vector<std::string>({splitId[1]}));
      } else {
        it->second.push_back(splitId[1]);
      }
    }
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get a list of filtered documents in a coordinator
///        All found documents will be inserted into result.
///        After execution documentIds will contain all id's of documents
///        that could not be found.
////////////////////////////////////////////////////////////////////////////////

int getFilteredDocumentsOnCoordinator(
    std::string const& dbname,
    std::vector<traverser::TraverserExpression*> const& expressions,
    std::unique_ptr<std::map<std::string, std::string>>& headers,
    std::unordered_set<std::string>& documentIds,
    std::unordered_map<std::string, std::shared_ptr<VPackBuffer<uint8_t>>>& result) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  std::unordered_map<ShardID, std::vector<std::string>> shardRequestMap;
  for (auto const& doc : documentIds) {
    insertIntoShardMap(ci, dbname, doc, shardRequestMap);
  }

  // Now start the request.
  // We do not have to care for shard attributes esp. shard by key.
  // If it is by key the key was only added to one key list, if not
  // it is contained multiple times.
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& shard : shardRequestMap) {
    std::unique_ptr<std::map<std::string, std::string>> headersCopy(
        new std::map<std::string, std::string>(*headers));
    arangodb::basics::Json reqBody(arangodb::basics::Json::Object, 2);
    reqBody("collection", arangodb::basics::Json(static_cast<std::string>(
                              shard.first)));  // ShardID is a string
    arangodb::basics::Json keyList(arangodb::basics::Json::Array,
                                   shard.second.size());
    for (auto const& key : shard.second) {
      keyList.add(arangodb::basics::Json(key));
    }
    reqBody("keys", keyList.steal());
    if (!expressions.empty()) {
      arangodb::basics::Json filter(Json::Array, expressions.size());
      for (auto const& e : expressions) {
        arangodb::basics::Json tmp(Json::Object);
        e->toJson(tmp, TRI_UNKNOWN_MEM_ZONE);
        filter.add(tmp.steal());
      }
      reqBody("filter", filter);
    }
    auto bodyString = std::make_shared<std::string>(reqBody.toString());

    cc->asyncRequest("", coordTransactionID, "shard:" + shard.first,
                     arangodb::GeneralRequest::RequestType::PUT,
                     "/_db/" + StringUtils::urlEncode(dbname) +
                         "/_api/simple/lookup-by-keys",
                     bodyString, headersCopy, nullptr, 60.0);
  }
  // All requests send, now collect results.
  for (size_t i = 0; i < shardRequestMap.size(); ++i) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      std::unique_ptr<TRI_json_t> resultBody(
          arangodb::basics::JsonHelper::fromString(res.answer->body()));

      if (!TRI_IsObjectJson(resultBody.get())) {
        THROW_ARANGO_EXCEPTION_MESSAGE(
            TRI_ERROR_INTERNAL, "Received an invalid result in cluster.");
      }
      bool isError = arangodb::basics::JsonHelper::checkAndGetBooleanValue(
          resultBody.get(), "error");
      if (isError) {
        int errorNum = arangodb::basics::JsonHelper::getNumericValue<int>(
            resultBody.get(), "errorNum", TRI_ERROR_INTERNAL);
        std::string message = arangodb::basics::JsonHelper::getStringValue(
            resultBody.get(), "errorMessage", "");
        THROW_ARANGO_EXCEPTION_MESSAGE(errorNum, message);
      }
      TRI_json_t* documents =
          TRI_LookupObjectJson(resultBody.get(), "documents");
      if (!TRI_IsArrayJson(documents)) {
        THROW_ARANGO_EXCEPTION_MESSAGE(
            TRI_ERROR_INTERNAL, "Received an invalid result in cluster.");
      }
      size_t resCount = TRI_LengthArrayJson(documents);
      for (size_t k = 0; k < resCount; ++k) {
        try {
          TRI_json_t const* element = TRI_LookupArrayJson(documents, k);
          std::string id = arangodb::basics::JsonHelper::checkAndGetStringValue(
              element, TRI_VOC_ATTRIBUTE_ID);
          auto tmpBuilder = basics::JsonHelper::toVelocyPack(element);
          result.emplace(id, tmpBuilder->steal());
          documentIds.erase(id);
        } catch (...) {
          // Ignore this error.
        }
      }
      TRI_json_t* filtered = TRI_LookupObjectJson(resultBody.get(), "filtered");
      if (filtered != nullptr && TRI_IsArrayJson(filtered)) {
        size_t resCount = TRI_LengthArrayJson(filtered);
        for (size_t k = 0; k < resCount; ++k) {
          TRI_json_t* element = TRI_LookupArrayJson(filtered, k);
          std::string def;
          std::string id =
              arangodb::basics::JsonHelper::getStringValue(element, def);
          documentIds.erase(id);
        }
      }
    }
  }

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get all documents in a coordinator
////////////////////////////////////////////////////////////////////////////////

int getAllDocumentsOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& returnType,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::string& contentType, std::string& resultBody) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);
  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    cc->asyncRequest("", coordTransactionID, "shard:" + p.first,
                     arangodb::GeneralRequest::RequestType::GET,
                     "/_db/" + StringUtils::urlEncode(dbname) +
                         "/_api/document?collection=" + p.first + "&type=" +
                         StringUtils::urlEncode(returnType),
                     std::shared_ptr<std::string const>(), headers, nullptr,
                     3600.0);
  }
  // Now listen to the results:
  int count;
  responseCode = arangodb::GeneralResponse::ResponseCode::OK;
  contentType = "application/json; charset=utf-8";

  arangodb::basics::Json result(arangodb::basics::Json::Object);
  arangodb::basics::Json documents(arangodb::basics::Json::Array);

  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);

    LOG(TRACE) << "Response status " << res.status;
    int error = handleGeneralCommErrors(&res);
    if (error != TRI_ERROR_NO_ERROR) {
      cc->drop("", coordTransactionID, 0, "");
      return error;
    }

    if (res.status == CL_COMM_ERROR || res.status == CL_COMM_DROPPED ||
        res.answer_code == arangodb::GeneralResponse::ResponseCode::NOT_FOUND) {
      cc->drop("", coordTransactionID, 0, "");
      return TRI_ERROR_INTERNAL;
    }

    std::unique_ptr<TRI_json_t> shardResult(
        TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, res.answer->body().c_str()));

    if (shardResult == nullptr || !TRI_IsObjectJson(shardResult.get())) {
      return TRI_ERROR_INTERNAL;
    }

    auto docs = TRI_LookupObjectJson(shardResult.get(), "documents");

    if (!TRI_IsArrayJson(docs)) {
      return TRI_ERROR_INTERNAL;
    }

    size_t const n = TRI_LengthArrayJson(docs);
    documents.reserve(n);

    for (size_t j = 0; j < n; ++j) {
      auto doc =
          static_cast<TRI_json_t*>(TRI_AtVector(&docs->_value._objects, j));

      // this will transfer the ownership for the JSON into "documents"
      documents.transfer(doc);
    }
  }

  result("documents", documents);

  resultBody = arangodb::basics::JsonHelper::toString(result.json());

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get all edges on coordinator
////////////////////////////////////////////////////////////////////////////////

int getAllEdgesOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& vertex, TRI_edge_direction_e const& direction,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::string& contentType, std::string& resultBody) {
  arangodb::basics::Json result(arangodb::basics::Json::Object);
  std::vector<traverser::TraverserExpression*> expTmp;
  int res =
      getFilteredEdgesOnCoordinator(dbname, collname, vertex, direction, expTmp,
                                    responseCode, contentType, result);
  resultBody = arangodb::basics::JsonHelper::toString(result.json());
  return res;
}

int getFilteredEdgesOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& vertex, TRI_edge_direction_e const& direction,
    std::vector<traverser::TraverserExpression*> const& expressions,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::string& contentType, arangodb::basics::Json& result) {
  TRI_ASSERT(result.isObject());
  TRI_ASSERT(result.members() == 0);

  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);
  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }

  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  std::string queryParameters = "?vertex=" + StringUtils::urlEncode(vertex);
  if (direction == TRI_EDGE_IN) {
    queryParameters += "&direction=in";
  } else if (direction == TRI_EDGE_OUT) {
    queryParameters += "&direction=out";
  }
  auto reqBodyString = std::make_shared<std::string>();
  if (!expressions.empty()) {
    arangodb::basics::Json body(Json::Array, expressions.size());
    for (auto& e : expressions) {
      arangodb::basics::Json tmp(Json::Object);
      e->toJson(tmp, TRI_UNKNOWN_MEM_ZONE);
      body.add(tmp.steal());
    }
    reqBodyString->append(body.toString());
  }
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    cc->asyncRequest("", coordTransactionID, "shard:" + p.first,
                     arangodb::GeneralRequest::RequestType::PUT,
                     "/_db/" + StringUtils::urlEncode(dbname) + "/_api/edges/" +
                         p.first + queryParameters,
                     reqBodyString, headers, nullptr, 3600.0);
  }
  // Now listen to the results:
  int count;
  responseCode = arangodb::GeneralResponse::ResponseCode::OK;
  contentType = "application/json; charset=utf-8";
  size_t filtered = 0;
  size_t scannedIndex = 0;

  arangodb::basics::Json documents(arangodb::basics::Json::Array);

  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    int error = handleGeneralCommErrors(&res);
    if (error != TRI_ERROR_NO_ERROR) {
      cc->drop("", coordTransactionID, 0, "");
      return error;
    }
    if (res.status == CL_COMM_ERROR || res.status == CL_COMM_DROPPED) {
      cc->drop("", coordTransactionID, 0, "");
      return TRI_ERROR_INTERNAL;
    }

    std::unique_ptr<TRI_json_t> shardResult(
        TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, res.answer->body().c_str()));

    if (shardResult == nullptr || !TRI_IsObjectJson(shardResult.get())) {
      return TRI_ERROR_INTERNAL;
    }

    bool const isError = arangodb::basics::JsonHelper::checkAndGetBooleanValue(
        shardResult.get(), "error");
    if (isError) {
      // shared returned an error
      return arangodb::basics::JsonHelper::getNumericValue<int>(
          shardResult.get(), "errorNum", TRI_ERROR_INTERNAL);
    }

    auto docs = TRI_LookupObjectJson(shardResult.get(), "edges");

    if (!TRI_IsArrayJson(docs)) {
      return TRI_ERROR_INTERNAL;
    }

    size_t const n = TRI_LengthArrayJson(docs);
    documents.reserve(n);

    for (size_t j = 0; j < n; ++j) {
      auto doc =
          static_cast<TRI_json_t*>(TRI_AtVector(&docs->_value._objects, j));

      // this will transfer the ownership for the JSON into "documents"
      documents.transfer(doc);
    }

    TRI_json_t* stats = arangodb::basics::JsonHelper::getObjectElement(
        shardResult.get(), "stats");
    // We do not own stats, do not delete it.

    if (stats != nullptr) {
      filtered += arangodb::basics::JsonHelper::getNumericValue<size_t>(
          stats, "filtered", 0);
      scannedIndex += arangodb::basics::JsonHelper::getNumericValue<size_t>(
          stats, "scannedIndex", 0);
    }
  }

  result("edges", documents);

  arangodb::basics::Json stats(arangodb::basics::Json::Object, 2);
  stats("scannedIndex",
        arangodb::basics::Json(static_cast<int32_t>(scannedIndex)));
  stats("filtered", arangodb::basics::Json(static_cast<int32_t>(filtered)));
  result("stats", stats);

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief modify a document in a coordinator
////////////////////////////////////////////////////////////////////////////////

int modifyDocumentOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& key, TRI_voc_rid_t const rev,
    arangodb::OperationOptions const& options, bool isPatch,
    VPackSlice const& slice,
    std::unique_ptr<std::map<std::string, std::string>>& headers,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::map<std::string, std::string>& resultHeaders,
    std::string& resultBody) {
  std::unique_ptr<TRI_json_t> json(
      arangodb::basics::VelocyPackHelper::velocyPackToJson(slice));
  return modifyDocumentOnCoordinator(dbname, collname, key, rev, options,
                                     isPatch, json, headers, responseCode,
                                     resultHeaders, resultBody);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief modify a document in a coordinator
////////////////////////////////////////////////////////////////////////////////

int modifyDocumentOnCoordinator(
    std::string const& dbname, std::string const& collname,
    std::string const& key, TRI_voc_rid_t const rev,
    arangodb::OperationOptions const& options, bool isPatch,
    std::unique_ptr<TRI_json_t>& json,
    std::unique_ptr<std::map<std::string, std::string>>& headers,
    arangodb::GeneralResponse::ResponseCode& responseCode,
    std::map<std::string, std::string>& resultHeaders,
    std::string& resultBody) {
  // Set a few variables needed for our work:
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();

  // First determine the collection ID from the name:
  std::shared_ptr<CollectionInfo> collinfo =
      ci->getCollection(dbname, collname);
  if (collinfo->empty()) {
    return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
  }
  std::string collid = StringUtils::itoa(collinfo->id());

  // We have a fast path and a slow path. The fast path only asks one shard
  // to do the job and the slow path asks them all and expects to get
  // "not found" from all but one shard. We have to cover the following
  // cases:
  //   isPatch == false    (this is a "replace" operation)
  //     Here, the complete new document is given, we assume that we
  //     can read off the responsible shard, therefore can use the fast
  //     path, this is always true if _key is the one and only sharding
  //     attribute, however, if there is any other sharding attribute,
  //     it is possible that the user has changed the values in any of
  //     them, in that case we will get a "not found" or a "sharding
  //     attributes changed answer" in the fast path. In the first case
  //     we have to delegate to the slow path.
  //   isPatch == true     (this is an "update" operation)
  //     In this case we might or might not have all sharding attributes
  //     specified in the partial document given. If _key is the one and
  //     only sharding attribute, it is always given, if not all sharding
  //     attributes are explicitly given (at least as value `null`), we must
  //     assume that the fast path cannot be used. If all sharding attributes
  //     are given, we first try the fast path, but might, as above,
  //     have to use the slow path after all.

  bool usesDefaultShardingAttributes;
  ShardID shardID;

  int error = ci->getResponsibleShard(collid, json.get(), !isPatch, shardID,
                                      usesDefaultShardingAttributes);

  if (error == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND) {
    return error;
  }

  // Some stuff to prepare cluster-internal requests:
  std::string revstr;
  if (!options.ignoreRevs && rev != 0) {
    headers->emplace("if-match", StringUtils::itoa(rev));
  }
  arangodb::GeneralRequest::RequestType reqType;
  if (isPatch) {
    reqType = arangodb::GeneralRequest::RequestType::PATCH;
    if (!options.keepNull) {
      revstr += "&keepNull=false";
    }
    if (options.mergeObjects) {
      revstr += "&mergeObjects=true";
    } else {
      revstr += "&mergeObjects=false";
    }
  } else {
    reqType = arangodb::GeneralRequest::RequestType::PUT;
  }
  if (options.returnNew) {
    revstr += "&returnNew=true";
  }

  if (options.returnOld) {
    revstr += "&returnOld=true";
  }

  auto body = std::make_shared<std::string const>(
      std::string(JsonHelper::toString(json.get())));

  if (!isPatch ||
      error != TRI_ERROR_CLUSTER_NOT_ALL_SHARDING_ATTRIBUTES_GIVEN) {
    // This is the fast method, we only have to ask one shard, unless
    // the we are in isPatch==false and the user has actually changed the
    // sharding attributes

    // Send a synchronous request to that shard using ClusterComm:
    auto res = cc->syncRequest(
        "", TRI_NewTickServer(), "shard:" + shardID, reqType,
        "/_db/" + StringUtils::urlEncode(dbname) + "/_api/document/" +
            StringUtils::urlEncode(shardID) + "/" +
            StringUtils::urlEncode(key) + "?waitForSync=" +
            (options.waitForSync ? "true" : "false") + revstr,
        *(body.get()), *headers, 60.0);

    int error = handleGeneralCommErrors(res.get());
    if (error != TRI_ERROR_NO_ERROR) {
      return error;
    }

    // Now we have to distinguish whether we still have to go the slow way:
    responseCode = static_cast<arangodb::GeneralResponse::ResponseCode>(
        res->result->getHttpReturnCode());
    if (responseCode < arangodb::GeneralResponse::ResponseCode::BAD ||
        responseCode == arangodb::GeneralResponse::ResponseCode::PRECONDITION_FAILED ||
        responseCode == arangodb::GeneralResponse::ResponseCode::CONFLICT) {
      // OK, we are done, let's report:
      resultHeaders = res->result->getHeaderFields();
      resultBody.assign(res->result->getBody().c_str(),
                        res->result->getBody().length());
      return TRI_ERROR_NO_ERROR;
    }
  }

  // If we get here, we have to do it the slow way and contact everybody:
  auto shards = collinfo->shardIds();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  for (auto const& p : *shards) {
    std::unique_ptr<std::map<std::string, std::string>> headersCopy(
        new std::map<std::string, std::string>(*headers));
    cc->asyncRequest("", coordTransactionID, "shard:" + p.first, reqType,
                     "/_db/" + StringUtils::urlEncode(dbname) +
                         "/_api/document/" + StringUtils::urlEncode(p.first) +
                         "/" + StringUtils::urlEncode(key) + "?waitForSync=" +
                         (options.waitForSync ? "true" : "false") + revstr,
                     body, headersCopy, nullptr, 60.0);
  }
  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)shards->size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code != arangodb::GeneralResponse::ResponseCode::NOT_FOUND ||
          (nrok == 0 && count == 1)) {
        nrok++;
        responseCode = res.answer_code;
        resultHeaders = res.answer->headers();
        resultHeaders["content-length"] =
            StringUtils::itoa(res.answer->contentLength());
        resultBody = res.answer->body();
      }
    }
  }

  // Note that nrok is always at least 1!
  if (nrok > 1) {
    return TRI_ERROR_CLUSTER_GOT_CONTRADICTING_ANSWERS;
  }
  return TRI_ERROR_NO_ERROR;  // the cluster operation was OK, however,
                              // the DBserver could have reported an error.
}

////////////////////////////////////////////////////////////////////////////////
/// @brief flush Wal on all DBservers
////////////////////////////////////////////////////////////////////////////////

int flushWalOnAllDBServers(bool waitForSync, bool waitForCollector) {
  ClusterInfo* ci = ClusterInfo::instance();
  ClusterComm* cc = ClusterComm::instance();
  std::vector<ServerID> DBservers = ci->getCurrentDBServers();
  CoordTransactionID coordTransactionID = TRI_NewTickServer();
  std::string url = std::string("/_admin/wal/flush?waitForSync=") +
                    (waitForSync ? "true" : "false") + "&waitForCollector=" +
                    (waitForCollector ? "true" : "false");
  auto body = std::make_shared<std::string const>();
  for (auto it = DBservers.begin(); it != DBservers.end(); ++it) {
    std::unique_ptr<std::map<std::string, std::string>> headers(
        new std::map<std::string, std::string>());
    // set collection name (shard id)
    cc->asyncRequest("", coordTransactionID, "server:" + *it,
                     arangodb::GeneralRequest::RequestType::PUT, url, body,
                     headers, nullptr, 120.0);
  }

  // Now listen to the results:
  int count;
  int nrok = 0;
  for (count = (int)DBservers.size(); count > 0; count--) {
    auto res = cc->wait("", coordTransactionID, 0, "", 0.0);
    if (res.status == CL_COMM_RECEIVED) {
      if (res.answer_code == arangodb::GeneralResponse::ResponseCode::OK) {
        nrok++;
      }
    }
  }

  if (nrok != (int)DBservers.size()) {
    return TRI_ERROR_INTERNAL;
  }

  return TRI_ERROR_NO_ERROR;
}

}  // namespace arangodb