diff --git a/CMakeLists.txt b/CMakeLists.txt index 495e493cfc..c734e51f26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,7 +125,7 @@ configure_file( find_program (GIT_EXE git) set (EXECUTE_COMMAND ${GIT_EXE} describe --all --tags --long --dirty=-dirty) -execute_process(COMMAND ${EXECUTE_COMMAND} OUTPUT_VARIABLE GIT_OUTPUT) +execute_process(COMMAND ${EXECUTE_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE GIT_OUTPUT) string(STRIP ${GIT_OUTPUT} REPOSITORY_VERSION) set(ARANGODB_BUILD_REPOSITORY ${REPOSITORY_VERSION}) diff --git a/arangod/GeneralServer/VppCommTask.cpp b/arangod/GeneralServer/VppCommTask.cpp index e8578e55d1..7c4d5981bb 100644 --- a/arangod/GeneralServer/VppCommTask.cpp +++ b/arangod/GeneralServer/VppCommTask.cpp @@ -25,6 +25,7 @@ #include "Basics/StringBuffer.h" #include "Basics/HybridLogicalClock.h" +#include "Basics/VelocyPackHelper.h" #include "GeneralServer/GeneralServer.h" #include "GeneralServer/GeneralServerFeature.h" #include "GeneralServer/RestHandler.h" @@ -137,18 +138,20 @@ std::unique_ptr createChunkForNetworkSingle( return createChunkForNetworkDetail(slices, true, 1, id, 0 /*unused*/); } -std::unique_ptr createChunkForNetworkMultiFirst( - std::vector const& slices, uint64_t id, uint32_t numberOfChunks, - uint32_t totalMessageLength) { - return createChunkForNetworkDetail(slices, true, numberOfChunks, id, - totalMessageLength); -} - -std::unique_ptr createChunkForNetworkMultiFollow( - std::vector const& slices, uint64_t id, uint32_t chunkNumber, - uint32_t totalMessageLength) { - return createChunkForNetworkDetail(slices, false, chunkNumber, id, 0); -} +// TODO FIXME make use of these functions +// std::unique_ptr createChunkForNetworkMultiFirst( +// std::vector const& slices, uint64_t id, uint32_t +// numberOfChunks, +// uint32_t totalMessageLength) { +// return createChunkForNetworkDetail(slices, true, numberOfChunks, id, +// totalMessageLength); +// } +// +// std::unique_ptr createChunkForNetworkMultiFollow( +// std::vector const& slices, uint64_t id, uint32_t chunkNumber, +// uint32_t totalMessageLength) { +// return createChunkForNetworkDetail(slices, false, chunkNumber, id, 0); +// } } VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock, @@ -156,7 +159,10 @@ VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock, : Task("VppCommTask"), GeneralCommTask(server, sock, std::move(info), timeout) { _protocol = "vpp"; - // connectionStatisticsAgentSetVpp(); + _readBuffer->reserve( + _bufferLength); // ATTENTION <- this is required so we do not + // loose information during a resize + // connectionStatisticsAgentSetVpp(); } void VppCommTask::addResponse(VppResponse* response, bool isError) { @@ -172,17 +178,24 @@ void VppCommTask::addResponse(VppResponse* response, bool isError) { std::vector slices; slices.push_back(response_message._header); + VPackBuilder builder; if (response_message._generateBody) { - slices.push_back(response_message._payload); + builder = basics::VelocyPackHelper::stanitizeExternalsChecked( + response_message._payload); + slices.push_back(builder.slice()); } // FIXME (obi) // If the message is big we will create many small chunks in a loop. // For the first tests we just send single Messages - LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << "got request:"; + LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << "got response:"; for (auto const& slice : slices) { - LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << slice.toJson(); + try { + LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << slice.toJson(); + } catch (arangodb::velocypack::Exception const& e) { + std::cout << "obi exception" << e.what(); + } } // adds chunk header infromation and creates SingBuffer* that can be @@ -345,7 +358,7 @@ bool VppCommTask::processRead() { std::size_t processedDataLen = std::distance(_readBuffer->begin(), prv._readBufferCursor); if (processedDataLen > prv._cleanupLength) { - _readBuffer->move_front(prv._cleanupLength); + _readBuffer->move_front(processedDataLen); prv._readBufferCursor = nullptr; // the positon will be set at the // begin of this function } @@ -358,11 +371,17 @@ bool VppCommTask::processRead() { << "got request:" << message._header.toJson(); _request = new VppRequest(_connectionInfo, std::move(message)); GeneralServerFeature::HANDLER_FACTORY->setRequestContext(_request); - _request->setClientTaskId(_taskId); - _protocolVersion = _request->protocolVersion(); - executeRequest(_request, - new VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR, - chunkHeader._messageID)); + if (_request->requestContext() == nullptr) { + handleSimpleError(GeneralResponse::ResponseCode::NOT_FOUND, + TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, + TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND)); + } else { + _request->setClientTaskId(_taskId); + _protocolVersion = _request->protocolVersion(); + executeRequest( + _request, new VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR, + chunkHeader._messageID)); + } } if (read_maybe_only_part_of_buffer) { diff --git a/arangod/GeneralServer/VppCommTask.h b/arangod/GeneralServer/VppCommTask.h index 018bb1daf7..77f967a4e2 100644 --- a/arangod/GeneralServer/VppCommTask.h +++ b/arangod/GeneralServer/VppCommTask.h @@ -83,11 +83,12 @@ class VppCommTask : public GeneralCommTask { }; std::unordered_map _incompleteMessages; + static size_t const _bufferLength = 4096UL; struct ProcessReadVariables { ProcessReadVariables() : _currentChunkLength(0), _readBufferCursor(nullptr), - _cleanupLength(4096UL) {} + _cleanupLength(_bufferLength / 2) {} uint32_t _currentChunkLength; // size of chunk processed or 0 when expecting // new chunk diff --git a/arangod/V8Server/v8-actions.cpp b/arangod/V8Server/v8-actions.cpp index 9cdacc0d79..b300122de5 100644 --- a/arangod/V8Server/v8-actions.cpp +++ b/arangod/V8Server/v8-actions.cpp @@ -521,21 +521,23 @@ static v8::Handle RequestCppToV8(v8::Isolate* isolate, TRI_GET_GLOBAL_STRING(ParametersKey); req->ForceSet(ParametersKey, valuesObject); - // copy cookies - v8::Handle cookiesObject = v8::Object::New(isolate); + // copy cookie -- only for http protocl + if (request->transportType() == Endpoint::TransportType::HTTP) { // FIXME + v8::Handle cookiesObject = v8::Object::New(isolate); - HttpRequest* httpRequest = dynamic_cast(request); - if (httpRequest == nullptr) { - // maybe we can just continue - THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL); - } else { - for (auto& it : httpRequest->cookieValues()) { - cookiesObject->ForceSet(TRI_V8_STD_STRING(it.first), - TRI_V8_STD_STRING(it.second)); + HttpRequest* httpRequest = dynamic_cast(request); + if (httpRequest == nullptr) { + // maybe we can just continue + THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL); + } else { + for (auto& it : httpRequest->cookieValues()) { + cookiesObject->ForceSet(TRI_V8_STD_STRING(it.first), + TRI_V8_STD_STRING(it.second)); + } } + TRI_GET_GLOBAL_STRING(CookiesKey); + req->ForceSet(CookiesKey, cookiesObject); } - TRI_GET_GLOBAL_STRING(CookiesKey); - req->ForceSet(CookiesKey, cookiesObject); return req; } @@ -567,7 +569,7 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g, TRI_GET_GLOBAL_STRING(ContentTypeKey); if (res->Has(ContentTypeKey)) { contentType = TRI_ObjectToString(res->Get(ContentTypeKey)); - if (contentType != "application/json") { + if (contentType.find("application/json") == std::string::npos) { jsonContent = false; } switch (response->transportType()) { @@ -655,8 +657,10 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g, case Endpoint::TransportType::VPP: { VPackBuilder builder; - v8::Handle v8_body = res->Get(BodyKey); + v8::Handle v8Body = res->Get(BodyKey); + // LOG(ERR) << v8Body->IsString(); std::string out; + bool done = false; // decode and set out if (transformArray->IsArray()) { @@ -681,16 +685,21 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g, } // out is not set - if (out.empty()) { - if (jsonContent && !V8Buffer::hasInstance(isolate, v8_body)) { - TRI_V8ToVPack(isolate, builder, v8_body, false); + if (out.empty() && !done) { + if (jsonContent && !V8Buffer::hasInstance(isolate, v8Body)) { + if (v8Body->IsString()) { + out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved + } else { + TRI_V8ToVPack(isolate, builder, v8Body, false); + done = true; + } // done } else if (V8Buffer::hasInstance( isolate, - v8_body)) { // body form buffer - could - // contain json or not + v8Body)) { // body form buffer - could + // contain json or not // REVIEW (fc) - is this correct? - auto obj = v8_body.As(); + auto obj = v8Body.As(); out = std::string(V8Buffer::data(obj), V8Buffer::length(obj)); } else { // body is text - does not contain json out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved @@ -708,11 +717,14 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g, } catch (...) { // do nothing // json could not be converted // there was no json - change content type? + LOG_TOPIC(DEBUG, Logger::COMMUNICATION) + << "failed to parse json:\n" << out; } } if (!gotJson) { - builder.add(VPackValue(out)); // add test to the builder + builder.add(VPackValue( + out)); // add output to the builder - when not added via parser } } @@ -829,13 +841,10 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g, static TRI_action_result_t ExecuteActionVocbase( TRI_vocbase_t* vocbase, v8::Isolate* isolate, TRI_action_t const* action, v8::Handle callback, GeneralRequest* request, - GeneralResponse* responseGeneral) { + GeneralResponse* response) { v8::HandleScope scope(isolate); v8::TryCatch tryCatch; - // TODO needs to generalized - auto response = dynamic_cast(responseGeneral); - if (response == nullptr) { THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL); } @@ -911,7 +920,11 @@ static TRI_action_result_t ExecuteActionVocbase( errorMessage = TRI_errno_string(errorCode); } - response->body().appendText(errorMessage); + // TODO (obi) + if (response->transportType() == Endpoint::TransportType::HTTP) { // FIXME + ((HttpResponse*)response)->body().appendText(errorMessage); + } + } else if (v8g->_canceled) { @@ -924,7 +937,11 @@ static TRI_action_result_t ExecuteActionVocbase( response->setResponseCode(GeneralResponse::ResponseCode::SERVER_ERROR); // TODO how to generalize this? - response->body().appendText(TRI_StringifyV8Exception(isolate, &tryCatch)); + if (response->transportType() == + Endpoint::TransportType::HTTP) { // FIXME + ((HttpResponse*)response)->body().appendText( + TRI_StringifyV8Exception(isolate, &tryCatch)); + } } else { v8g->_canceled = true; result.isValid = false; diff --git a/lib/Basics/VelocyPackHelper.cpp b/lib/Basics/VelocyPackHelper.cpp index 181c66c9e7..f4b018bfe0 100644 --- a/lib/Basics/VelocyPackHelper.cpp +++ b/lib/Basics/VelocyPackHelper.cpp @@ -42,7 +42,8 @@ #include extern "C" { - unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); +unsigned long long XXH64(const void* input, size_t length, + unsigned long long seed); } using VelocyPackHelper = arangodb::basics::VelocyPackHelper; @@ -56,16 +57,21 @@ struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler { VPackValueLength keyLength; char const* p = key.getString(keyLength); - if (p == nullptr || *p != '_' || keyLength < 3 || keyLength > 5 || nesting > 0) { + if (p == nullptr || *p != '_' || keyLength < 3 || keyLength > 5 || + nesting > 0) { // keep attribute return true; } // exclude these attributes (but not _key!) - if ((keyLength == 3 && memcmp(p, "_id", static_cast(keyLength)) == 0) || - (keyLength == 4 && memcmp(p, "_rev", static_cast(keyLength)) == 0) || - (keyLength == 3 && memcmp(p, "_to", static_cast(keyLength)) == 0) || - (keyLength == 5 && memcmp(p, "_from", static_cast(keyLength)) == 0)) { + if ((keyLength == 3 && + memcmp(p, "_id", static_cast(keyLength)) == 0) || + (keyLength == 4 && + memcmp(p, "_rev", static_cast(keyLength)) == 0) || + (keyLength == 3 && + memcmp(p, "_to", static_cast(keyLength)) == 0) || + (keyLength == 5 && + memcmp(p, "_from", static_cast(keyLength)) == 0)) { return true; } @@ -85,9 +91,9 @@ void VelocyPackHelper::initialize() { Translator.reset(new VPackAttributeTranslator); // these attribute names will be translated into short integer values - Translator->add(StaticStrings::KeyString, KeyAttribute - AttributeBase); + Translator->add(StaticStrings::KeyString, KeyAttribute - AttributeBase); Translator->add(StaticStrings::RevString, RevAttribute - AttributeBase); - Translator->add(StaticStrings::IdString, IdAttribute - AttributeBase); + Translator->add(StaticStrings::IdString, IdAttribute - AttributeBase); Translator->add(StaticStrings::FromString, FromAttribute - AttributeBase); Translator->add(StaticStrings::ToString, ToAttribute - AttributeBase); @@ -95,21 +101,34 @@ void VelocyPackHelper::initialize() { // set the attribute translator in the global options VPackOptions::Defaults.attributeTranslator = Translator.get(); - VPackOptions::Defaults.unsupportedTypeBehavior = VPackOptions::ConvertUnsupportedType; - VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set when converting to JSON for HTTP xfer + VPackOptions::Defaults.unsupportedTypeBehavior = + VPackOptions::ConvertUnsupportedType; + VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set + // when converting to JSON for + // HTTP xfer // run quick selfs test with the attribute translator - TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::KeyString)).getUInt() == KeyAttribute - AttributeBase); - TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::RevString)).getUInt() == RevAttribute - AttributeBase); - TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::IdString)).getUInt() == IdAttribute - AttributeBase); - TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::FromString)).getUInt() == FromAttribute - AttributeBase); - TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::ToString)).getUInt() == ToAttribute - AttributeBase); - - TRI_ASSERT(VPackSlice(Translator->translate(KeyAttribute - AttributeBase)).copyString() == StaticStrings::KeyString); - TRI_ASSERT(VPackSlice(Translator->translate(RevAttribute - AttributeBase)).copyString() == StaticStrings::RevString); - TRI_ASSERT(VPackSlice(Translator->translate(IdAttribute - AttributeBase)).copyString() == StaticStrings::IdString); - TRI_ASSERT(VPackSlice(Translator->translate(FromAttribute - AttributeBase)).copyString() == StaticStrings::FromString); - TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase)).copyString() == StaticStrings::ToString); + TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::KeyString)) + .getUInt() == KeyAttribute - AttributeBase); + TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::RevString)) + .getUInt() == RevAttribute - AttributeBase); + TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::IdString)) + .getUInt() == IdAttribute - AttributeBase); + TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::FromString)) + .getUInt() == FromAttribute - AttributeBase); + TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::ToString)) + .getUInt() == ToAttribute - AttributeBase); + + TRI_ASSERT(VPackSlice(Translator->translate(KeyAttribute - AttributeBase)) + .copyString() == StaticStrings::KeyString); + TRI_ASSERT(VPackSlice(Translator->translate(RevAttribute - AttributeBase)) + .copyString() == StaticStrings::RevString); + TRI_ASSERT(VPackSlice(Translator->translate(IdAttribute - AttributeBase)) + .copyString() == StaticStrings::IdString); + TRI_ASSERT(VPackSlice(Translator->translate(FromAttribute - AttributeBase)) + .copyString() == StaticStrings::FromString); + TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase)) + .copyString() == StaticStrings::ToString); // initialize exclude handler for system attributes ExcludeHandler.reset(new SystemAttributeExcludeHandler); @@ -127,7 +146,8 @@ void VelocyPackHelper::disableAssemblerFunctions() { /// @brief return the (global) attribute exclude handler instance //////////////////////////////////////////////////////////////////////////////// -arangodb::velocypack::AttributeExcludeHandler* VelocyPackHelper::getExcludeHandler() { +arangodb::velocypack::AttributeExcludeHandler* +VelocyPackHelper::getExcludeHandler() { return ExcludeHandler.get(); } @@ -139,15 +159,14 @@ arangodb::velocypack::AttributeTranslator* VelocyPackHelper::getTranslator() { return Translator.get(); } - -bool VelocyPackHelper::AttributeSorterUTF8::operator()(std::string const& l, - std::string const& r) const { +bool VelocyPackHelper::AttributeSorterUTF8::operator()( + std::string const& l, std::string const& r) const { // use UTF-8-based comparison of attribute names return TRI_compare_utf8(l.c_str(), l.size(), r.c_str(), r.size()) < 0; } -bool VelocyPackHelper::AttributeSorterBinary::operator()(std::string const& l, - std::string const& r) const { +bool VelocyPackHelper::AttributeSorterBinary::operator()( + std::string const& l, std::string const& r) const { // use binary comparison of attribute names size_t cmpLength = (std::min)(l.size(), r.size()); int res = memcmp(l.c_str(), r.c_str(), cmpLength); @@ -164,15 +183,19 @@ size_t VelocyPackHelper::VPackHash::operator()(VPackSlice const& slice) const { return static_cast(slice.normalizedHash()); }; -size_t VelocyPackHelper::VPackStringHash::operator()(VPackSlice const& slice) const noexcept { +size_t VelocyPackHelper::VPackStringHash::operator()( + VPackSlice const& slice) const noexcept { return static_cast(slice.hashString()); }; -bool VelocyPackHelper::VPackEqual::operator()(VPackSlice const& lhs, VPackSlice const& rhs) const { +bool VelocyPackHelper::VPackEqual::operator()(VPackSlice const& lhs, + VPackSlice const& rhs) const { return VelocyPackHelper::compare(lhs, rhs, false, _options) == 0; }; -bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, VPackSlice const& rhs) const noexcept { +bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, + VPackSlice const& rhs) const + noexcept { auto const lh = lhs.head(); auto const rh = rhs.head(); @@ -183,21 +206,26 @@ bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, VPack VPackValueLength size; if (lh == 0xbf) { // long UTF-8 String - size = static_cast(velocypack::readInteger(lhs.begin() + 1, 8)); - if (size !=static_cast(velocypack::readInteger(rhs.begin() + 1, 8))) { + size = static_cast( + velocypack::readInteger(lhs.begin() + 1, 8)); + if (size != + static_cast( + velocypack::readInteger(rhs.begin() + 1, 8))) { return false; } - return (memcmp(lhs.start() + 1 + 8, rhs.start() + 1 + 8, static_cast(size)) == 0); - } - + return (memcmp(lhs.start() + 1 + 8, rhs.start() + 1 + 8, + static_cast(size)) == 0); + } + size = static_cast(lh - 0x40); - return (memcmp(lhs.start() + 1, rhs.start() + 1, static_cast(size)) == 0); + return (memcmp(lhs.start() + 1, rhs.start() + 1, static_cast(size)) == + 0); }; static int TypeWeight(VPackSlice const& slice) { switch (slice.type()) { case VPackValueType::MinKey: - return -99; // must be lowest + return -99; // must be lowest case VPackValueType::Illegal: return -1; case VPackValueType::None: @@ -224,14 +252,14 @@ static int TypeWeight(VPackSlice const& slice) { case VPackValueType::External: return TypeWeight(slice.resolveExternal()); case VPackValueType::MaxKey: - return 99; // must be highest + return 99; // must be highest default: // All other values have equal weight return 0; } } -int VelocyPackHelper::compareNumberValues(VPackValueType lhsType, +int VelocyPackHelper::compareNumberValues(VPackValueType lhsType, VPackSlice lhs, VPackSlice rhs) { if (lhsType == rhs.type()) { // both types are equal @@ -315,14 +343,12 @@ std::string VelocyPackHelper::checkAndGetStringValue(VPackSlice const& slice, std::string const& name) { TRI_ASSERT(slice.isObject()); if (!slice.hasKey(name)) { - std::string msg = - "The attribute '" + name + "' was not found."; + std::string msg = "The attribute '" + name + "' was not found."; THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg); } VPackSlice const sub = slice.get(name); if (!sub.isString()) { - std::string msg = - "The attribute '" + name + "' is not a string."; + std::string msg = "The attribute '" + name + "' is not a string."; THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg); } return sub.copyString(); @@ -346,8 +372,7 @@ std::string VelocyPackHelper::getStringValue(VPackSlice const& slice, /// or it is not a string //////////////////////////////////////////////////////////////////////////////// -std::string VelocyPackHelper::getStringValue(VPackSlice slice, - char const* name, +std::string VelocyPackHelper::getStringValue(VPackSlice slice, char const* name, std::string const& defaultValue) { if (slice.isExternal()) { slice = VPackSlice(slice.getExternal()); @@ -396,7 +421,8 @@ uint64_t VelocyPackHelper::stringUInt64(VPackSlice const& slice) { return 0; } -TRI_json_t* VelocyPackHelper::velocyPackToJson(VPackSlice const& slice, VPackOptions const* options) { +TRI_json_t* VelocyPackHelper::velocyPackToJson(VPackSlice const& slice, + VPackOptions const* options) { return JsonHelper::fromString(slice.toJson(options)); } @@ -487,19 +513,22 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename, TRI_UnlinkFile(tmp.c_str()); } - int fd = TRI_CREATE(tmp.c_str(), O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC, + int fd = TRI_CREATE(tmp.c_str(), + O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); if (fd < 0) { TRI_set_errno(TRI_ERROR_SYS_ERROR); - LOG(ERR) << "cannot create json file '" << tmp << "': " << TRI_LAST_ERROR_STR; + LOG(ERR) << "cannot create json file '" << tmp + << "': " << TRI_LAST_ERROR_STR; return false; } if (!PrintVelocyPack(fd, slice, true)) { TRI_CLOSE(fd); TRI_set_errno(TRI_ERROR_SYS_ERROR); - LOG(ERR) << "cannot write to json file '" << tmp << "': " << TRI_LAST_ERROR_STR; + LOG(ERR) << "cannot write to json file '" << tmp + << "': " << TRI_LAST_ERROR_STR; TRI_UnlinkFile(tmp.c_str()); return false; } @@ -510,7 +539,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename, if (!TRI_fsync(fd)) { TRI_CLOSE(fd); TRI_set_errno(TRI_ERROR_SYS_ERROR); - LOG(ERR) << "cannot sync saved json '" << tmp << "': " << TRI_LAST_ERROR_STR; + LOG(ERR) << "cannot sync saved json '" << tmp + << "': " << TRI_LAST_ERROR_STR; TRI_UnlinkFile(tmp.c_str()); return false; } @@ -520,7 +550,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename, if (res < 0) { TRI_set_errno(TRI_ERROR_SYS_ERROR); - LOG(ERR) << "cannot close saved file '" << tmp << "': " << TRI_LAST_ERROR_STR; + LOG(ERR) << "cannot close saved file '" << tmp + << "': " << TRI_LAST_ERROR_STR; TRI_UnlinkFile(tmp.c_str()); return false; } @@ -529,7 +560,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename, if (res != TRI_ERROR_NO_ERROR) { TRI_set_errno(res); - LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename << "': " << TRI_LAST_ERROR_STR; + LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename + << "': " << TRI_LAST_ERROR_STR; TRI_UnlinkFile(tmp.c_str()); return false; @@ -538,9 +570,10 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename, return true; } -int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, - bool useUTF8, VPackOptions const* options, - VPackSlice const* lhsBase, VPackSlice const* rhsBase) { +int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, bool useUTF8, + VPackOptions const* options, + VPackSlice const* lhsBase, + VPackSlice const* rhsBase) { { // will resolve externals... int lWeight = TypeWeight(lhs); @@ -553,12 +586,12 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, if (lWeight > rWeight) { return 1; } - + TRI_ASSERT(lWeight == rWeight); } - - lhs = lhs.resolveExternal(); // follow externals - rhs = rhs.resolveExternal(); // follow externals + + lhs = lhs.resolveExternal(); // follow externals + rhs = rhs.resolveExternal(); // follow externals // lhs and rhs have equal weights if (lhs.isNone() || rhs.isNone()) { @@ -578,7 +611,7 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, case VPackValueType::MaxKey: case VPackValueType::None: case VPackValueType::Null: - return 0; + return 0; case VPackValueType::Bool: { bool left = lhs.getBoolean(); bool right = rhs.getBoolean(); @@ -602,11 +635,13 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, VPackValueLength nl; char const* left; if (lhs.isCustom()) { - if (lhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) { + if (lhsBase == nullptr || options == nullptr || + options->customTypeHandler == nullptr) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "Could not extract custom attribute."); } - lhsString.assign(options->customTypeHandler->toString(lhs, options, *lhsBase)); + lhsString.assign( + options->customTypeHandler->toString(lhs, options, *lhsBase)); left = lhsString.c_str(); nl = lhsString.size(); } else { @@ -618,11 +653,13 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, VPackValueLength nr; char const* right; if (rhs.isCustom()) { - if (rhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) { + if (rhsBase == nullptr || options == nullptr || + options->customTypeHandler == nullptr) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "Could not extract custom attribute."); } - rhsString.assign(options->customTypeHandler->toString(rhs, options, *rhsBase)); + rhsString.assign( + options->customTypeHandler->toString(rhs, options, *rhsBase)); right = rhsString.c_str(); nr = rhsString.size(); } else { @@ -632,7 +669,8 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, int res; if (useUTF8) { - res = TRI_compare_utf8(left, static_cast(nl), right, static_cast(nr)); + res = TRI_compare_utf8(left, static_cast(nl), right, + static_cast(nr)); } else { size_t len = static_cast(nl < nr ? nl : nr); res = memcmp(left, right, len); @@ -743,7 +781,8 @@ double VelocyPackHelper::toDouble(VPackSlice const& slice, bool& failed) { if (n == 0) { return 0.0; } else if (n == 1) { - return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(), failed); + return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(), + failed); } break; } @@ -793,7 +832,8 @@ uint64_t VelocyPackHelper::hashByAttributes( return hash; } -void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& output) { +void VelocyPackHelper::SanitizeExternals(VPackSlice const input, + VPackBuilder& output) { if (input.isExternal()) { output.add(input.resolveExternal()); } else if (input.isObject()) { @@ -814,8 +854,44 @@ void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& o } } -arangodb::LoggerStream& operator<< (arangodb::LoggerStream& logger, - VPackSlice const& slice) { +bool VelocyPackHelper::hasExternals(VPackSlice input) { + if (input.isExternal()) { + return true; + } else if (input.isObject()) { + for (auto const& it : VPackObjectIterator(input)) { + if (hasExternals(it.value) == true) { + return true; + }; + } + } else if (input.isArray()) { + for (auto const& it : VPackArrayIterator(input)) { + if (hasExternals(it) == true) { + return true; + } + } + } + return false; +} + +VPackBuilder VelocyPackHelper::stanitizeExternalsChecked(VPackSlice input, + bool checkExternals) { + VPackBuilder builder; + bool resolveExt = true; + if (checkExternals) { + resolveExt = hasExternals(input); + } + + if (resolveExt) { // resolve + SanitizeExternals(input, builder); + } else { + builder.add(input); + } + + return builder; // elided +} + +arangodb::LoggerStream& operator<<(arangodb::LoggerStream& logger, + VPackSlice const& slice) { size_t const cutoff = 100; std::string sliceStr(slice.toJson()); bool longer = sliceStr.size() > cutoff; diff --git a/lib/Basics/VelocyPackHelper.h b/lib/Basics/VelocyPackHelper.h index 957e0b3c8e..d2697da4c8 100644 --- a/lib/Basics/VelocyPackHelper.h +++ b/lib/Basics/VelocyPackHelper.h @@ -47,10 +47,9 @@ class VelocyPackHelper { ~VelocyPackHelper() = delete; public: - -//////////////////////////////////////////////////////////////////////////////// -/// @brief static initializer for all VPack values -//////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + /// @brief static initializer for all VPack values + //////////////////////////////////////////////////////////////////////////////// static void initialize(); static void disableAssemblerFunctions(); @@ -66,20 +65,21 @@ class VelocyPackHelper { size_t operator()(arangodb::velocypack::Slice const&) const noexcept; }; -//////////////////////////////////////////////////////////////////////////////// -/// @brief equality comparator for VelocyPack values -//////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + /// @brief equality comparator for VelocyPack values + //////////////////////////////////////////////////////////////////////////////// struct VPackEqual { - private: - arangodb::velocypack::Options const* _options; - public: - VPackEqual() : _options(nullptr) {} - explicit VPackEqual(arangodb::velocypack::Options const* opts) - : _options(opts) {} + private: + arangodb::velocypack::Options const* _options; - bool operator()(arangodb::velocypack::Slice const&, - arangodb::velocypack::Slice const&) const; + public: + VPackEqual() : _options(nullptr) {} + explicit VPackEqual(arangodb::velocypack::Options const* opts) + : _options(opts) {} + + bool operator()(arangodb::velocypack::Slice const&, + arangodb::velocypack::Slice const&) const; }; struct VPackStringEqual { @@ -87,9 +87,9 @@ class VelocyPackHelper { arangodb::velocypack::Slice const&) const noexcept; }; -//////////////////////////////////////////////////////////////////////////////// -/// @brief less comparator for VelocyPack values -//////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + /// @brief less comparator for VelocyPack values + //////////////////////////////////////////////////////////////////////////////// template struct VPackLess { @@ -159,7 +159,7 @@ class VelocyPackHelper { struct AttributeSorterUTF8 { bool operator()(std::string const& l, std::string const& r) const; }; - + struct AttributeSorterBinary { bool operator()(std::string const& l, std::string const& r) const; }; @@ -206,8 +206,9 @@ class VelocyPackHelper { ////////////////////////////////////////////////////////////////////////////// static std::string checkAndGetStringValue(VPackSlice const&, char const*); - - static std::string checkAndGetStringValue(VPackSlice const&, std::string const&); + + static std::string checkAndGetStringValue(VPackSlice const&, + std::string const&); ////////////////////////////////////////////////////////////////////////////// /// @brief returns a Numeric sub-element, or throws if does not exist @@ -243,8 +244,10 @@ class VelocyPackHelper { /// or it is not a string ////////////////////////////////////////////////////////////////////////////// - static std::string getStringValue(VPackSlice, char const*, std::string const&); - static std::string getStringValue(VPackSlice, std::string const&, std::string const&); + static std::string getStringValue(VPackSlice, char const*, + std::string const&); + static std::string getStringValue(VPackSlice, std::string const&, + std::string const&); ////////////////////////////////////////////////////////////////////////////// /// @brief convert a Object sub value into a uint64 @@ -256,7 +259,8 @@ class VelocyPackHelper { /// @brief Build TRI_json_t from VelocyPack. Just a temporary solution ////////////////////////////////////////////////////////////////////////////// - static TRI_json_t* velocyPackToJson(VPackSlice const&, VPackOptions const* = &VPackOptions::Defaults); + static TRI_json_t* velocyPackToJson( + VPackSlice const&, VPackOptions const* = &VPackOptions::Defaults); ////////////////////////////////////////////////////////////////////////////// /// @brief parses a json file to VelocyPack @@ -268,7 +272,7 @@ class VelocyPackHelper { /// @brief writes a VelocyPack to a file ////////////////////////////////////////////////////////////////////////////// - static bool velocyPackToFile(std::string const& filename, + static bool velocyPackToFile(std::string const& filename, VPackSlice const& slice, bool syncFile); ////////////////////////////////////////////////////////////////////////////// @@ -284,8 +288,9 @@ class VelocyPackHelper { ////////////////////////////////////////////////////////////////////////////// static int compare(arangodb::velocypack::Slice lhs, - arangodb::velocypack::Slice rhs, - bool useUTF8, arangodb::velocypack::Options const* options = &arangodb::velocypack::Options::Defaults, + arangodb::velocypack::Slice rhs, bool useUTF8, + arangodb::velocypack::Options const* options = + &arangodb::velocypack::Options::Defaults, arangodb::velocypack::Slice const* lhsBase = nullptr, arangodb::velocypack::Slice const* rhsBase = nullptr); @@ -325,7 +330,7 @@ class VelocyPackHelper { } return arangodb::velocypack::Slice::falseSlice(); } - + static inline arangodb::velocypack::Slice ZeroValue() { return arangodb::velocypack::Slice::zeroSlice(); } @@ -350,6 +355,10 @@ class VelocyPackHelper { static void SanitizeExternals(arangodb::velocypack::Slice const, arangodb::velocypack::Builder&); + static bool hasExternals(arangodb::velocypack::Slice const); + static VPackBuilder stanitizeExternalsChecked( + arangodb::velocypack::Slice const, bool checkExternals = true); + static uint8_t const KeyAttribute = 0x31; static uint8_t const RevAttribute = 0x32; static uint8_t const IdAttribute = 0x33; @@ -358,10 +367,12 @@ class VelocyPackHelper { static uint8_t const AttributeBase = 0x30; - static_assert(KeyAttribute < RevAttribute, "invalid value for _key attribute"); + static_assert(KeyAttribute < RevAttribute, + "invalid value for _key attribute"); static_assert(RevAttribute < IdAttribute, "invalid value for _rev attribute"); static_assert(IdAttribute < FromAttribute, "invalid value for _id attribute"); - static_assert(FromAttribute < ToAttribute, "invalid value for _from attribute"); + static_assert(FromAttribute < ToAttribute, + "invalid value for _from attribute"); }; } } @@ -371,6 +382,6 @@ class VelocyPackHelper { ////////////////////////////////////////////////////////////////////////////// arangodb::LoggerStream& operator<<(arangodb::LoggerStream&, - arangodb::velocypack::Slice const&); + arangodb::velocypack::Slice const&); #endif diff --git a/lib/Logger/LogTopic.cpp b/lib/Logger/LogTopic.cpp index a09fcfd6be..9d5d13fe28 100644 --- a/lib/Logger/LogTopic.cpp +++ b/lib/Logger/LogTopic.cpp @@ -39,7 +39,7 @@ std::map LogTopic::_names; LogTopic Logger::AGENCY("agency", LogLevel::INFO); LogTopic Logger::AGENCYCOMM("agencycomm", LogLevel::INFO); LogTopic Logger::COLLECTOR("collector"); -LogTopic Logger::COMMUNICATION("COMMUNICATION", LogLevel::INFO); +LogTopic Logger::COMMUNICATION("communication", LogLevel::INFO); LogTopic Logger::COMPACTOR("compactor"); LogTopic Logger::CONFIG("config"); LogTopic Logger::DATAFILES("datafiles", LogLevel::INFO); diff --git a/lib/Rest/GeneralRequest.h b/lib/Rest/GeneralRequest.h index 46542d2aa6..42deb7822b 100644 --- a/lib/Rest/GeneralRequest.h +++ b/lib/Rest/GeneralRequest.h @@ -165,6 +165,7 @@ class GeneralRequest { std::vector const& suffix() const { return _suffix; } void addSuffix(std::string&& part); + virtual arangodb::Endpoint::TransportType transportType() = 0; virtual int64_t contentLength() const = 0; // get value from headers map. The key must be lowercase. virtual std::string const& header(std::string const& key) const = 0; diff --git a/lib/Rest/HttpRequest.h b/lib/Rest/HttpRequest.h index f00c909b18..84b40ccfd5 100644 --- a/lib/Rest/HttpRequest.h +++ b/lib/Rest/HttpRequest.h @@ -70,6 +70,9 @@ class HttpRequest : public GeneralRequest { bool isHttp11() const { return _version == ProtocolVersion::HTTP_1_1; } public: + arangodb::Endpoint::TransportType transportType() override { + return arangodb::Endpoint::TransportType::HTTP; + } // the content length int64_t contentLength() const override { return _contentLength; } diff --git a/lib/Rest/VppRequest.h b/lib/Rest/VppRequest.h index 4ddfb90b25..3842f706cc 100644 --- a/lib/Rest/VppRequest.h +++ b/lib/Rest/VppRequest.h @@ -71,6 +71,10 @@ class VppRequest : public GeneralRequest { return _message._payload.byteSize(); } + virtual arangodb::Endpoint::TransportType transportType() override { + return arangodb::Endpoint::TransportType::VPP; + }; + std::unordered_map const& headers() const override; // get value from headers map. The key must be lowercase. std::string const& header(std::string const& key) const override;