Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'obi-velocystream' into engine-vs-velocystream 

* obi-velocystream:
  fix buffer clean up
  avoid bug that occurred during buffer resize by allocating up front
  i fix type of execepton so it gets caught
  avoid dumper exception
  failed try to resolve externals
  add sanitizeExternalsChecked function
  added handleSimpleError for database not found
  fix build issue
  silence complier warnings
  fix: now returns object instead of string
  try to fix simple route
This commit is contained in:
Jan Christoph Uhde 2016-08-16 11:12:23 +02:00
parent 000fd9cbdc 9afe3a1016
commit d8d6fb1ed0
10 changed files with 284 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -125,7 +125,7 @@ configure_file(
find_program (GIT_EXE git) find_program (GIT_EXE git)
set (EXECUTE_COMMAND ${GIT_EXE} describe --all --tags --long --dirty=-dirty) set (EXECUTE_COMMAND ${GIT_EXE} describe --all --tags --long --dirty=-dirty)
execute_process(COMMAND ${EXECUTE_COMMAND} OUTPUT_VARIABLE GIT_OUTPUT) execute_process(COMMAND ${EXECUTE_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE GIT_OUTPUT)
string(STRIP ${GIT_OUTPUT} REPOSITORY_VERSION) string(STRIP ${GIT_OUTPUT} REPOSITORY_VERSION)
set(ARANGODB_BUILD_REPOSITORY ${REPOSITORY_VERSION}) set(ARANGODB_BUILD_REPOSITORY ${REPOSITORY_VERSION})

53
arangod/GeneralServer/VppCommTask.cpp
View File

@ -25,6 +25,7 @@
#include "Basics/StringBuffer.h" #include "Basics/StringBuffer.h"
#include "Basics/HybridLogicalClock.h" #include "Basics/HybridLogicalClock.h"
#include "Basics/VelocyPackHelper.h"
#include "GeneralServer/GeneralServer.h" #include "GeneralServer/GeneralServer.h"
#include "GeneralServer/GeneralServerFeature.h" #include "GeneralServer/GeneralServerFeature.h"
#include "GeneralServer/RestHandler.h" #include "GeneralServer/RestHandler.h"
@ -137,18 +138,20 @@ std::unique_ptr<basics::StringBuffer> createChunkForNetworkSingle(
return createChunkForNetworkDetail(slices, true, 1, id, 0 /*unused*/); return createChunkForNetworkDetail(slices, true, 1, id, 0 /*unused*/);
} }
std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFirst( // TODO FIXME make use of these functions
std::vector<VPackSlice> const& slices, uint64_t id, uint32_t numberOfChunks, // std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFirst(
uint32_t totalMessageLength) { // std::vector<VPackSlice> const& slices, uint64_t id, uint32_t
return createChunkForNetworkDetail(slices, true, numberOfChunks, id, // numberOfChunks,
totalMessageLength); // uint32_t totalMessageLength) {
} // return createChunkForNetworkDetail(slices, true, numberOfChunks, id,
// totalMessageLength);
std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFollow( // }
std::vector<VPackSlice> const& slices, uint64_t id, uint32_t chunkNumber, //
uint32_t totalMessageLength) { // std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFollow(
return createChunkForNetworkDetail(slices, false, chunkNumber, id, 0); // std::vector<VPackSlice> const& slices, uint64_t id, uint32_t chunkNumber,
} // uint32_t totalMessageLength) {
// return createChunkForNetworkDetail(slices, false, chunkNumber, id, 0);
// }
} }
VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock, VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock,
@ -156,6 +159,9 @@ VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock,
: Task("VppCommTask"), : Task("VppCommTask"),
GeneralCommTask(server, sock, std::move(info), timeout) { GeneralCommTask(server, sock, std::move(info), timeout) {
_protocol = "vpp"; _protocol = "vpp";
_readBuffer->reserve(
_bufferLength); // ATTENTION <- this is required so we do not
// loose information during a resize
// connectionStatisticsAgentSetVpp(); // connectionStatisticsAgentSetVpp();
} }
@ -172,17 +178,24 @@ void VppCommTask::addResponse(VppResponse* response, bool isError) {
std::vector<VPackSlice> slices; std::vector<VPackSlice> slices;
slices.push_back(response_message._header); slices.push_back(response_message._header);
VPackBuilder builder;
if (response_message._generateBody) { if (response_message._generateBody) {
slices.push_back(response_message._payload); builder = basics::VelocyPackHelper::stanitizeExternalsChecked(
response_message._payload);
slices.push_back(builder.slice());
} }
// FIXME (obi) // FIXME (obi)
// If the message is big we will create many small chunks in a loop. // If the message is big we will create many small chunks in a loop.
// For the first tests we just send single Messages // For the first tests we just send single Messages
LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << "got request:"; LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << "got response:";
for (auto const& slice : slices) { for (auto const& slice : slices) {
try {
LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << slice.toJson(); LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << slice.toJson();
} catch (arangodb::velocypack::Exception const& e) {
std::cout << "obi exception" << e.what();
}
} }
// adds chunk header infromation and creates SingBuffer* that can be // adds chunk header infromation and creates SingBuffer* that can be
@ -345,7 +358,7 @@ bool VppCommTask::processRead() {
std::size_t processedDataLen = std::size_t processedDataLen =
std::distance(_readBuffer->begin(), prv._readBufferCursor); std::distance(_readBuffer->begin(), prv._readBufferCursor);
if (processedDataLen > prv._cleanupLength) { if (processedDataLen > prv._cleanupLength) {
_readBuffer->move_front(prv._cleanupLength); _readBuffer->move_front(processedDataLen);
prv._readBufferCursor = nullptr; // the positon will be set at the prv._readBufferCursor = nullptr; // the positon will be set at the
// begin of this function // begin of this function
} }
@ -358,12 +371,18 @@ bool VppCommTask::processRead() {
<< "got request:" << message._header.toJson(); << "got request:" << message._header.toJson();
_request = new VppRequest(_connectionInfo, std::move(message)); _request = new VppRequest(_connectionInfo, std::move(message));
GeneralServerFeature::HANDLER_FACTORY->setRequestContext(_request); GeneralServerFeature::HANDLER_FACTORY->setRequestContext(_request);
if (_request->requestContext() == nullptr) {
handleSimpleError(GeneralResponse::ResponseCode::NOT_FOUND,
TRI_ERROR_ARANGO_DATABASE_NOT_FOUND,
TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND));
} else {
_request->setClientTaskId(_taskId); _request->setClientTaskId(_taskId);
_protocolVersion = _request->protocolVersion(); _protocolVersion = _request->protocolVersion();
executeRequest(_request, executeRequest(
new VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR, _request, new VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR,
chunkHeader._messageID)); chunkHeader._messageID));
} }
}
if (read_maybe_only_part_of_buffer) { if (read_maybe_only_part_of_buffer) {
if (prv._readBufferCursor == _readBuffer->end()) { if (prv._readBufferCursor == _readBuffer->end()) {

3
arangod/GeneralServer/VppCommTask.h
View File

@ -83,11 +83,12 @@ class VppCommTask : public GeneralCommTask {
}; };
std::unordered_map<MessageID, IncompleteVPackMessage> _incompleteMessages; std::unordered_map<MessageID, IncompleteVPackMessage> _incompleteMessages;
static size_t const _bufferLength = 4096UL;
struct ProcessReadVariables { struct ProcessReadVariables {
ProcessReadVariables() ProcessReadVariables()
: _currentChunkLength(0), : _currentChunkLength(0),
_readBufferCursor(nullptr), _readBufferCursor(nullptr),
_cleanupLength(4096UL) {} _cleanupLength(_bufferLength / 2) {}
uint32_t uint32_t
_currentChunkLength; // size of chunk processed or 0 when expecting _currentChunkLength; // size of chunk processed or 0 when expecting
// new chunk // new chunk

47
arangod/V8Server/v8-actions.cpp
View File

@ -521,7 +521,8 @@ static v8::Handle<v8::Object> RequestCppToV8(v8::Isolate* isolate,
TRI_GET_GLOBAL_STRING(ParametersKey); TRI_GET_GLOBAL_STRING(ParametersKey);
req->ForceSet(ParametersKey, valuesObject); req->ForceSet(ParametersKey, valuesObject);
// copy cookies // copy cookie -- only for http protocl
if (request->transportType() == Endpoint::TransportType::HTTP) { // FIXME
v8::Handle<v8::Object> cookiesObject = v8::Object::New(isolate); v8::Handle<v8::Object> cookiesObject = v8::Object::New(isolate);
HttpRequest* httpRequest = dynamic_cast<HttpRequest*>(request); HttpRequest* httpRequest = dynamic_cast<HttpRequest*>(request);
@ -536,6 +537,7 @@ static v8::Handle<v8::Object> RequestCppToV8(v8::Isolate* isolate,
} }
TRI_GET_GLOBAL_STRING(CookiesKey); TRI_GET_GLOBAL_STRING(CookiesKey);
req->ForceSet(CookiesKey, cookiesObject); req->ForceSet(CookiesKey, cookiesObject);
}
return req; return req;
} }
@ -567,7 +569,7 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
TRI_GET_GLOBAL_STRING(ContentTypeKey); TRI_GET_GLOBAL_STRING(ContentTypeKey);
if (res->Has(ContentTypeKey)) { if (res->Has(ContentTypeKey)) {
contentType = TRI_ObjectToString(res->Get(ContentTypeKey)); contentType = TRI_ObjectToString(res->Get(ContentTypeKey));
if (contentType != "application/json") { if (contentType.find("application/json") == std::string::npos) {
jsonContent = false; jsonContent = false;
} }
switch (response->transportType()) { switch (response->transportType()) {
@ -655,8 +657,10 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
case Endpoint::TransportType::VPP: { case Endpoint::TransportType::VPP: {
VPackBuilder builder; VPackBuilder builder;
v8::Handle<v8::Value> v8_body = res->Get(BodyKey); v8::Handle<v8::Value> v8Body = res->Get(BodyKey);
// LOG(ERR) << v8Body->IsString();
std::string out; std::string out;
bool done = false;
// decode and set out // decode and set out
if (transformArray->IsArray()) { if (transformArray->IsArray()) {
@ -681,16 +685,21 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
} }
// out is not set // out is not set
if (out.empty()) { if (out.empty() && !done) {
if (jsonContent && !V8Buffer::hasInstance(isolate, v8_body)) { if (jsonContent && !V8Buffer::hasInstance(isolate, v8Body)) {
TRI_V8ToVPack(isolate, builder, v8_body, false); if (v8Body->IsString()) {
out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved
} else {
TRI_V8ToVPack(isolate, builder, v8Body, false);
done = true;
}
// done // done
} else if (V8Buffer::hasInstance( } else if (V8Buffer::hasInstance(
isolate, isolate,
v8_body)) { // body form buffer - could v8Body)) { // body form buffer - could
// contain json or not // contain json or not
// REVIEW (fc) - is this correct? // REVIEW (fc) - is this correct?
auto obj = v8_body.As<v8::Object>(); auto obj = v8Body.As<v8::Object>();
out = std::string(V8Buffer::data(obj), V8Buffer::length(obj)); out = std::string(V8Buffer::data(obj), V8Buffer::length(obj));
} else { // body is text - does not contain json } else { // body is text - does not contain json
out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved
@ -708,11 +717,14 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
} catch (...) { // do nothing } catch (...) { // do nothing
// json could not be converted // json could not be converted
// there was no json - change content type? // there was no json - change content type?
LOG_TOPIC(DEBUG, Logger::COMMUNICATION)
<< "failed to parse json:\n" << out;
} }
} }
if (!gotJson) { if (!gotJson) {
builder.add(VPackValue(out)); // add test to the builder builder.add(VPackValue(
out)); // add output to the builder - when not added via parser
} }
} }
@ -829,13 +841,10 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
static TRI_action_result_t ExecuteActionVocbase( static TRI_action_result_t ExecuteActionVocbase(
TRI_vocbase_t* vocbase, v8::Isolate* isolate, TRI_action_t const* action, TRI_vocbase_t* vocbase, v8::Isolate* isolate, TRI_action_t const* action,
v8::Handle<v8::Function> callback, GeneralRequest* request, v8::Handle<v8::Function> callback, GeneralRequest* request,
GeneralResponse* responseGeneral) { GeneralResponse* response) {
v8::HandleScope scope(isolate); v8::HandleScope scope(isolate);
v8::TryCatch tryCatch; v8::TryCatch tryCatch;
// TODO needs to generalized
auto response = dynamic_cast<HttpResponse*>(responseGeneral);
if (response == nullptr) { if (response == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL); THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
} }
@ -911,7 +920,11 @@ static TRI_action_result_t ExecuteActionVocbase(
errorMessage = TRI_errno_string(errorCode); errorMessage = TRI_errno_string(errorCode);
} }
response->body().appendText(errorMessage); // TODO (obi)
if (response->transportType() == Endpoint::TransportType::HTTP) { // FIXME
((HttpResponse*)response)->body().appendText(errorMessage);
}
} }
else if (v8g->_canceled) { else if (v8g->_canceled) {
@ -924,7 +937,11 @@ static TRI_action_result_t ExecuteActionVocbase(
response->setResponseCode(GeneralResponse::ResponseCode::SERVER_ERROR); response->setResponseCode(GeneralResponse::ResponseCode::SERVER_ERROR);
// TODO how to generalize this? // TODO how to generalize this?
response->body().appendText(TRI_StringifyV8Exception(isolate, &tryCatch)); if (response->transportType() ==
Endpoint::TransportType::HTTP) { // FIXME
((HttpResponse*)response)->body().appendText(
TRI_StringifyV8Exception(isolate, &tryCatch));
}
} else { } else {
v8g->_canceled = true; v8g->_canceled = true;
result.isValid = false; result.isValid = false;

186
lib/Basics/VelocyPackHelper.cpp
View File

@ -42,7 +42,8 @@
#include <velocypack/velocypack-aliases.h> #include <velocypack/velocypack-aliases.h>
extern "C" { extern "C" {
unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); unsigned long long XXH64(const void* input, size_t length,
unsigned long long seed);
} }
using VelocyPackHelper = arangodb::basics::VelocyPackHelper; using VelocyPackHelper = arangodb::basics::VelocyPackHelper;
@ -56,16 +57,21 @@ struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler {
VPackValueLength keyLength; VPackValueLength keyLength;
char const* p = key.getString(keyLength); char const* p = key.getString(keyLength);
if (p == nullptr || *p != '_' || keyLength < 3 || keyLength > 5 || nesting > 0) { if (p == nullptr || *p != '_' || keyLength < 3 || keyLength > 5 ||
nesting > 0) {
// keep attribute // keep attribute
return true; return true;
} }
// exclude these attributes (but not _key!) // exclude these attributes (but not _key!)
if ((keyLength == 3 && memcmp(p, "_id", static_cast<size_t>(keyLength)) == 0) || if ((keyLength == 3 &&
(keyLength == 4 && memcmp(p, "_rev", static_cast<size_t>(keyLength)) == 0) || memcmp(p, "_id", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 3 && memcmp(p, "_to", static_cast<size_t>(keyLength)) == 0) || (keyLength == 4 &&
(keyLength == 5 && memcmp(p, "_from", static_cast<size_t>(keyLength)) == 0)) { memcmp(p, "_rev", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 3 &&
memcmp(p, "_to", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 5 &&
memcmp(p, "_from", static_cast<size_t>(keyLength)) == 0)) {
return true; return true;
} }
@ -95,21 +101,34 @@ void VelocyPackHelper::initialize() {
// set the attribute translator in the global options // set the attribute translator in the global options
VPackOptions::Defaults.attributeTranslator = Translator.get(); VPackOptions::Defaults.attributeTranslator = Translator.get();
VPackOptions::Defaults.unsupportedTypeBehavior = VPackOptions::ConvertUnsupportedType; VPackOptions::Defaults.unsupportedTypeBehavior =
VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set when converting to JSON for HTTP xfer VPackOptions::ConvertUnsupportedType;
VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set
// when converting to JSON for
// HTTP xfer
// run quick selfs test with the attribute translator // run quick selfs test with the attribute translator
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::KeyString)).getUInt() == KeyAttribute - AttributeBase); TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::KeyString))
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::RevString)).getUInt() == RevAttribute - AttributeBase); .getUInt() == KeyAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::IdString)).getUInt() == IdAttribute - AttributeBase); TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::RevString))
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::FromString)).getUInt() == FromAttribute - AttributeBase); .getUInt() == RevAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::ToString)).getUInt() == ToAttribute - AttributeBase); TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::IdString))
.getUInt() == IdAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::FromString))
.getUInt() == FromAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::ToString))
.getUInt() == ToAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(KeyAttribute - AttributeBase)).copyString() == StaticStrings::KeyString); TRI_ASSERT(VPackSlice(Translator->translate(KeyAttribute - AttributeBase))
TRI_ASSERT(VPackSlice(Translator->translate(RevAttribute - AttributeBase)).copyString() == StaticStrings::RevString); .copyString() == StaticStrings::KeyString);
TRI_ASSERT(VPackSlice(Translator->translate(IdAttribute - AttributeBase)).copyString() == StaticStrings::IdString); TRI_ASSERT(VPackSlice(Translator->translate(RevAttribute - AttributeBase))
TRI_ASSERT(VPackSlice(Translator->translate(FromAttribute - AttributeBase)).copyString() == StaticStrings::FromString); .copyString() == StaticStrings::RevString);
TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase)).copyString() == StaticStrings::ToString); TRI_ASSERT(VPackSlice(Translator->translate(IdAttribute - AttributeBase))
.copyString() == StaticStrings::IdString);
TRI_ASSERT(VPackSlice(Translator->translate(FromAttribute - AttributeBase))
.copyString() == StaticStrings::FromString);
TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase))
.copyString() == StaticStrings::ToString);
// initialize exclude handler for system attributes // initialize exclude handler for system attributes
ExcludeHandler.reset(new SystemAttributeExcludeHandler); ExcludeHandler.reset(new SystemAttributeExcludeHandler);
@ -127,7 +146,8 @@ void VelocyPackHelper::disableAssemblerFunctions() {
/// @brief return the (global) attribute exclude handler instance /// @brief return the (global) attribute exclude handler instance
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
arangodb::velocypack::AttributeExcludeHandler* VelocyPackHelper::getExcludeHandler() { arangodb::velocypack::AttributeExcludeHandler*
VelocyPackHelper::getExcludeHandler() {
return ExcludeHandler.get(); return ExcludeHandler.get();
} }
@ -139,15 +159,14 @@ arangodb::velocypack::AttributeTranslator* VelocyPackHelper::getTranslator() {
return Translator.get(); return Translator.get();
} }
bool VelocyPackHelper::AttributeSorterUTF8::operator()(
bool VelocyPackHelper::AttributeSorterUTF8::operator()(std::string const& l, std::string const& l, std::string const& r) const {
std::string const& r) const {
// use UTF-8-based comparison of attribute names // use UTF-8-based comparison of attribute names
return TRI_compare_utf8(l.c_str(), l.size(), r.c_str(), r.size()) < 0; return TRI_compare_utf8(l.c_str(), l.size(), r.c_str(), r.size()) < 0;
} }
bool VelocyPackHelper::AttributeSorterBinary::operator()(std::string const& l, bool VelocyPackHelper::AttributeSorterBinary::operator()(
std::string const& r) const { std::string const& l, std::string const& r) const {
// use binary comparison of attribute names // use binary comparison of attribute names
size_t cmpLength = (std::min)(l.size(), r.size()); size_t cmpLength = (std::min)(l.size(), r.size());
int res = memcmp(l.c_str(), r.c_str(), cmpLength); int res = memcmp(l.c_str(), r.c_str(), cmpLength);
@ -164,15 +183,19 @@ size_t VelocyPackHelper::VPackHash::operator()(VPackSlice const& slice) const {
return static_cast<size_t>(slice.normalizedHash()); return static_cast<size_t>(slice.normalizedHash());
}; };
size_t VelocyPackHelper::VPackStringHash::operator()(VPackSlice const& slice) const noexcept { size_t VelocyPackHelper::VPackStringHash::operator()(
VPackSlice const& slice) const noexcept {
return static_cast<size_t>(slice.hashString()); return static_cast<size_t>(slice.hashString());
}; };
bool VelocyPackHelper::VPackEqual::operator()(VPackSlice const& lhs, VPackSlice const& rhs) const { bool VelocyPackHelper::VPackEqual::operator()(VPackSlice const& lhs,
VPackSlice const& rhs) const {
return VelocyPackHelper::compare(lhs, rhs, false, _options) == 0; return VelocyPackHelper::compare(lhs, rhs, false, _options) == 0;
}; };
bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, VPackSlice const& rhs) const noexcept { bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs,
VPackSlice const& rhs) const
noexcept {
auto const lh = lhs.head(); auto const lh = lhs.head();
auto const rh = rhs.head(); auto const rh = rhs.head();
@ -183,15 +206,20 @@ bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, VPack
VPackValueLength size; VPackValueLength size;
if (lh == 0xbf) { if (lh == 0xbf) {
// long UTF-8 String // long UTF-8 String
size = static_cast<VPackValueLength>(velocypack::readInteger<VPackValueLength>(lhs.begin() + 1, 8)); size = static_cast<VPackValueLength>(
if (size !=static_cast<VPackValueLength>(velocypack::readInteger<VPackValueLength>(rhs.begin() + 1, 8))) { velocypack::readInteger<VPackValueLength>(lhs.begin() + 1, 8));
if (size !=
static_cast<VPackValueLength>(
velocypack::readInteger<VPackValueLength>(rhs.begin() + 1, 8))) {
return false; return false;
} }
return (memcmp(lhs.start() + 1 + 8, rhs.start() + 1 + 8, static_cast<size_t>(size)) == 0); return (memcmp(lhs.start() + 1 + 8, rhs.start() + 1 + 8,
static_cast<size_t>(size)) == 0);
} }
size = static_cast<VPackValueLength>(lh - 0x40); size = static_cast<VPackValueLength>(lh - 0x40);
return (memcmp(lhs.start() + 1, rhs.start() + 1, static_cast<size_t>(size)) == 0); return (memcmp(lhs.start() + 1, rhs.start() + 1, static_cast<size_t>(size)) ==
0);
}; };
static int TypeWeight(VPackSlice const& slice) { static int TypeWeight(VPackSlice const& slice) {
@ -315,14 +343,12 @@ std::string VelocyPackHelper::checkAndGetStringValue(VPackSlice const& slice,
std::string const& name) { std::string const& name) {
TRI_ASSERT(slice.isObject()); TRI_ASSERT(slice.isObject());
if (!slice.hasKey(name)) { if (!slice.hasKey(name)) {
std::string msg = std::string msg = "The attribute '" + name + "' was not found.";
"The attribute '" + name + "' was not found.";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
} }
VPackSlice const sub = slice.get(name); VPackSlice const sub = slice.get(name);
if (!sub.isString()) { if (!sub.isString()) {
std::string msg = std::string msg = "The attribute '" + name + "' is not a string.";
"The attribute '" + name + "' is not a string.";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
} }
return sub.copyString(); return sub.copyString();
@ -346,8 +372,7 @@ std::string VelocyPackHelper::getStringValue(VPackSlice const& slice,
/// or it is not a string /// or it is not a string
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
std::string VelocyPackHelper::getStringValue(VPackSlice slice, std::string VelocyPackHelper::getStringValue(VPackSlice slice, char const* name,
char const* name,
std::string const& defaultValue) { std::string const& defaultValue) {
if (slice.isExternal()) { if (slice.isExternal()) {
slice = VPackSlice(slice.getExternal()); slice = VPackSlice(slice.getExternal());
@ -396,7 +421,8 @@ uint64_t VelocyPackHelper::stringUInt64(VPackSlice const& slice) {
return 0; return 0;
} }
TRI_json_t* VelocyPackHelper::velocyPackToJson(VPackSlice const& slice, VPackOptions const* options) { TRI_json_t* VelocyPackHelper::velocyPackToJson(VPackSlice const& slice,
VPackOptions const* options) {
return JsonHelper::fromString(slice.toJson(options)); return JsonHelper::fromString(slice.toJson(options));
} }
@ -487,19 +513,22 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
TRI_UnlinkFile(tmp.c_str()); TRI_UnlinkFile(tmp.c_str());
} }
int fd = TRI_CREATE(tmp.c_str(), O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC, int fd = TRI_CREATE(tmp.c_str(),
O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
if (fd < 0) { if (fd < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot create json file '" << tmp << "': " << TRI_LAST_ERROR_STR; LOG(ERR) << "cannot create json file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
return false; return false;
} }
if (!PrintVelocyPack(fd, slice, true)) { if (!PrintVelocyPack(fd, slice, true)) {
TRI_CLOSE(fd); TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot write to json file '" << tmp << "': " << TRI_LAST_ERROR_STR; LOG(ERR) << "cannot write to json file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str()); TRI_UnlinkFile(tmp.c_str());
return false; return false;
} }
@ -510,7 +539,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
if (!TRI_fsync(fd)) { if (!TRI_fsync(fd)) {
TRI_CLOSE(fd); TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot sync saved json '" << tmp << "': " << TRI_LAST_ERROR_STR; LOG(ERR) << "cannot sync saved json '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str()); TRI_UnlinkFile(tmp.c_str());
return false; return false;
} }
@ -520,7 +550,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
if (res < 0) { if (res < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot close saved file '" << tmp << "': " << TRI_LAST_ERROR_STR; LOG(ERR) << "cannot close saved file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str()); TRI_UnlinkFile(tmp.c_str());
return false; return false;
} }
@ -529,7 +560,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res); TRI_set_errno(res);
LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename << "': " << TRI_LAST_ERROR_STR; LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str()); TRI_UnlinkFile(tmp.c_str());
return false; return false;
@ -538,9 +570,10 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
return true; return true;
} }
int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, bool useUTF8,
bool useUTF8, VPackOptions const* options, VPackOptions const* options,
VPackSlice const* lhsBase, VPackSlice const* rhsBase) { VPackSlice const* lhsBase,
VPackSlice const* rhsBase) {
{ {
// will resolve externals... // will resolve externals...
int lWeight = TypeWeight(lhs); int lWeight = TypeWeight(lhs);
@ -602,11 +635,13 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
VPackValueLength nl; VPackValueLength nl;
char const* left; char const* left;
if (lhs.isCustom()) { if (lhs.isCustom()) {
if (lhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) { if (lhsBase == nullptr || options == nullptr ||
options->customTypeHandler == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"Could not extract custom attribute."); "Could not extract custom attribute.");
} }
lhsString.assign(options->customTypeHandler->toString(lhs, options, *lhsBase)); lhsString.assign(
options->customTypeHandler->toString(lhs, options, *lhsBase));
left = lhsString.c_str(); left = lhsString.c_str();
nl = lhsString.size(); nl = lhsString.size();
} else { } else {
@ -618,11 +653,13 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
VPackValueLength nr; VPackValueLength nr;
char const* right; char const* right;
if (rhs.isCustom()) { if (rhs.isCustom()) {
if (rhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) { if (rhsBase == nullptr || options == nullptr ||
options->customTypeHandler == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"Could not extract custom attribute."); "Could not extract custom attribute.");
} }
rhsString.assign(options->customTypeHandler->toString(rhs, options, *rhsBase)); rhsString.assign(
options->customTypeHandler->toString(rhs, options, *rhsBase));
right = rhsString.c_str(); right = rhsString.c_str();
nr = rhsString.size(); nr = rhsString.size();
} else { } else {
@ -632,7 +669,8 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
int res; int res;
if (useUTF8) { if (useUTF8) {
res = TRI_compare_utf8(left, static_cast<size_t>(nl), right, static_cast<size_t>(nr)); res = TRI_compare_utf8(left, static_cast<size_t>(nl), right,
static_cast<size_t>(nr));
} else { } else {
size_t len = static_cast<size_t>(nl < nr ? nl : nr); size_t len = static_cast<size_t>(nl < nr ? nl : nr);
res = memcmp(left, right, len); res = memcmp(left, right, len);
@ -743,7 +781,8 @@ double VelocyPackHelper::toDouble(VPackSlice const& slice, bool& failed) {
if (n == 0) { if (n == 0) {
return 0.0; return 0.0;
} else if (n == 1) { } else if (n == 1) {
return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(), failed); return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(),
failed);
} }
break; break;
} }
@ -793,7 +832,8 @@ uint64_t VelocyPackHelper::hashByAttributes(
return hash; return hash;
} }
void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& output) { void VelocyPackHelper::SanitizeExternals(VPackSlice const input,
VPackBuilder& output) {
if (input.isExternal()) { if (input.isExternal()) {
output.add(input.resolveExternal()); output.add(input.resolveExternal());
} else if (input.isObject()) { } else if (input.isObject()) {
@ -814,7 +854,43 @@ void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& o
} }
} }
arangodb::LoggerStream& operator<< (arangodb::LoggerStream& logger, bool VelocyPackHelper::hasExternals(VPackSlice input) {
if (input.isExternal()) {
return true;
} else if (input.isObject()) {
for (auto const& it : VPackObjectIterator(input)) {
if (hasExternals(it.value) == true) {
return true;
};
}
} else if (input.isArray()) {
for (auto const& it : VPackArrayIterator(input)) {
if (hasExternals(it) == true) {
return true;
}
}
}
return false;
}
VPackBuilder VelocyPackHelper::stanitizeExternalsChecked(VPackSlice input,
bool checkExternals) {
VPackBuilder builder;
bool resolveExt = true;
if (checkExternals) {
resolveExt = hasExternals(input);
}
if (resolveExt) { // resolve
SanitizeExternals(input, builder);
} else {
builder.add(input);
}
return builder; // elided
}
arangodb::LoggerStream& operator<<(arangodb::LoggerStream& logger,
VPackSlice const& slice) { VPackSlice const& slice) {
size_t const cutoff = 100; size_t const cutoff = 100;
std::string sliceStr(slice.toJson()); std::string sliceStr(slice.toJson());

47
lib/Basics/VelocyPackHelper.h
View File

@ -47,10 +47,9 @@ class VelocyPackHelper {
~VelocyPackHelper() = delete; ~VelocyPackHelper() = delete;
public: public:
////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////// /// @brief static initializer for all VPack values
/// @brief static initializer for all VPack values ////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
static void initialize(); static void initialize();
static void disableAssemblerFunctions(); static void disableAssemblerFunctions();
@ -66,13 +65,14 @@ class VelocyPackHelper {
size_t operator()(arangodb::velocypack::Slice const&) const noexcept; size_t operator()(arangodb::velocypack::Slice const&) const noexcept;
}; };
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/// @brief equality comparator for VelocyPack values /// @brief equality comparator for VelocyPack values
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
struct VPackEqual { struct VPackEqual {
private: private:
arangodb::velocypack::Options const* _options; arangodb::velocypack::Options const* _options;
public: public:
VPackEqual() : _options(nullptr) {} VPackEqual() : _options(nullptr) {}
explicit VPackEqual(arangodb::velocypack::Options const* opts) explicit VPackEqual(arangodb::velocypack::Options const* opts)
@ -87,9 +87,9 @@ class VelocyPackHelper {
arangodb::velocypack::Slice const&) const noexcept; arangodb::velocypack::Slice const&) const noexcept;
}; };
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/// @brief less comparator for VelocyPack values /// @brief less comparator for VelocyPack values
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
template <bool useUtf8> template <bool useUtf8>
struct VPackLess { struct VPackLess {
@ -207,7 +207,8 @@ class VelocyPackHelper {
static std::string checkAndGetStringValue(VPackSlice const&, char const*); static std::string checkAndGetStringValue(VPackSlice const&, char const*);
static std::string checkAndGetStringValue(VPackSlice const&, std::string const&); static std::string checkAndGetStringValue(VPackSlice const&,
std::string const&);
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
/// @brief returns a Numeric sub-element, or throws if <name> does not exist /// @brief returns a Numeric sub-element, or throws if <name> does not exist
@ -243,8 +244,10 @@ class VelocyPackHelper {
/// or it is not a string /// or it is not a string
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
static std::string getStringValue(VPackSlice, char const*, std::string const&); static std::string getStringValue(VPackSlice, char const*,
static std::string getStringValue(VPackSlice, std::string const&, std::string const&); std::string const&);
static std::string getStringValue(VPackSlice, std::string const&,
std::string const&);
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
/// @brief convert a Object sub value into a uint64 /// @brief convert a Object sub value into a uint64
@ -256,7 +259,8 @@ class VelocyPackHelper {
/// @brief Build TRI_json_t from VelocyPack. Just a temporary solution /// @brief Build TRI_json_t from VelocyPack. Just a temporary solution
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
static TRI_json_t* velocyPackToJson(VPackSlice const&, VPackOptions const* = &VPackOptions::Defaults); static TRI_json_t* velocyPackToJson(
VPackSlice const&, VPackOptions const* = &VPackOptions::Defaults);
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
/// @brief parses a json file to VelocyPack /// @brief parses a json file to VelocyPack
@ -284,8 +288,9 @@ class VelocyPackHelper {
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
static int compare(arangodb::velocypack::Slice lhs, static int compare(arangodb::velocypack::Slice lhs,
arangodb::velocypack::Slice rhs, arangodb::velocypack::Slice rhs, bool useUTF8,
bool useUTF8, arangodb::velocypack::Options const* options = &arangodb::velocypack::Options::Defaults, arangodb::velocypack::Options const* options =
&arangodb::velocypack::Options::Defaults,
arangodb::velocypack::Slice const* lhsBase = nullptr, arangodb::velocypack::Slice const* lhsBase = nullptr,
arangodb::velocypack::Slice const* rhsBase = nullptr); arangodb::velocypack::Slice const* rhsBase = nullptr);
@ -350,6 +355,10 @@ class VelocyPackHelper {
static void SanitizeExternals(arangodb::velocypack::Slice const, static void SanitizeExternals(arangodb::velocypack::Slice const,
arangodb::velocypack::Builder&); arangodb::velocypack::Builder&);
static bool hasExternals(arangodb::velocypack::Slice const);
static VPackBuilder stanitizeExternalsChecked(
arangodb::velocypack::Slice const, bool checkExternals = true);
static uint8_t const KeyAttribute = 0x31; static uint8_t const KeyAttribute = 0x31;
static uint8_t const RevAttribute = 0x32; static uint8_t const RevAttribute = 0x32;
static uint8_t const IdAttribute = 0x33; static uint8_t const IdAttribute = 0x33;
@ -358,10 +367,12 @@ class VelocyPackHelper {
static uint8_t const AttributeBase = 0x30; static uint8_t const AttributeBase = 0x30;
static_assert(KeyAttribute < RevAttribute, "invalid value for _key attribute"); static_assert(KeyAttribute < RevAttribute,
"invalid value for _key attribute");
static_assert(RevAttribute < IdAttribute, "invalid value for _rev attribute"); static_assert(RevAttribute < IdAttribute, "invalid value for _rev attribute");
static_assert(IdAttribute < FromAttribute, "invalid value for _id attribute"); static_assert(IdAttribute < FromAttribute, "invalid value for _id attribute");
static_assert(FromAttribute < ToAttribute, "invalid value for _from attribute"); static_assert(FromAttribute < ToAttribute,
"invalid value for _from attribute");
}; };
} }
} }

2
lib/Logger/LogTopic.cpp
View File

@ -39,7 +39,7 @@ std::map<std::string, LogTopic*> LogTopic::_names;
LogTopic Logger::AGENCY("agency", LogLevel::INFO); LogTopic Logger::AGENCY("agency", LogLevel::INFO);
LogTopic Logger::AGENCYCOMM("agencycomm", LogLevel::INFO); LogTopic Logger::AGENCYCOMM("agencycomm", LogLevel::INFO);
LogTopic Logger::COLLECTOR("collector"); LogTopic Logger::COLLECTOR("collector");
LogTopic Logger::COMMUNICATION("COMMUNICATION", LogLevel::INFO); LogTopic Logger::COMMUNICATION("communication", LogLevel::INFO);
LogTopic Logger::COMPACTOR("compactor"); LogTopic Logger::COMPACTOR("compactor");
LogTopic Logger::CONFIG("config"); LogTopic Logger::CONFIG("config");
LogTopic Logger::DATAFILES("datafiles", LogLevel::INFO); LogTopic Logger::DATAFILES("datafiles", LogLevel::INFO);

1
lib/Rest/GeneralRequest.h
View File

@ -165,6 +165,7 @@ class GeneralRequest {
std::vector<std::string> const& suffix() const { return _suffix; } std::vector<std::string> const& suffix() const { return _suffix; }
void addSuffix(std::string&& part); void addSuffix(std::string&& part);
virtual arangodb::Endpoint::TransportType transportType() = 0;
virtual int64_t contentLength() const = 0; virtual int64_t contentLength() const = 0;
// get value from headers map. The key must be lowercase. // get value from headers map. The key must be lowercase.
virtual std::string const& header(std::string const& key) const = 0; virtual std::string const& header(std::string const& key) const = 0;

3
lib/Rest/HttpRequest.h
View File

@ -70,6 +70,9 @@ class HttpRequest : public GeneralRequest {
bool isHttp11() const { return _version == ProtocolVersion::HTTP_1_1; } bool isHttp11() const { return _version == ProtocolVersion::HTTP_1_1; }
public: public:
arangodb::Endpoint::TransportType transportType() override {
return arangodb::Endpoint::TransportType::HTTP;
}
// the content length // the content length
int64_t contentLength() const override { return _contentLength; } int64_t contentLength() const override { return _contentLength; }

4
lib/Rest/VppRequest.h
View File

@ -71,6 +71,10 @@ class VppRequest : public GeneralRequest {
return _message._payload.byteSize(); return _message._payload.byteSize();
} }
virtual arangodb::Endpoint::TransportType transportType() override {
return arangodb::Endpoint::TransportType::VPP;
};
std::unordered_map<std::string, std::string> const& headers() const override; std::unordered_map<std::string, std::string> const& headers() const override;
// get value from headers map. The key must be lowercase. // get value from headers map. The key must be lowercase.
std::string const& header(std::string const& key) const override; std::string const& header(std::string const& key) const override;