Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'obi-velocystream' into engine-vs-velocystream 

* obi-velocystream:
  fix buffer clean up
  avoid bug that occurred during buffer resize by allocating up front
  i fix type of execepton so it gets caught
  avoid dumper exception
  failed try to resolve externals
  add sanitizeExternalsChecked function
  added handleSimpleError for database not found
  fix build issue
  silence complier warnings
  fix: now returns object instead of string
  try to fix simple route
This commit is contained in:
Jan Christoph Uhde 2016-08-16 11:12:23 +02:00
parent 000fd9cbdc 9afe3a1016
commit d8d6fb1ed0
10 changed files with 284 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -125,7 +125,7 @@ configure_file(
find_program (GIT_EXE git)
set (EXECUTE_COMMAND ${GIT_EXE} describe --all --tags --long --dirty=-dirty)
execute_process(COMMAND ${EXECUTE_COMMAND} OUTPUT_VARIABLE GIT_OUTPUT)
execute_process(COMMAND ${EXECUTE_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE GIT_OUTPUT)
string(STRIP ${GIT_OUTPUT} REPOSITORY_VERSION)
set(ARANGODB_BUILD_REPOSITORY ${REPOSITORY_VERSION})

63
arangod/GeneralServer/VppCommTask.cpp
View File

@ -25,6 +25,7 @@
#include "Basics/StringBuffer.h"
#include "Basics/HybridLogicalClock.h"
#include "Basics/VelocyPackHelper.h"
#include "GeneralServer/GeneralServer.h"
#include "GeneralServer/GeneralServerFeature.h"
#include "GeneralServer/RestHandler.h"
@ -137,18 +138,20 @@ std::unique_ptr<basics::StringBuffer> createChunkForNetworkSingle(
return createChunkForNetworkDetail(slices, true, 1, id, 0 /*unused*/);
}
std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFirst(
std::vector<VPackSlice> const& slices, uint64_t id, uint32_t numberOfChunks,
uint32_t totalMessageLength) {
return createChunkForNetworkDetail(slices, true, numberOfChunks, id,
totalMessageLength);
}
std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFollow(
std::vector<VPackSlice> const& slices, uint64_t id, uint32_t chunkNumber,
uint32_t totalMessageLength) {
return createChunkForNetworkDetail(slices, false, chunkNumber, id, 0);
}
// TODO FIXME make use of these functions
// std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFirst(
// std::vector<VPackSlice> const& slices, uint64_t id, uint32_t
// numberOfChunks,
// uint32_t totalMessageLength) {
// return createChunkForNetworkDetail(slices, true, numberOfChunks, id,
// totalMessageLength);
// }
//
// std::unique_ptr<basics::StringBuffer> createChunkForNetworkMultiFollow(
// std::vector<VPackSlice> const& slices, uint64_t id, uint32_t chunkNumber,
// uint32_t totalMessageLength) {
// return createChunkForNetworkDetail(slices, false, chunkNumber, id, 0);
// }
}
VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock,
@ -156,7 +159,10 @@ VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock,
: Task("VppCommTask"),
GeneralCommTask(server, sock, std::move(info), timeout) {
_protocol = "vpp";
// connectionStatisticsAgentSetVpp();
_readBuffer->reserve(
_bufferLength); // ATTENTION <- this is required so we do not
// loose information during a resize
// connectionStatisticsAgentSetVpp();
}
void VppCommTask::addResponse(VppResponse* response, bool isError) {
@ -172,17 +178,24 @@ void VppCommTask::addResponse(VppResponse* response, bool isError) {
std::vector<VPackSlice> slices;
slices.push_back(response_message._header);
VPackBuilder builder;
if (response_message._generateBody) {
slices.push_back(response_message._payload);
builder = basics::VelocyPackHelper::stanitizeExternalsChecked(
response_message._payload);
slices.push_back(builder.slice());
}
// FIXME (obi)
// If the message is big we will create many small chunks in a loop.
// For the first tests we just send single Messages
LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << "got request:";
LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << "got response:";
for (auto const& slice : slices) {
LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << slice.toJson();
try {
LOG_TOPIC(DEBUG, Logger::COMMUNICATION) << slice.toJson();
} catch (arangodb::velocypack::Exception const& e) {
std::cout << "obi exception" << e.what();
}
}
// adds chunk header infromation and creates SingBuffer* that can be
@ -345,7 +358,7 @@ bool VppCommTask::processRead() {
std::size_t processedDataLen =
std::distance(_readBuffer->begin(), prv._readBufferCursor);
if (processedDataLen > prv._cleanupLength) {
_readBuffer->move_front(prv._cleanupLength);
_readBuffer->move_front(processedDataLen);
prv._readBufferCursor = nullptr; // the positon will be set at the
// begin of this function
}
@ -358,11 +371,17 @@ bool VppCommTask::processRead() {
<< "got request:" << message._header.toJson();
_request = new VppRequest(_connectionInfo, std::move(message));
GeneralServerFeature::HANDLER_FACTORY->setRequestContext(_request);
_request->setClientTaskId(_taskId);
_protocolVersion = _request->protocolVersion();
executeRequest(_request,
new VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR,
chunkHeader._messageID));
if (_request->requestContext() == nullptr) {
handleSimpleError(GeneralResponse::ResponseCode::NOT_FOUND,
TRI_ERROR_ARANGO_DATABASE_NOT_FOUND,
TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND));
} else {
_request->setClientTaskId(_taskId);
_protocolVersion = _request->protocolVersion();
executeRequest(
_request, new VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR,
chunkHeader._messageID));
}
}
if (read_maybe_only_part_of_buffer) {

3
arangod/GeneralServer/VppCommTask.h
View File

@ -83,11 +83,12 @@ class VppCommTask : public GeneralCommTask {
};
std::unordered_map<MessageID, IncompleteVPackMessage> _incompleteMessages;
static size_t const _bufferLength = 4096UL;
struct ProcessReadVariables {
ProcessReadVariables()
: _currentChunkLength(0),
_readBufferCursor(nullptr),
_cleanupLength(4096UL) {}
_cleanupLength(_bufferLength / 2) {}
uint32_t
_currentChunkLength; // size of chunk processed or 0 when expecting
// new chunk

71
arangod/V8Server/v8-actions.cpp
View File

@ -521,21 +521,23 @@ static v8::Handle<v8::Object> RequestCppToV8(v8::Isolate* isolate,
TRI_GET_GLOBAL_STRING(ParametersKey);
req->ForceSet(ParametersKey, valuesObject);
// copy cookies
v8::Handle<v8::Object> cookiesObject = v8::Object::New(isolate);
// copy cookie -- only for http protocl
if (request->transportType() == Endpoint::TransportType::HTTP) { // FIXME
v8::Handle<v8::Object> cookiesObject = v8::Object::New(isolate);
HttpRequest* httpRequest = dynamic_cast<HttpRequest*>(request);
if (httpRequest == nullptr) {
// maybe we can just continue
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
} else {
for (auto& it : httpRequest->cookieValues()) {
cookiesObject->ForceSet(TRI_V8_STD_STRING(it.first),
TRI_V8_STD_STRING(it.second));
HttpRequest* httpRequest = dynamic_cast<HttpRequest*>(request);
if (httpRequest == nullptr) {
// maybe we can just continue
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
} else {
for (auto& it : httpRequest->cookieValues()) {
cookiesObject->ForceSet(TRI_V8_STD_STRING(it.first),
TRI_V8_STD_STRING(it.second));
}
}
TRI_GET_GLOBAL_STRING(CookiesKey);
req->ForceSet(CookiesKey, cookiesObject);
}
TRI_GET_GLOBAL_STRING(CookiesKey);
req->ForceSet(CookiesKey, cookiesObject);
return req;
}
@ -567,7 +569,7 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
TRI_GET_GLOBAL_STRING(ContentTypeKey);
if (res->Has(ContentTypeKey)) {
contentType = TRI_ObjectToString(res->Get(ContentTypeKey));
if (contentType != "application/json") {
if (contentType.find("application/json") == std::string::npos) {
jsonContent = false;
}
switch (response->transportType()) {
@ -655,8 +657,10 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
case Endpoint::TransportType::VPP: {
VPackBuilder builder;
v8::Handle<v8::Value> v8_body = res->Get(BodyKey);
v8::Handle<v8::Value> v8Body = res->Get(BodyKey);
// LOG(ERR) << v8Body->IsString();
std::string out;
bool done = false;
// decode and set out
if (transformArray->IsArray()) {
@ -681,16 +685,21 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
}
// out is not set
if (out.empty()) {
if (jsonContent && !V8Buffer::hasInstance(isolate, v8_body)) {
TRI_V8ToVPack(isolate, builder, v8_body, false);
if (out.empty() && !done) {
if (jsonContent && !V8Buffer::hasInstance(isolate, v8Body)) {
if (v8Body->IsString()) {
out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved
} else {
TRI_V8ToVPack(isolate, builder, v8Body, false);
done = true;
}
// done
} else if (V8Buffer::hasInstance(
isolate,
v8_body)) { // body form buffer - could
// contain json or not
v8Body)) { // body form buffer - could
// contain json or not
// REVIEW (fc) - is this correct?
auto obj = v8_body.As<v8::Object>();
auto obj = v8Body.As<v8::Object>();
out = std::string(V8Buffer::data(obj), V8Buffer::length(obj));
} else { // body is text - does not contain json
out = TRI_ObjectToString(res->Get(BodyKey)); // should get moved
@ -708,11 +717,14 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
} catch (...) { // do nothing
// json could not be converted
// there was no json - change content type?
LOG_TOPIC(DEBUG, Logger::COMMUNICATION)
<< "failed to parse json:\n" << out;
}
}
if (!gotJson) {
builder.add(VPackValue(out)); // add test to the builder
builder.add(VPackValue(
out)); // add output to the builder - when not added via parser
}
}
@ -829,13 +841,10 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
static TRI_action_result_t ExecuteActionVocbase(
TRI_vocbase_t* vocbase, v8::Isolate* isolate, TRI_action_t const* action,
v8::Handle<v8::Function> callback, GeneralRequest* request,
GeneralResponse* responseGeneral) {
GeneralResponse* response) {
v8::HandleScope scope(isolate);
v8::TryCatch tryCatch;
// TODO needs to generalized
auto response = dynamic_cast<HttpResponse*>(responseGeneral);
if (response == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
}
@ -911,7 +920,11 @@ static TRI_action_result_t ExecuteActionVocbase(
errorMessage = TRI_errno_string(errorCode);
}
response->body().appendText(errorMessage);
// TODO (obi)
if (response->transportType() == Endpoint::TransportType::HTTP) { // FIXME
((HttpResponse*)response)->body().appendText(errorMessage);
}
}
else if (v8g->_canceled) {
@ -924,7 +937,11 @@ static TRI_action_result_t ExecuteActionVocbase(
response->setResponseCode(GeneralResponse::ResponseCode::SERVER_ERROR);
// TODO how to generalize this?
response->body().appendText(TRI_StringifyV8Exception(isolate, &tryCatch));
if (response->transportType() ==
Endpoint::TransportType::HTTP) { // FIXME
((HttpResponse*)response)->body().appendText(
TRI_StringifyV8Exception(isolate, &tryCatch));
}
} else {
v8g->_canceled = true;
result.isValid = false;

214
lib/Basics/VelocyPackHelper.cpp
View File

@ -42,7 +42,8 @@
#include <velocypack/velocypack-aliases.h>
extern "C" {
unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
unsigned long long XXH64(const void* input, size_t length,
unsigned long long seed);
}
using VelocyPackHelper = arangodb::basics::VelocyPackHelper;
@ -56,16 +57,21 @@ struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler {
VPackValueLength keyLength;
char const* p = key.getString(keyLength);
if (p == nullptr || *p != '_' || keyLength < 3 || keyLength > 5 || nesting > 0) {
if (p == nullptr || *p != '_' || keyLength < 3 || keyLength > 5 ||
nesting > 0) {
// keep attribute
return true;
}
// exclude these attributes (but not _key!)
if ((keyLength == 3 && memcmp(p, "_id", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 4 && memcmp(p, "_rev", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 3 && memcmp(p, "_to", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 5 && memcmp(p, "_from", static_cast<size_t>(keyLength)) == 0)) {
if ((keyLength == 3 &&
memcmp(p, "_id", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 4 &&
memcmp(p, "_rev", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 3 &&
memcmp(p, "_to", static_cast<size_t>(keyLength)) == 0) ||
(keyLength == 5 &&
memcmp(p, "_from", static_cast<size_t>(keyLength)) == 0)) {
return true;
}
@ -85,9 +91,9 @@ void VelocyPackHelper::initialize() {
Translator.reset(new VPackAttributeTranslator);
// these attribute names will be translated into short integer values
Translator->add(StaticStrings::KeyString, KeyAttribute - AttributeBase);
Translator->add(StaticStrings::KeyString, KeyAttribute - AttributeBase);
Translator->add(StaticStrings::RevString, RevAttribute - AttributeBase);
Translator->add(StaticStrings::IdString, IdAttribute - AttributeBase);
Translator->add(StaticStrings::IdString, IdAttribute - AttributeBase);
Translator->add(StaticStrings::FromString, FromAttribute - AttributeBase);
Translator->add(StaticStrings::ToString, ToAttribute - AttributeBase);
@ -95,21 +101,34 @@ void VelocyPackHelper::initialize() {
// set the attribute translator in the global options
VPackOptions::Defaults.attributeTranslator = Translator.get();
VPackOptions::Defaults.unsupportedTypeBehavior = VPackOptions::ConvertUnsupportedType;
VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set when converting to JSON for HTTP xfer
VPackOptions::Defaults.unsupportedTypeBehavior =
VPackOptions::ConvertUnsupportedType;
VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set
// when converting to JSON for
// HTTP xfer
// run quick selfs test with the attribute translator
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::KeyString)).getUInt() == KeyAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::RevString)).getUInt() == RevAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::IdString)).getUInt() == IdAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::FromString)).getUInt() == FromAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::ToString)).getUInt() == ToAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(KeyAttribute - AttributeBase)).copyString() == StaticStrings::KeyString);
TRI_ASSERT(VPackSlice(Translator->translate(RevAttribute - AttributeBase)).copyString() == StaticStrings::RevString);
TRI_ASSERT(VPackSlice(Translator->translate(IdAttribute - AttributeBase)).copyString() == StaticStrings::IdString);
TRI_ASSERT(VPackSlice(Translator->translate(FromAttribute - AttributeBase)).copyString() == StaticStrings::FromString);
TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase)).copyString() == StaticStrings::ToString);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::KeyString))
.getUInt() == KeyAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::RevString))
.getUInt() == RevAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::IdString))
.getUInt() == IdAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::FromString))
.getUInt() == FromAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(StaticStrings::ToString))
.getUInt() == ToAttribute - AttributeBase);
TRI_ASSERT(VPackSlice(Translator->translate(KeyAttribute - AttributeBase))
.copyString() == StaticStrings::KeyString);
TRI_ASSERT(VPackSlice(Translator->translate(RevAttribute - AttributeBase))
.copyString() == StaticStrings::RevString);
TRI_ASSERT(VPackSlice(Translator->translate(IdAttribute - AttributeBase))
.copyString() == StaticStrings::IdString);
TRI_ASSERT(VPackSlice(Translator->translate(FromAttribute - AttributeBase))
.copyString() == StaticStrings::FromString);
TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase))
.copyString() == StaticStrings::ToString);
// initialize exclude handler for system attributes
ExcludeHandler.reset(new SystemAttributeExcludeHandler);
@ -127,7 +146,8 @@ void VelocyPackHelper::disableAssemblerFunctions() {
/// @brief return the (global) attribute exclude handler instance
////////////////////////////////////////////////////////////////////////////////
arangodb::velocypack::AttributeExcludeHandler* VelocyPackHelper::getExcludeHandler() {
arangodb::velocypack::AttributeExcludeHandler*
VelocyPackHelper::getExcludeHandler() {
return ExcludeHandler.get();
}
@ -139,15 +159,14 @@ arangodb::velocypack::AttributeTranslator* VelocyPackHelper::getTranslator() {
return Translator.get();
}
bool VelocyPackHelper::AttributeSorterUTF8::operator()(std::string const& l,
std::string const& r) const {
bool VelocyPackHelper::AttributeSorterUTF8::operator()(
std::string const& l, std::string const& r) const {
// use UTF-8-based comparison of attribute names
return TRI_compare_utf8(l.c_str(), l.size(), r.c_str(), r.size()) < 0;
}
bool VelocyPackHelper::AttributeSorterBinary::operator()(std::string const& l,
std::string const& r) const {
bool VelocyPackHelper::AttributeSorterBinary::operator()(
std::string const& l, std::string const& r) const {
// use binary comparison of attribute names
size_t cmpLength = (std::min)(l.size(), r.size());
int res = memcmp(l.c_str(), r.c_str(), cmpLength);
@ -164,15 +183,19 @@ size_t VelocyPackHelper::VPackHash::operator()(VPackSlice const& slice) const {
return static_cast<size_t>(slice.normalizedHash());
};
size_t VelocyPackHelper::VPackStringHash::operator()(VPackSlice const& slice) const noexcept {
size_t VelocyPackHelper::VPackStringHash::operator()(
VPackSlice const& slice) const noexcept {
return static_cast<size_t>(slice.hashString());
};
bool VelocyPackHelper::VPackEqual::operator()(VPackSlice const& lhs, VPackSlice const& rhs) const {
bool VelocyPackHelper::VPackEqual::operator()(VPackSlice const& lhs,
VPackSlice const& rhs) const {
return VelocyPackHelper::compare(lhs, rhs, false, _options) == 0;
};
bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, VPackSlice const& rhs) const noexcept {
bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs,
VPackSlice const& rhs) const
noexcept {
auto const lh = lhs.head();
auto const rh = rhs.head();
@ -183,21 +206,26 @@ bool VelocyPackHelper::VPackStringEqual::operator()(VPackSlice const& lhs, VPack
VPackValueLength size;
if (lh == 0xbf) {
// long UTF-8 String
size = static_cast<VPackValueLength>(velocypack::readInteger<VPackValueLength>(lhs.begin() + 1, 8));
if (size !=static_cast<VPackValueLength>(velocypack::readInteger<VPackValueLength>(rhs.begin() + 1, 8))) {
size = static_cast<VPackValueLength>(
velocypack::readInteger<VPackValueLength>(lhs.begin() + 1, 8));
if (size !=
static_cast<VPackValueLength>(
velocypack::readInteger<VPackValueLength>(rhs.begin() + 1, 8))) {
return false;
}
return (memcmp(lhs.start() + 1 + 8, rhs.start() + 1 + 8, static_cast<size_t>(size)) == 0);
}
return (memcmp(lhs.start() + 1 + 8, rhs.start() + 1 + 8,
static_cast<size_t>(size)) == 0);
}
size = static_cast<VPackValueLength>(lh - 0x40);
return (memcmp(lhs.start() + 1, rhs.start() + 1, static_cast<size_t>(size)) == 0);
return (memcmp(lhs.start() + 1, rhs.start() + 1, static_cast<size_t>(size)) ==
0);
};
static int TypeWeight(VPackSlice const& slice) {
switch (slice.type()) {
case VPackValueType::MinKey:
return -99; // must be lowest
return -99; // must be lowest
case VPackValueType::Illegal:
return -1;
case VPackValueType::None:
@ -224,14 +252,14 @@ static int TypeWeight(VPackSlice const& slice) {
case VPackValueType::External:
return TypeWeight(slice.resolveExternal());
case VPackValueType::MaxKey:
return 99; // must be highest
return 99; // must be highest
default:
// All other values have equal weight
return 0;
}
}
int VelocyPackHelper::compareNumberValues(VPackValueType lhsType,
int VelocyPackHelper::compareNumberValues(VPackValueType lhsType,
VPackSlice lhs, VPackSlice rhs) {
if (lhsType == rhs.type()) {
// both types are equal
@ -315,14 +343,12 @@ std::string VelocyPackHelper::checkAndGetStringValue(VPackSlice const& slice,
std::string const& name) {
TRI_ASSERT(slice.isObject());
if (!slice.hasKey(name)) {
std::string msg =
"The attribute '" + name + "' was not found.";
std::string msg = "The attribute '" + name + "' was not found.";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
}
VPackSlice const sub = slice.get(name);
if (!sub.isString()) {
std::string msg =
"The attribute '" + name + "' is not a string.";
std::string msg = "The attribute '" + name + "' is not a string.";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
}
return sub.copyString();
@ -346,8 +372,7 @@ std::string VelocyPackHelper::getStringValue(VPackSlice const& slice,
/// or it is not a string
////////////////////////////////////////////////////////////////////////////////
std::string VelocyPackHelper::getStringValue(VPackSlice slice,
char const* name,
std::string VelocyPackHelper::getStringValue(VPackSlice slice, char const* name,
std::string const& defaultValue) {
if (slice.isExternal()) {
slice = VPackSlice(slice.getExternal());
@ -396,7 +421,8 @@ uint64_t VelocyPackHelper::stringUInt64(VPackSlice const& slice) {
return 0;
}
TRI_json_t* VelocyPackHelper::velocyPackToJson(VPackSlice const& slice, VPackOptions const* options) {
TRI_json_t* VelocyPackHelper::velocyPackToJson(VPackSlice const& slice,
VPackOptions const* options) {
return JsonHelper::fromString(slice.toJson(options));
}
@ -487,19 +513,22 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
TRI_UnlinkFile(tmp.c_str());
}
int fd = TRI_CREATE(tmp.c_str(), O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
int fd = TRI_CREATE(tmp.c_str(),
O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
if (fd < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot create json file '" << tmp << "': " << TRI_LAST_ERROR_STR;
LOG(ERR) << "cannot create json file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
return false;
}
if (!PrintVelocyPack(fd, slice, true)) {
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot write to json file '" << tmp << "': " << TRI_LAST_ERROR_STR;
LOG(ERR) << "cannot write to json file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
@ -510,7 +539,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
if (!TRI_fsync(fd)) {
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot sync saved json '" << tmp << "': " << TRI_LAST_ERROR_STR;
LOG(ERR) << "cannot sync saved json '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
@ -520,7 +550,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
if (res < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot close saved file '" << tmp << "': " << TRI_LAST_ERROR_STR;
LOG(ERR) << "cannot close saved file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
@ -529,7 +560,8 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename << "': " << TRI_LAST_ERROR_STR;
LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename
<< "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
@ -538,9 +570,10 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
return true;
}
int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
bool useUTF8, VPackOptions const* options,
VPackSlice const* lhsBase, VPackSlice const* rhsBase) {
int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, bool useUTF8,
VPackOptions const* options,
VPackSlice const* lhsBase,
VPackSlice const* rhsBase) {
{
// will resolve externals...
int lWeight = TypeWeight(lhs);
@ -553,12 +586,12 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
if (lWeight > rWeight) {
return 1;
}
TRI_ASSERT(lWeight == rWeight);
}
lhs = lhs.resolveExternal(); // follow externals
rhs = rhs.resolveExternal(); // follow externals
lhs = lhs.resolveExternal(); // follow externals
rhs = rhs.resolveExternal(); // follow externals
// lhs and rhs have equal weights
if (lhs.isNone() || rhs.isNone()) {
@ -578,7 +611,7 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
case VPackValueType::MaxKey:
case VPackValueType::None:
case VPackValueType::Null:
return 0;
return 0;
case VPackValueType::Bool: {
bool left = lhs.getBoolean();
bool right = rhs.getBoolean();
@ -602,11 +635,13 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
VPackValueLength nl;
char const* left;
if (lhs.isCustom()) {
if (lhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) {
if (lhsBase == nullptr || options == nullptr ||
options->customTypeHandler == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"Could not extract custom attribute.");
}
lhsString.assign(options->customTypeHandler->toString(lhs, options, *lhsBase));
lhsString.assign(
options->customTypeHandler->toString(lhs, options, *lhsBase));
left = lhsString.c_str();
nl = lhsString.size();
} else {
@ -618,11 +653,13 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
VPackValueLength nr;
char const* right;
if (rhs.isCustom()) {
if (rhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) {
if (rhsBase == nullptr || options == nullptr ||
options->customTypeHandler == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"Could not extract custom attribute.");
}
rhsString.assign(options->customTypeHandler->toString(rhs, options, *rhsBase));
rhsString.assign(
options->customTypeHandler->toString(rhs, options, *rhsBase));
right = rhsString.c_str();
nr = rhsString.size();
} else {
@ -632,7 +669,8 @@ int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs,
int res;
if (useUTF8) {
res = TRI_compare_utf8(left, static_cast<size_t>(nl), right, static_cast<size_t>(nr));
res = TRI_compare_utf8(left, static_cast<size_t>(nl), right,
static_cast<size_t>(nr));
} else {
size_t len = static_cast<size_t>(nl < nr ? nl : nr);
res = memcmp(left, right, len);
@ -743,7 +781,8 @@ double VelocyPackHelper::toDouble(VPackSlice const& slice, bool& failed) {
if (n == 0) {
return 0.0;
} else if (n == 1) {
return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(), failed);
return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(),
failed);
}
break;
}
@ -793,7 +832,8 @@ uint64_t VelocyPackHelper::hashByAttributes(
return hash;
}
void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& output) {
void VelocyPackHelper::SanitizeExternals(VPackSlice const input,
VPackBuilder& output) {
if (input.isExternal()) {
output.add(input.resolveExternal());
} else if (input.isObject()) {
@ -814,8 +854,44 @@ void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& o
}
}
arangodb::LoggerStream& operator<< (arangodb::LoggerStream& logger,
VPackSlice const& slice) {
bool VelocyPackHelper::hasExternals(VPackSlice input) {
if (input.isExternal()) {
return true;
} else if (input.isObject()) {
for (auto const& it : VPackObjectIterator(input)) {
if (hasExternals(it.value) == true) {
return true;
};
}
} else if (input.isArray()) {
for (auto const& it : VPackArrayIterator(input)) {
if (hasExternals(it) == true) {
return true;
}
}
}
return false;
}
VPackBuilder VelocyPackHelper::stanitizeExternalsChecked(VPackSlice input,
bool checkExternals) {
VPackBuilder builder;
bool resolveExt = true;
if (checkExternals) {
resolveExt = hasExternals(input);
}
if (resolveExt) { // resolve
SanitizeExternals(input, builder);
} else {
builder.add(input);
}
return builder; // elided
}
arangodb::LoggerStream& operator<<(arangodb::LoggerStream& logger,
VPackSlice const& slice) {
size_t const cutoff = 100;
std::string sliceStr(slice.toJson());
bool longer = sliceStr.size() > cutoff;

73
lib/Basics/VelocyPackHelper.h
View File

@ -47,10 +47,9 @@ class VelocyPackHelper {
~VelocyPackHelper() = delete;
public:
////////////////////////////////////////////////////////////////////////////////
/// @brief static initializer for all VPack values
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief static initializer for all VPack values
////////////////////////////////////////////////////////////////////////////////
static void initialize();
static void disableAssemblerFunctions();
@ -66,20 +65,21 @@ class VelocyPackHelper {
size_t operator()(arangodb::velocypack::Slice const&) const noexcept;
};
////////////////////////////////////////////////////////////////////////////////
/// @brief equality comparator for VelocyPack values
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief equality comparator for VelocyPack values
////////////////////////////////////////////////////////////////////////////////
struct VPackEqual {
private:
arangodb::velocypack::Options const* _options;
public:
VPackEqual() : _options(nullptr) {}
explicit VPackEqual(arangodb::velocypack::Options const* opts)
: _options(opts) {}
private:
arangodb::velocypack::Options const* _options;
bool operator()(arangodb::velocypack::Slice const&,
arangodb::velocypack::Slice const&) const;
public:
VPackEqual() : _options(nullptr) {}
explicit VPackEqual(arangodb::velocypack::Options const* opts)
: _options(opts) {}
bool operator()(arangodb::velocypack::Slice const&,
arangodb::velocypack::Slice const&) const;
};
struct VPackStringEqual {
@ -87,9 +87,9 @@ class VelocyPackHelper {
arangodb::velocypack::Slice const&) const noexcept;
};
////////////////////////////////////////////////////////////////////////////////
/// @brief less comparator for VelocyPack values
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief less comparator for VelocyPack values
////////////////////////////////////////////////////////////////////////////////
template <bool useUtf8>
struct VPackLess {
@ -159,7 +159,7 @@ class VelocyPackHelper {
struct AttributeSorterUTF8 {
bool operator()(std::string const& l, std::string const& r) const;
};
struct AttributeSorterBinary {
bool operator()(std::string const& l, std::string const& r) const;
};
@ -206,8 +206,9 @@ class VelocyPackHelper {
//////////////////////////////////////////////////////////////////////////////
static std::string checkAndGetStringValue(VPackSlice const&, char const*);
static std::string checkAndGetStringValue(VPackSlice const&, std::string const&);
static std::string checkAndGetStringValue(VPackSlice const&,
std::string const&);
//////////////////////////////////////////////////////////////////////////////
/// @brief returns a Numeric sub-element, or throws if <name> does not exist
@ -243,8 +244,10 @@ class VelocyPackHelper {
/// or it is not a string
//////////////////////////////////////////////////////////////////////////////
static std::string getStringValue(VPackSlice, char const*, std::string const&);
static std::string getStringValue(VPackSlice, std::string const&, std::string const&);
static std::string getStringValue(VPackSlice, char const*,
std::string const&);
static std::string getStringValue(VPackSlice, std::string const&,
std::string const&);
//////////////////////////////////////////////////////////////////////////////
/// @brief convert a Object sub value into a uint64
@ -256,7 +259,8 @@ class VelocyPackHelper {
/// @brief Build TRI_json_t from VelocyPack. Just a temporary solution
//////////////////////////////////////////////////////////////////////////////
static TRI_json_t* velocyPackToJson(VPackSlice const&, VPackOptions const* = &VPackOptions::Defaults);
static TRI_json_t* velocyPackToJson(
VPackSlice const&, VPackOptions const* = &VPackOptions::Defaults);
//////////////////////////////////////////////////////////////////////////////
/// @brief parses a json file to VelocyPack
@ -268,7 +272,7 @@ class VelocyPackHelper {
/// @brief writes a VelocyPack to a file
//////////////////////////////////////////////////////////////////////////////
static bool velocyPackToFile(std::string const& filename,
static bool velocyPackToFile(std::string const& filename,
VPackSlice const& slice, bool syncFile);
//////////////////////////////////////////////////////////////////////////////
@ -284,8 +288,9 @@ class VelocyPackHelper {
//////////////////////////////////////////////////////////////////////////////
static int compare(arangodb::velocypack::Slice lhs,
arangodb::velocypack::Slice rhs,
bool useUTF8, arangodb::velocypack::Options const* options = &arangodb::velocypack::Options::Defaults,
arangodb::velocypack::Slice rhs, bool useUTF8,
arangodb::velocypack::Options const* options =
&arangodb::velocypack::Options::Defaults,
arangodb::velocypack::Slice const* lhsBase = nullptr,
arangodb::velocypack::Slice const* rhsBase = nullptr);
@ -325,7 +330,7 @@ class VelocyPackHelper {
}
return arangodb::velocypack::Slice::falseSlice();
}
static inline arangodb::velocypack::Slice ZeroValue() {
return arangodb::velocypack::Slice::zeroSlice();
}
@ -350,6 +355,10 @@ class VelocyPackHelper {
static void SanitizeExternals(arangodb::velocypack::Slice const,
arangodb::velocypack::Builder&);
static bool hasExternals(arangodb::velocypack::Slice const);
static VPackBuilder stanitizeExternalsChecked(
arangodb::velocypack::Slice const, bool checkExternals = true);
static uint8_t const KeyAttribute = 0x31;
static uint8_t const RevAttribute = 0x32;
static uint8_t const IdAttribute = 0x33;
@ -358,10 +367,12 @@ class VelocyPackHelper {
static uint8_t const AttributeBase = 0x30;
static_assert(KeyAttribute < RevAttribute, "invalid value for _key attribute");
static_assert(KeyAttribute < RevAttribute,
"invalid value for _key attribute");
static_assert(RevAttribute < IdAttribute, "invalid value for _rev attribute");
static_assert(IdAttribute < FromAttribute, "invalid value for _id attribute");
static_assert(FromAttribute < ToAttribute, "invalid value for _from attribute");
static_assert(FromAttribute < ToAttribute,
"invalid value for _from attribute");
};
}
}
@ -371,6 +382,6 @@ class VelocyPackHelper {
//////////////////////////////////////////////////////////////////////////////
arangodb::LoggerStream& operator<<(arangodb::LoggerStream&,
arangodb::velocypack::Slice const&);
arangodb::velocypack::Slice const&);
#endif

2
lib/Logger/LogTopic.cpp
View File

@ -39,7 +39,7 @@ std::map<std::string, LogTopic*> LogTopic::_names;
LogTopic Logger::AGENCY("agency", LogLevel::INFO);
LogTopic Logger::AGENCYCOMM("agencycomm", LogLevel::INFO);
LogTopic Logger::COLLECTOR("collector");
LogTopic Logger::COMMUNICATION("COMMUNICATION", LogLevel::INFO);
LogTopic Logger::COMMUNICATION("communication", LogLevel::INFO);
LogTopic Logger::COMPACTOR("compactor");
LogTopic Logger::CONFIG("config");
LogTopic Logger::DATAFILES("datafiles", LogLevel::INFO);

1
lib/Rest/GeneralRequest.h
View File

@ -165,6 +165,7 @@ class GeneralRequest {
std::vector<std::string> const& suffix() const { return _suffix; }
void addSuffix(std::string&& part);
virtual arangodb::Endpoint::TransportType transportType() = 0;
virtual int64_t contentLength() const = 0;
// get value from headers map. The key must be lowercase.
virtual std::string const& header(std::string const& key) const = 0;

3
lib/Rest/HttpRequest.h
View File

@ -70,6 +70,9 @@ class HttpRequest : public GeneralRequest {
bool isHttp11() const { return _version == ProtocolVersion::HTTP_1_1; }
public:
arangodb::Endpoint::TransportType transportType() override {
return arangodb::Endpoint::TransportType::HTTP;
}
// the content length
int64_t contentLength() const override { return _contentLength; }

4
lib/Rest/VppRequest.h
View File

@ -71,6 +71,10 @@ class VppRequest : public GeneralRequest {
return _message._payload.byteSize();
}
virtual arangodb::Endpoint::TransportType transportType() override {
return arangodb::Endpoint::TransportType::VPP;
};
std::unordered_map<std::string, std::string> const& headers() const override;
// get value from headers map. The key must be lowercase.
std::string const& header(std::string const& key) const override;