1
0
Fork 0

Merge branch 'obi-velocystream-attribute-translator' into obi-velocystream

* obi-velocystream-attribute-translator:
  add options to Slice::get and use them for VPackHeader
  add translator to VelocyPackHelper and headerOption to Vpp Protocol

Conflicts:
	arangod/GeneralServer/VppCommTask.cpp
	lib/Rest/CommonDefines.h
This commit is contained in:
Jan Christoph Uhde 2016-08-24 09:18:35 +02:00
commit 76679e1636
11 changed files with 122 additions and 72 deletions

View File

@ -93,28 +93,28 @@ class Slice {
// creates a slice of type None
static Slice noneSlice() noexcept { return Slice("\x00"); }
// creates a slice of type Illegal
static Slice illegalSlice() noexcept { return Slice("\x17"); }
// creates a slice of type Null
static Slice nullSlice() noexcept { return Slice("\x18"); }
// creates a slice of type Boolean with false value
static Slice falseSlice() noexcept { return Slice("\x19"); }
// creates a slice of type Boolean with true value
static Slice trueSlice() noexcept { return Slice("\x1a"); }
// creates a slice of type Smallint(0)
static Slice zeroSlice() noexcept { return Slice("\x30"); }
// creates a slice of type Array, empty
static Slice emptyArraySlice() noexcept { return Slice("\x01"); }
// creates a slice of type Object, empty
static Slice emptyObjectSlice() noexcept { return Slice("\x0a"); }
// creates a slice of type MinKey
static Slice minKeySlice() noexcept { return Slice("\x1e"); }
@ -124,7 +124,7 @@ class Slice {
// creates a Slice from Json and adds it to a scope
static Slice fromJson(SliceScope& scope, std::string const& json,
Options const* options = &Options::Defaults);
// creates a Slice from a pointer to a uint8_t array
explicit Slice(uint8_t const* start) noexcept
: _start(start) {}
@ -188,7 +188,7 @@ class Slice {
// check if slice is a None object
bool isNone() const noexcept { return isType(ValueType::None); }
// check if slice is an Illegal object
bool isIllegal() const noexcept { return isType(ValueType::Illegal); }
@ -377,7 +377,7 @@ class Slice {
Slice key = getNthKey(index, false);
return Slice(key.start() + key.byteSize());
}
// extract the nth value from an Object
Slice getNthValue(ValueLength index) const {
Slice key = getNthKey(index, false);
@ -386,7 +386,7 @@ class Slice {
// look for the specified attribute path inside an Object
// returns a Slice(ValueType::None) if not found
Slice get(std::vector<std::string> const& attributes,
Slice get(std::vector<std::string> const& attributes,
bool resolveExternals = false) const {
size_t const n = attributes.size();
if (n == 0) {
@ -414,7 +414,7 @@ class Slice {
return last;
}
// look for the specified attribute path inside an Object
// returns a Slice(ValueType::None) if not found
Slice get(std::vector<char const*> const& attributes) const {
@ -440,8 +440,9 @@ class Slice {
// look for the specified attribute inside an Object
// returns a Slice(ValueType::None) if not found
Slice get(std::string const& attribute) const;
Slice get(std::string const& attribute,
Options const* options = &Options::Defaults) const;
// look for the specified attribute inside an Object
// returns a Slice(ValueType::None) if not found
Slice get(char const* attribute) const {
@ -469,7 +470,7 @@ class Slice {
}
return extractValue<char const*>();
}
// returns the Slice managed by an External or the Slice itself if it's not
// an External
Slice resolveExternal() const {
@ -478,7 +479,7 @@ class Slice {
}
return *this;
}
// returns the Slice managed by an External or the Slice itself if it's not
// an External, recursive version
Slice resolveExternals() const {
@ -497,7 +498,7 @@ class Slice {
// translates an integer key into a string
Slice translate() const;
// return the value for an Int object
int64_t getInt() const;
@ -758,26 +759,26 @@ class Slice {
auto const h = head();
VELOCYPACK_ASSERT(h >= 0xf4);
switch (h) {
case 0xf4:
case 0xf5:
case 0xf4:
case 0xf5:
case 0xf6: {
return 2 + readInteger<ValueLength>(_start + 1, 1);
}
case 0xf7:
case 0xf8:
case 0xf7:
case 0xf8:
case 0xf9: {
return 3 + readInteger<ValueLength>(_start + 1, 2);
}
case 0xfa:
case 0xfb:
case 0xfa:
case 0xfb:
case 0xfc: {
return 5 + readInteger<ValueLength>(_start + 1, 4);
}
case 0xfd:
case 0xfe:
case 0xfd:
case 0xfe:
case 0xff: {
return 9 + readInteger<ValueLength>(_start + 1, 8);
}
@ -794,7 +795,7 @@ class Slice {
throw Exception(Exception::InternalError);
}
ValueLength findDataOffset(uint8_t head) const noexcept {
// Must be called for a nonempty array or object at start():
VELOCYPACK_ASSERT(head <= 0x12);
@ -810,7 +811,7 @@ class Slice {
}
return 9;
}
// get the offset for the nth member from an Array type
ValueLength getNthOffset(ValueLength index) const;
@ -834,7 +835,7 @@ class Slice {
return (memcmp(start(), other.start(),
arangodb::velocypack::checkOverflow(size)) == 0);
}
bool operator==(Slice const& other) const { return equals(other); }
bool operator!=(Slice const& other) const { return !equals(other); }
@ -845,10 +846,10 @@ class Slice {
std::string toJson(Options const* options = &Options::Defaults) const;
std::string toString(Options const* options = &Options::Defaults) const;
std::string hexType() const;
private:
// get the total byte size for a String slice, including the head byte
// not check is done if the type of the slice is actually String
// not check is done if the type of the slice is actually String
ValueLength stringSliceLength() const noexcept {
// check if the type has a fixed length first
auto const h = head();
@ -865,7 +866,7 @@ class Slice {
uint64_t getUIntUnchecked() const;
// translates an integer key into a string, without checks
Slice translateUnchecked() const;
Slice translateUnchecked(Options const* = &Options::Defaults) const;
Slice getFromCompactObject(std::string const& attribute) const;

View File

@ -54,7 +54,7 @@ ValueLength const SliceStaticData::FixedTypeLengths[256] = {
/* 0x16 */ 0, /* 0x17 */ 1,
/* 0x18 */ 1, /* 0x19 */ 1,
/* 0x1a */ 1, /* 0x1b */ 1 + sizeof(double),
/* 0x1c */ 1 + sizeof(int64_t), /* 0x1d */ 1 + sizeof(char*),
/* 0x1c */ 1 + sizeof(int64_t), /* 0x1d */ 1 + sizeof(char*),
/* 0x1e */ 1, /* 0x1f */ 1,
/* 0x20 */ 2, /* 0x21 */ 3,
/* 0x22 */ 4, /* 0x23 */ 5,
@ -140,26 +140,26 @@ ValueLength const SliceStaticData::FixedTypeLengths[256] = {
/* 0xc2 */ 0, /* 0xc3 */ 0,
/* 0xc4 */ 0, /* 0xc5 */ 0,
/* 0xc6 */ 0, /* 0xc7 */ 0,
/* 0xc8 */ 0, /* 0xc9 */ 0,
/* 0xca */ 0, /* 0xcb */ 0,
/* 0xcc */ 0, /* 0xcd */ 0,
/* 0xce */ 0, /* 0xcf */ 0,
/* 0xd0 */ 0, /* 0xd1 */ 0,
/* 0xd2 */ 0, /* 0xd3 */ 0,
/* 0xd4 */ 0, /* 0xd5 */ 0,
/* 0xd6 */ 0, /* 0xd7 */ 0,
/* 0xd8 */ 0, /* 0xd9 */ 0,
/* 0xda */ 0, /* 0xdb */ 0,
/* 0xdc */ 0, /* 0xdd */ 0,
/* 0xde */ 0, /* 0xdf */ 0,
/* 0xe0 */ 0, /* 0xe1 */ 0,
/* 0xe2 */ 0, /* 0xe3 */ 0,
/* 0xe4 */ 0, /* 0xe5 */ 0,
/* 0xe6 */ 0, /* 0xe7 */ 0,
/* 0xe8 */ 0, /* 0xe9 */ 0,
/* 0xea */ 0, /* 0xeb */ 0,
/* 0xec */ 0, /* 0xed */ 0,
/* 0xee */ 0, /* 0xef */ 0,
/* 0xc8 */ 0, /* 0xc9 */ 0,
/* 0xca */ 0, /* 0xcb */ 0,
/* 0xcc */ 0, /* 0xcd */ 0,
/* 0xce */ 0, /* 0xcf */ 0,
/* 0xd0 */ 0, /* 0xd1 */ 0,
/* 0xd2 */ 0, /* 0xd3 */ 0,
/* 0xd4 */ 0, /* 0xd5 */ 0,
/* 0xd6 */ 0, /* 0xd7 */ 0,
/* 0xd8 */ 0, /* 0xd9 */ 0,
/* 0xda */ 0, /* 0xdb */ 0,
/* 0xdc */ 0, /* 0xdd */ 0,
/* 0xde */ 0, /* 0xdf */ 0,
/* 0xe0 */ 0, /* 0xe1 */ 0,
/* 0xe2 */ 0, /* 0xe3 */ 0,
/* 0xe4 */ 0, /* 0xe5 */ 0,
/* 0xe6 */ 0, /* 0xe7 */ 0,
/* 0xe8 */ 0, /* 0xe9 */ 0,
/* 0xea */ 0, /* 0xeb */ 0,
/* 0xec */ 0, /* 0xed */ 0,
/* 0xee */ 0, /* 0xef */ 0,
/* 0xf0 */ 2, /* 0xf1 */ 3,
/* 0xf2 */ 5, /* 0xf3 */ 9,
/* 0xf4 */ 0, /* 0xf5 */ 0,
@ -168,7 +168,7 @@ ValueLength const SliceStaticData::FixedTypeLengths[256] = {
/* 0xfa */ 0, /* 0xfb */ 0,
/* 0xfc */ 0, /* 0xfd */ 0,
/* 0xfe */ 0, /* 0xff */ 0};
VT const SliceStaticData::TypeMap[256] = {
/* 0x00 */ VT::None, /* 0x01 */ VT::Array,
/* 0x02 */ VT::Array, /* 0x03 */ VT::Array,
@ -382,8 +382,8 @@ uint64_t Slice::getUIntUnchecked() const {
}
// translates an integer key into a string, without checks
Slice Slice::translateUnchecked() const {
uint8_t const* result = Options::Defaults.attributeTranslator->translate(getUIntUnchecked());
Slice Slice::translateUnchecked(Options const* options) const {
uint8_t const* result = options->attributeTranslator->translate(getUIntUnchecked());
if (result != nullptr) {
return Slice(result);
}
@ -410,7 +410,7 @@ std::string Slice::toString(Options const* options) const {
}
std::string Slice::hexType() const { return HexDump::toHex(head()); }
uint64_t Slice::normalizedHash(uint64_t seed) const {
uint64_t value;
@ -447,7 +447,7 @@ uint64_t Slice::normalizedHash(uint64_t seed) const {
// look for the specified attribute inside an Object
// returns a Slice(ValueType::None) if not found
Slice Slice::get(std::string const& attribute) const {
Slice Slice::get(std::string const& attribute, Options const* options) const {
if (!isObject()) {
throw Exception(Exception::InvalidValueType, "Expecting Object");
}
@ -485,10 +485,10 @@ Slice Slice::get(std::string const& attribute) const {
// fall through to returning None Slice below
} else if (key.isSmallInt() || key.isUInt()) {
// translate key
if (Options::Defaults.attributeTranslator == nullptr) {
if (options->attributeTranslator == nullptr) {
throw Exception(Exception::NeedAttributeTranslator);
}
if (key.translateUnchecked().isEqualString(attribute)) {
if (key.translateUnchecked(options).isEqualString(attribute)) {
return Slice(key.start() + key.byteSize());
}
}
@ -649,7 +649,7 @@ ValueLength Slice::getNthOffset(ValueLength index) const {
// compact Array or Object
return getNthOffsetFromCompact(index);
}
if (h == 0x01 || h == 0x0a) {
// special case: empty Array or empty Object
throw Exception(Exception::IndexOutOfBounds);

View File

@ -31,6 +31,7 @@
#include "GeneralServer/GeneralServerFeature.h"
#include "GeneralServer/RestHandler.h"
#include "GeneralServer/RestHandlerFactory.h"
#include "Rest/CommonDefines.h"
#include "Logger/LoggerFeature.h"
#include "Scheduler/Scheduler.h"
#include "Scheduler/SchedulerFeature.h"
@ -160,12 +161,15 @@ std::unique_ptr<basics::StringBuffer> createChunkForNetworkSingle(
VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock,
ConnectionInfo&& info, double timeout)
: Task("VppCommTask"),
GeneralCommTask(server, sock, std::move(info), timeout) {
GeneralCommTask(server, sock, std::move(info), timeout),
_headerOptions(VPackOptions::Defaults) {
_protocol = "vpp";
_readBuffer->reserve(
_bufferLength); // ATTENTION <- this is required so we do not
// loose information during a resize
// connectionStatisticsAgentSetVpp();
_headerOptions.attributeTranslator =
basics::VelocyPackHelper::getHeaderTranslator();
}
void VppCommTask::addResponse(VppResponse* response) {
@ -397,10 +401,10 @@ bool VppCommTask::processRead() {
if (doExecute) {
VPackSlice header = message.header();
LOG_TOPIC(DEBUG, Logger::COMMUNICATION)
<< "got request:" << header.toJson();
<< "got request:" << header.toJson(&_headerOptions);
int type = meta::underlyingValue(rest::RequestType::ILLEGAL);
try {
type = header.get("type").getInt();
type = header.get("type", &_headerOptions).getInt();
} catch (std::exception const& e) {
throw std::runtime_error(
std::string("Error during Parsing of VppHeader: ") + e.what());
@ -412,6 +416,7 @@ bool VppCommTask::processRead() {
// the handler will take ownersip of this pointer
std::unique_ptr<VppRequest> request(new VppRequest(
_connectionInfo, std::move(message), chunkHeader._messageID));
request->setHeaderOptions(&_headerOptions);
GeneralServerFeature::HANDLER_FACTORY->setRequestContext(request.get());
// make sure we have a database
if (request->requestContext() == nullptr) {
@ -426,6 +431,7 @@ bool VppCommTask::processRead() {
std::unique_ptr<VppResponse> response(
new VppResponse(rest::ResponseCode::SERVER_ERROR,
chunkHeader._messageID));
response->setHeaderOptions(&_headerOptions);
executeRequest(std::move(request), std::move(response));
}
}

View File

@ -113,6 +113,7 @@ class VppCommTask : public GeneralCommTask {
// will be cleaned
};
ProcessReadVariables _processReadVariables;
velocypack::Options _headerOptions;
struct ChunkHeader {
std::size_t _headerLength;

View File

@ -50,6 +50,7 @@ unsigned long long XXH64(const void* input, size_t length,
using VelocyPackHelper = arangodb::basics::VelocyPackHelper;
static std::unique_ptr<VPackAttributeTranslator> Translator;
static std::unique_ptr<VPackAttributeTranslator> HeaderTranslator;
static std::unique_ptr<VPackAttributeExcludeHandler> ExcludeHandler;
// attribute exclude handler for skipping over system attributes
@ -131,6 +132,17 @@ void VelocyPackHelper::initialize() {
TRI_ASSERT(VPackSlice(Translator->translate(ToAttribute - AttributeBase))
.copyString() == StaticStrings::ToString);
HeaderTranslator.reset(new VPackAttributeTranslator);
HeaderTranslator->add("version", 6);
HeaderTranslator->add("type", 7);
HeaderTranslator->add("database", 8);
HeaderTranslator->add("requestType", 9);
HeaderTranslator->add("request", 0);
HeaderTranslator->add("parameter", -1);
HeaderTranslator->add("meta", -2);
HeaderTranslator->add("responseCode", -3);
HeaderTranslator->seal();
// initialize exclude handler for system attributes
ExcludeHandler.reset(new SystemAttributeExcludeHandler);
}
@ -160,6 +172,11 @@ arangodb::velocypack::AttributeTranslator* VelocyPackHelper::getTranslator() {
return Translator.get();
}
arangodb::velocypack::AttributeTranslator*
VelocyPackHelper::getHeaderTranslator() {
return HeaderTranslator.get();
}
bool VelocyPackHelper::AttributeSorterUTF8::operator()(
std::string const& l, std::string const& r) const {
// use UTF-8-based comparison of attribute names

View File

@ -57,6 +57,7 @@ class VelocyPackHelper {
static arangodb::velocypack::AttributeExcludeHandler* getExcludeHandler();
static arangodb::velocypack::AttributeTranslator* getTranslator();
static arangodb::velocypack::AttributeTranslator* getHeaderTranslator();
struct VPackHash {
size_t operator()(arangodb::velocypack::Slice const&) const;

View File

@ -49,6 +49,7 @@ enum class RequestType {
ILLEGAL // must be last
};
enum class ContentType {
CUSTOM, // use Content-Type from _headers
JSON, // application/json
@ -59,6 +60,7 @@ enum class ContentType {
UNSET
};
enum class ProtocolVersion { HTTP_1_0, HTTP_1_1, VPP_1_0, UNKNOWN };
enum ConnectionType {

View File

@ -64,7 +64,8 @@ VppRequest::VppRequest(ConnectionInfo const& connectionInfo,
: GeneralRequest(connectionInfo),
_message(std::move(message)),
_headers(nullptr),
_messageId(messageId) {
_messageId(messageId),
_headerOptions(nullptr) {
_protocol = "vpp";
_contentType = ContentType::VPACK;
_contentTypeResponse = ContentType::VPACK;
@ -106,11 +107,12 @@ void VppRequest::parseHeaderInformation() {
using namespace std;
auto vHeader = _message.header();
try {
_databaseName = vHeader.get("database").copyString();
_requestPath = vHeader.get("request").copyString();
_type = meta::toEnum<RequestType>(vHeader.get("requestType").getInt());
_databaseName = vHeader.get("database", _headerOptions).copyString();
_requestPath = vHeader.get("request", _headerOptions).copyString();
_type = meta::toEnum<RequestType>(
vHeader.get("requestType", _headerOptions).getInt());
VPackSlice params = vHeader.get("parameter");
VPackSlice params = vHeader.get("parameter", _headerOptions);
for (auto const& it : VPackObjectIterator(params)) {
if (it.value.isArray()) {
vector<string> tmp;

View File

@ -77,6 +77,8 @@ class VppRequest : public GeneralRequest {
return arangodb::Endpoint::TransportType::VPP;
};
void setHeaderOptions(VPackOptions* options) { _headerOptions = options; }
std::unordered_map<std::string, std::string> const& headers() const override;
// get value from headers map. The key must be lowercase.
std::string const& header(std::string const& key) const override;
@ -102,6 +104,7 @@ class VppRequest : public GeneralRequest {
std::unordered_map<std::string, std::vector<std::string>> _arrayValues;
uint64_t _messageId;
const std::unordered_map<std::string, std::string> _cookies; // TODO remove
VPackOptions* _headerOptions;
void parseHeaderInformation();
};

View File

@ -27,6 +27,8 @@
#include <velocypack/Builder.h>
#include <velocypack/Dumper.h>
#include <velocypack/Options.h>
#include <velocypack/AttributeTranslator.h>
#include <velocypack/Buffer.h>
#include <velocypack/velocypack-aliases.h>
#include "Basics/Exceptions.h"
@ -38,13 +40,19 @@
#include "Meta/conversion.h"
#include "Rest/VppRequest.h"
#include "CommonDefines.h"
using namespace arangodb;
using namespace arangodb::basics;
bool VppResponse::HIDE_PRODUCT_HEADER = false;
VppResponse::VppResponse(ResponseCode code, uint64_t id)
: GeneralResponse(code), _header(nullptr), _payload(), _messageId(id) {
: GeneralResponse(code),
_header(nullptr),
_payload(),
_messageId(id),
_headerOptions(nullptr) {
_contentType = ContentType::VPACK;
_connectionType = rest::CONNECTION_KEEP_ALIVE;
}
@ -69,14 +77,20 @@ void VppResponse::setPayload(arangodb::velocypack::Slice const& slice,
VPackMessageNoOwnBuffer VppResponse::prepareForNetwork() {
// initalize builder with vpackbuffer. then we do not need to
// steal the header and can avoid the shared pointer
VPackBuilder builder;
// VPackBuffer<uint8_t> buffer;
// VPackBuilder builder(buffer);
VPackBuilder builder(_headerOptions);
builder.openObject();
builder.add("version", VPackValue(int(1)));
builder.add("type", VPackValue(int(1))); // 2 == response
builder.add("type", VPackValue(int(2))); // 2 == response
builder.add(
"responseCode",
VPackValue(static_cast<int>(meta::underlyingValue(_responseCode))));
builder.close();
// options.Defaults.attributeTranslator = nullptr;
_header = builder.steal();
return VPackMessageNoOwnBuffer(VPackSlice(_header->data()),
VPackSlice(_payload.data()), _messageId,

View File

@ -57,6 +57,8 @@ class VppResponse : public GeneralResponse {
return arangodb::Endpoint::TransportType::VPP;
};
void setHeaderOptions(VPackOptions* options) { _headerOptions = options; }
VPackMessageNoOwnBuffer prepareForNetwork();
private:
@ -67,6 +69,7 @@ class VppResponse : public GeneralResponse {
VPackBuffer<uint8_t> _payload;
uint64_t _messageId;
bool _generateBody; // this must be true if payload should be send
VPackOptions* _headerOptions;
};
}