Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

This commit is contained in:
Simon Grätzer 2016-12-17 14:10:03 +01:00
parent f084bd7b12 ea3db60b64
commit 7afebc2223
29 changed files with 497 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
3rdParty/velocypack/include/velocypack/AttributeTranslator.h vendored
View File

@ -35,6 +35,7 @@
#include "velocypack/velocypack-common.h"
#include "velocypack/Options.h"
#include "velocypack/Slice.h"
#include "velocypack/StringRef.h"
namespace arangodb {
namespace velocypack {
@ -68,7 +69,8 @@ class AttributeTranslator {
private:
Builder* _builder;
std::unordered_map<std::string, uint8_t const*> _keyToId;
std::unordered_map<std::string, uint8_t const*> _keyToIdString;
std::unordered_map<StringRef, uint8_t const*> _keyToIdStringRef;
std::unordered_map<uint64_t, uint8_t const*> _idToKey;
size_t _count;
};

7
3rdParty/velocypack/include/velocypack/Builder.h vendored
View File

@ -143,6 +143,11 @@ class Builder {
throw Exception(Exception::InternalError, "Options cannot be a nullptr");
}
}
explicit Builder(Slice const& slice, Options const* options = &Options::Defaults)
: Builder(options) {
add(slice);
}
explicit Builder(Options const* options = &Options::Defaults)
: _buffer(new Buffer<uint8_t>()),
@ -156,7 +161,7 @@ class Builder {
throw Exception(Exception::InternalError, "Options cannot be a nullptr");
}
}
explicit Builder(Buffer<uint8_t>& buffer,
Options const* options = &Options::Defaults)
: _pos(buffer.size()), _keyWritten(false), options(options) {

5
3rdParty/velocypack/include/velocypack/Iterator.h vendored
View File

@ -180,10 +180,10 @@ class ObjectIterator {
ObjectIterator() = delete;
ObjectIterator(Slice const& slice, bool allowRandomIteration = false)
explicit ObjectIterator(Slice const& slice, bool allowRandomIteration = false)
: _slice(slice), _size(_slice.length()), _position(0), _current(nullptr),
_allowRandomIteration(allowRandomIteration) {
if (slice.type() != ValueType::Object) {
if (!slice.isObject()) {
throw Exception(Exception::InvalidValueType, "Expecting Object slice");
}
@ -195,7 +195,6 @@ class ObjectIterator {
_current = slice.begin() + slice.findDataOffset(h);
}
}
}
ObjectIterator(ObjectIterator const& other)

209
3rdParty/velocypack/include/velocypack/StringRef.h vendored Normal file
View File

@ -0,0 +1,209 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief Library to build up VPack documents.
///
/// DISCLAIMER
///
/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Max Neunhoeffer
/// @author Jan Steemann
/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#ifndef VELOCYPACK_STRINGREF_H
#define VELOCYPACK_STRINGREF_H 1
#include <cstdint>
#include <cstring>
#include "velocypack/velocypack-common.h"
#include "velocypack/Slice.h"
namespace arangodb {
namespace velocypack {
class StringRef {
public:
/// @brief create an empty StringRef
constexpr StringRef() noexcept : _data(""), _length(0) {}
/// @brief create a StringRef from an std::string
explicit StringRef(std::string const& str) : StringRef(str.c_str(), str.size()) {}
/// @brief create a StringRef from a null-terminated C string
explicit StringRef(char const* data) : StringRef(data, strlen(data)) {}
/// @brief create a StringRef from a VPack slice (must be of type String)
explicit StringRef(arangodb::velocypack::Slice const& slice) : StringRef() {
VELOCYPACK_ASSERT(slice.isString());
arangodb::velocypack::ValueLength l;
_data = slice.getString(l);
_length = l;
}
/// @brief create a StringRef from a C string plus length
StringRef(char const* data, size_t length) : _data(data), _length(length) {}
/// @brief create a StringRef from another StringRef
StringRef(StringRef const& other) noexcept
: _data(other._data), _length(other._length) {}
/// @brief create a StringRef from another StringRef
StringRef& operator=(StringRef const& other) {
_data = other._data;
_length = other._length;
return *this;
}
/// @brief create a StringRef from an std::string
StringRef& operator=(std::string const& other) {
_data = other.c_str();
_length = other.size();
return *this;
}
/// @brief create a StringRef from a null-terminated C string
StringRef& operator=(char const* other) {
_data = other;
_length = strlen(other);
return *this;
}
/// @brief create a StringRef from a VPack slice of type String
StringRef& operator=(arangodb::velocypack::Slice const& slice) {
arangodb::velocypack::ValueLength l;
_data = slice.getString(l);
_length = l;
return *this;
}
int compare(std::string const& other) const {
int res = memcmp(_data, other.c_str(), (std::min)(_length, other.size()));
if (res != 0) {
return res;
}
return static_cast<int>(_length) - static_cast<int>(other.size());
}
int compare(StringRef const& other) const {
int res = memcmp(_data, other._data, (std::min)(_length, other._length));
if (res != 0) {
return res;
}
return static_cast<int>(_length) - static_cast<int>(other._length);
}
inline std::string toString() const {
return std::string(_data, _length);
}
inline bool empty() const {
return (_length == 0);
}
inline char const* begin() const {
return _data;
}
inline char const* end() const {
return _data + _length;
}
inline char front() const { return _data[0]; }
inline char back() const { return _data[_length - 1]; }
inline char operator[](size_t index) const noexcept {
return _data[index];
}
inline char const* data() const noexcept {
return _data;
}
inline size_t size() const noexcept {
return _length;
}
inline size_t length() const noexcept {
return _length;
}
private:
char const* _data;
size_t _length;
};
}
}
/*
inline bool operator==(arangodb::velocypack::StringRef const& lhs, arangodb::velocypack::StringRef const& rhs) {
return (lhs.size() == rhs.size() && memcmp(lhs.data(), rhs.data(), lhs.size()) == 0);
}
inline bool operator!=(arangodb::velocypack::StringRef const& lhs, arangodb::velocypack::StringRef const& rhs) {
return !(lhs == rhs);
}
inline bool operator==(arangodb::velocypack::StringRef const& lhs, std::string const& rhs) {
return (lhs.size() == rhs.size() && memcmp(lhs.data(), rhs.c_str(), lhs.size()) == 0);
}
inline bool operator!=(arangodb::velocypack::StringRef const& lhs, std::string const& rhs) {
return !(lhs == rhs);
}
inline bool operator==(arangodb::velocypack::StringRef const& lhs, char const* rhs) {
size_t const len = strlen(rhs);
return (lhs.size() == len && memcmp(lhs.data(), rhs, lhs.size()) == 0);
}
inline bool operator!=(arangodb::velocypack::StringRef const& lhs, char const* rhs) {
return !(lhs == rhs);
}
inline bool operator<(arangodb::StringRef const& lhs, arangodb::StringRef const& rhs) {
return (lhs.compare(rhs) < 0);
}
inline bool operator>(arangodb::StringRef const& lhs, arangodb::StringRef const& rhs) {
return (lhs.compare(rhs) > 0);
}
*/
namespace std {
template <>
struct hash<arangodb::velocypack::StringRef> {
size_t operator()(arangodb::velocypack::StringRef const& value) const noexcept {
return VELOCYPACK_HASH(value.data(), value.size(), 0xdeadbeef);
}
};
template <>
struct equal_to<arangodb::velocypack::StringRef> {
bool operator()(arangodb::velocypack::StringRef const& lhs,
arangodb::velocypack::StringRef const& rhs) const noexcept {
return (lhs.size() == rhs.size() &&
(memcmp(lhs.data(), rhs.data(), lhs.size()) == 0));
}
};
}
#endif

21
3rdParty/velocypack/src/AttributeTranslator.cpp vendored
View File

@ -59,17 +59,26 @@ void AttributeTranslator::seal() {
ObjectIterator it(s);
while (it.valid()) {
_keyToId.emplace(it.key(false).copyString(), it.value().begin());
_idToKey.emplace(it.value().getUInt(), it.key(false).begin());
Slice const key(it.key(false));
VELOCYPACK_ASSERT(key.isString());
// extract key value
ValueLength len;
char const* p = key.getString(len);
// insert into string and char lookup maps
_keyToIdString.emplace(key.copyString(), it.value().begin());
_keyToIdStringRef.emplace(StringRef(p, len), it.value().begin());
// insert into id to slice lookup map
_idToKey.emplace(it.value().getUInt(), key.begin());
it.next();
}
}
// translate from string to id
uint8_t const* AttributeTranslator::translate(std::string const& key) const {
auto it = _keyToId.find(key);
auto it = _keyToIdString.find(key);
if (it == _keyToId.end()) {
if (it == _keyToIdString.end()) {
return nullptr;
}
@ -79,9 +88,9 @@ uint8_t const* AttributeTranslator::translate(std::string const& key) const {
// translate from string to id
uint8_t const* AttributeTranslator::translate(char const* key,
ValueLength length) const {
auto it = _keyToId.find(std::string(key, checkOverflow(length)));
auto it = _keyToIdStringRef.find(StringRef(key, length));
if (it == _keyToId.end()) {
if (it == _keyToIdStringRef.end()) {
return nullptr;
}

18
3rdParty/velocypack/src/Builder.cpp vendored
View File

@ -31,6 +31,7 @@
#include "velocypack/Dumper.h"
#include "velocypack/Iterator.h"
#include "velocypack/Sink.h"
#include "velocypack/StringRef.h"
using namespace arangodb::velocypack;
@ -840,13 +841,14 @@ uint8_t* Builder::set(ValuePair const& pair) {
void Builder::checkAttributeUniqueness(Slice const& obj) const {
VELOCYPACK_ASSERT(options->checkAttributeUniqueness == true);
ValueLength const n = obj.length();
if (obj.isSorted()) {
// object attributes are sorted
Slice previous = obj.keyAt(0);
ValueLength len;
char const* p = previous.getString(len);
ValueLength const n = obj.length();
// compare each two adjacent attribute names
for (ValueLength i = 1; i < n; ++i) {
@ -866,17 +868,17 @@ void Builder::checkAttributeUniqueness(Slice const& obj) const {
p = q;
}
} else {
std::unordered_set<std::string> keys;
std::unordered_set<StringRef> keys;
ObjectIterator it(obj, true);
for (ValueLength i = 0; i < n; ++i) {
// note: keyAt() already translates integer attributes
Slice key = obj.keyAt(i);
// keyAt() guarantees a string as returned type
while (it.valid()) {
Slice const key = it.key(true);
// key() guarantees a string as returned type
VELOCYPACK_ASSERT(key.isString());
if (!keys.emplace(key.copyString()).second) {
if (!keys.emplace(StringRef(key)).second) {
throw Exception(Exception::DuplicateAttributeName);
}
it.next();
}
}
}

89
arangod/Agency/AgencyComm.cpp
View File

@ -45,6 +45,8 @@
#include "SimpleHttpClient/SimpleHttpClient.h"
#include "SimpleHttpClient/SimpleHttpResult.h"
#include <thread>
using namespace arangodb;
using namespace arangodb::application_features;
using namespace arangodb::httpclient;
@ -297,7 +299,7 @@ int AgencyCommResult::httpCode() const { return _statusCode; }
int AgencyCommResult::errorCode() const {
try {
std::shared_ptr<VPackBuilder> bodyBuilder =
VPackParser::fromJson(_body.c_str());
VPackParser::fromJson(_body);
VPackSlice body = bodyBuilder->slice();
if (!body.isObject()) {
return 0;
@ -322,7 +324,7 @@ std::string AgencyCommResult::errorMessage() const {
try {
std::shared_ptr<VPackBuilder> bodyBuilder =
VPackParser::fromJson(_body.c_str());
VPackParser::fromJson(_body);
VPackSlice body = bodyBuilder->slice();
if (!body.isObject()) {
@ -367,7 +369,7 @@ VPackSlice AgencyCommResult::slice() const { return _vpack->slice(); }
std::unique_ptr<AgencyCommManager> AgencyCommManager::MANAGER;
AgencyConnectionOptions
AgencyCommManager::CONNECTION_OPTIONS (15.0, 120.0, 120.0, 10);
AgencyCommManager::CONNECTION_OPTIONS (2.0, 15.0, 15.0, 5);
void AgencyCommManager::initialize(std::string const& prefix) {
MANAGER.reset(new AgencyCommManager(prefix));
@ -678,7 +680,7 @@ std::string AgencyComm::version() {
AgencyCommResult result =
sendWithFailover(arangodb::rest::RequestType::GET,
AgencyCommManager::CONNECTION_OPTIONS._requestTimeout,
"/_api/version", "", false);
"/_api/version", "");
if (result.successful()) {
return result._body;
@ -752,14 +754,14 @@ AgencyCommResult AgencyComm::getValues(std::string const& key) {
AgencyCommResult result =
sendWithFailover(arangodb::rest::RequestType::POST,
AgencyCommManager::CONNECTION_OPTIONS._requestTimeout,
url, builder.toJson(), false);
url, builder.toJson());
if (!result.successful()) {
return result;
}
try {
result.setVPack(VPackParser::fromJson(result.body().c_str()));
result.setVPack(VPackParser::fromJson(result.bodyRef()));
if (!result.slice().isArray()) {
result._statusCode = 500;
@ -774,7 +776,7 @@ AgencyCommResult AgencyComm::getValues(std::string const& key) {
result._body.clear();
result._statusCode = 200;
} catch (std::exception& e) {
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::AGENCYCOMM) << "Error transforming result. "
<< e.what();
result.clear();
@ -997,10 +999,10 @@ AgencyCommResult AgencyComm::sendTransactionWithFailover(
arangodb::rest::RequestType::POST,
(timeout == 0.0 ? AgencyCommManager::CONNECTION_OPTIONS._requestTimeout
: timeout),
url, builder.slice().toJson(), false);
url, builder.slice().toJson());
try {
result.setVPack(VPackParser::fromJson(result.body().c_str()));
result.setVPack(VPackParser::fromJson(result.bodyRef()));
if (!transaction.validate(result)) {
result._statusCode = 500;
@ -1009,7 +1011,7 @@ AgencyCommResult AgencyComm::sendTransactionWithFailover(
result._body.clear();
} catch (std::exception& e) {
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::AGENCYCOMM) << "Error transforming result. "
<< e.what();
result.clear();
@ -1180,57 +1182,64 @@ bool AgencyComm::unlock(std::string const& key, VPackSlice const& slice,
AgencyCommResult AgencyComm::sendWithFailover(
arangodb::rest::RequestType method, double const timeout,
std::string const& initialUrl, std::string const& body, bool isWatch) {
std::string const& initialUrl, std::string const& body) {
std::string endpoint;
std::unique_ptr<GeneralClientConnection> connection =
AgencyCommManager::MANAGER->acquire(endpoint);
AgencyCommManager::MANAGER->acquire(endpoint);
AgencyCommResult result;
std::string url = initialUrl;
std::chrono::duration<double> waitInterval (.25); // seconds
auto timeOut = std::chrono::steady_clock::now() +
std::chrono::duration<double>(timeout);
double conTimeout = 1.0;
for (uint64_t tries = 0; tries < MAX_TRIES; ++tries) {
auto waitUntil = std::chrono::steady_clock::now() + waitInterval;
if (waitInterval.count() < 5.0) {
waitInterval *= 2;
}
if (connection == nullptr) {
AgencyCommResult result(400, "No endpoints for agency found.");
LOG_TOPIC(ERR, Logger::AGENCYCOMM) << result._message;
return result;
}
if (1 < tries) {
LOG_TOPIC(WARN, Logger::AGENCYCOMM)
<< "Retrying agency communication at '" << endpoint
<< "', tries: " << tries;
<< "Retrying agency communication at '" << endpoint
<< "', tries: " << tries;
}
// try to send; if we fail completely, do not retry
try {
result = send(connection.get(), method, timeout, url, body);
result = send(connection.get(), method, conTimeout, url, body);
} catch (...) {
AgencyCommManager::MANAGER->failed(std::move(connection), endpoint);
result = AgencyCommResult(0, "could not send request to agency");
result._connected = false;
break;
connection = AgencyCommManager::MANAGER->acquire(endpoint);
continue;
}
// got a result, we are done
if (result.successful()) {
AgencyCommManager::MANAGER->release(std::move(connection), endpoint);
break;
}
// break on a watch timeout (drop connection)
if (result._statusCode == 0) {
AgencyCommManager::MANAGER->failed(std::move(connection), endpoint);
if (isWatch) {
break;
} else {
connection = AgencyCommManager::MANAGER->acquire(endpoint);
continue;
}
connection = AgencyCommManager::MANAGER->acquire(endpoint);
continue;
}
// sometimes the agency will return a 307 (temporary redirect)
// in this case we have to pick it up and use the new location returned
if (result._statusCode ==
@ -1240,15 +1249,27 @@ AgencyCommResult AgencyComm::sendWithFailover(
connection = AgencyCommManager::MANAGER->acquire(endpoint);
continue;
}
// do not retry on client errors
if (result._statusCode >= 400 && result._statusCode <= 499) {
AgencyCommManager::MANAGER->release(std::move(connection), endpoint);
break;
}
AgencyCommManager::MANAGER->failed(std::move(connection), endpoint);
connection = AgencyCommManager::MANAGER->acquire(endpoint);
if (std::chrono::steady_clock::now() < timeOut) {
std::this_thread::sleep_for(waitUntil-std::chrono::steady_clock::now());
} else {
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< "Unsuccessful AgencyComm: Timeout"
<< "errorCode: " << result.errorCode()
<< " errorMessage: " << result.errorMessage()
<< " errorDetails: " << result.errorDetails();
return result;
}
}
if (!result.successful() && result.httpCode() != 412) {

4
arangod/Agency/AgencyComm.h
View File

@ -233,6 +233,7 @@ class AgencyCommResult {
std::string const location() const { return _location; }
std::string const body() const { return _body; }
std::string const& bodyRef() const { return _body; }
void clear();
@ -541,8 +542,7 @@ class AgencyComm {
bool unlock(std::string const&, arangodb::velocypack::Slice const&, double);
AgencyCommResult sendWithFailover(arangodb::rest::RequestType, double,
std::string const&, std::string const&,
bool);
std::string const&, std::string const&);
AgencyCommResult send(httpclient::GeneralClientConnection*, rest::RequestType,
double, std::string const&, std::string const&);

21
arangod/Agency/FailedLeader.cpp
View File

@ -61,7 +61,7 @@ FailedLeader::~FailedLeader() {}
bool FailedLeader::create() {
LOG_TOPIC(INFO, Logger::AGENCY)
<< "Todo: failed Leader for " + _shard + " from " + _from + " to " + _to;
<< "Handle failed Leader for " + _shard + " from " + _from + " to " + _to;
std::string path = _agencyPrefix + toDoPrefix + _jobId;
@ -114,10 +114,9 @@ bool FailedLeader::start() {
Node const& current = _snapshot(curPath);
if (current.slice().length() == 1) {
LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to change leadership for shard " +
_shard + " from " + _from + " to " +
_to + ". No in-sync followers:" +
current.slice().toJson();
LOG_TOPIC(ERR, Logger::AGENCY)
<< "Failed to change leadership for shard " + _shard + " from " + _from
+ " to " + _to + ". No in-sync followers:" + current.slice().toJson();
return false;
}
@ -130,15 +129,15 @@ bool FailedLeader::start() {
try {
_snapshot(toDoPrefix + _jobId).toBuilder(todo);
} catch (std::exception const&) {
LOG_TOPIC(INFO, Logger::AGENCY) << "Failed to get key " + toDoPrefix +
_jobId + " from agency snapshot";
LOG_TOPIC(INFO, Logger::AGENCY)
<< "Failed to get key " + toDoPrefix + _jobId + " from agency snapshot";
return false;
}
} else {
todo.add(_jb->slice().get(_agencyPrefix + toDoPrefix + _jobId).valueAt(0));
}
todo.close();
// Transaction
pending.openArray();
@ -201,11 +200,11 @@ bool FailedLeader::start() {
write_ret_t res = transact(_agent, pending);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(INFO, Logger::AGENCY) << "Pending: Change leadership " + _shard +
" from " + _from + " to " + _to;
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Pending: Change leadership " + _shard + " from " + _from + " to " + _to;
return true;
}
LOG_TOPIC(INFO, Logger::AGENCY)
<< "Precondition failed for starting job " + _jobId;
return false;

12
arangod/Agency/FailedServer.cpp
View File

@ -55,19 +55,19 @@ FailedServer::~FailedServer() {}
bool FailedServer::start() {
LOG_TOPIC(INFO, Logger::AGENCY)
<< "Trying to start FailedServer job" + _jobId + " for server " + _server;
<< "Start FailedServer job" + _jobId + " for server " + _server;
// Copy todo to pending
Builder todo, pending;
// Get todo entry
todo.openArray();
if (_jb == nullptr) {
try {
_snapshot(toDoPrefix + _jobId).toBuilder(todo);
} catch (std::exception const&) {
LOG_TOPIC(INFO, Logger::AGENCY) << "Failed to get key " + toDoPrefix +
_jobId + " from agency snapshot";
LOG_TOPIC(INFO, Logger::AGENCY)
<< "Failed to get key " + toDoPrefix + _jobId + " from agency snapshot";
return false;
}
} else {
@ -118,7 +118,7 @@ bool FailedServer::start() {
write_ret_t res = transact(_agent, pending);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(INFO, Logger::AGENCY)
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Pending job for failed DB Server " << _server;
auto const& databases = _snapshot("/Plan/Collections").children();
@ -206,7 +206,7 @@ bool FailedServer::start() {
}
bool FailedServer::create() {
LOG_TOPIC(INFO, Logger::AGENCY) << "Todo: DB Server " + _server + " failed.";
LOG_TOPIC(DEBUG, Logger::AGENCY) << "Todo: Handle failed db server " + _server;
std::string path = _agencyPrefix + toDoPrefix + _jobId;

12
arangod/Agency/Inception.cpp
View File

@ -301,8 +301,10 @@ void Inception::reportIn(query_t const& query) {
MUTEX_LOCKER(lock, _mLock);
_measurements.push_back(
std::vector<double>(
{slice.get("mean").getNumber<double>(), slice.get("stdev").getNumber<double>(),
slice.get("max").getNumber<double>(), slice.get("min").getNumber<double>()} ));
{slice.get("mean").getNumber<double>(),
slice.get("stdev").getNumber<double>(),
slice.get("max").getNumber<double>(),
slice.get("min").getNumber<double>()} ));
}
@ -445,12 +447,12 @@ bool Inception::estimateRAFTInterval() {
double precision = 1.0e-2;
mn = precision *
std::ceil((1. / precision)*(.5 + precision * (maxmean + 3.*maxstdev)));
std::ceil((1. / precision)*(1. + precision * (maxmean + 3.*maxstdev)));
if (config.waitForSync()) {
mn *= 4.;
}
if (mn > 2.0) {
mn = 2.0;
if (mn > 5.0) {
mn = 5.0;
}
mx = 5. * mn;

1
arangod/CMakeLists.txt
View File

@ -257,6 +257,7 @@ SET(ARANGOD_SOURCES
RestServer/DatabaseFeature.cpp
RestServer/DatabasePathFeature.cpp
RestServer/EndpointFeature.cpp
RestServer/FeatureCacheFeature.cpp
RestServer/FileDescriptorsFeature.cpp
RestServer/FrontendFeature.cpp
RestServer/InitDatabaseFeature.cpp

1
arangod/Cluster/HeartbeatThread.cpp
View File

@ -169,6 +169,7 @@ void HeartbeatThread::runDBServer() {
// we check Current/Version every few heartbeats:
int const currentCountStart = 1; // set to 1 by Max to speed up discovery
int currentCount = currentCountStart;
// size_t hearbeats = 0;
while (!isStopping()) {
try {

11
arangod/GeneralServer/GeneralServerFeature.cpp
View File

@ -70,6 +70,7 @@
#include "RestHandler/WorkMonitorHandler.h"
#include "RestServer/DatabaseFeature.h"
#include "RestServer/EndpointFeature.h"
#include "RestServer/FeatureCacheFeature.h"
#include "RestServer/QueryRegistryFeature.h"
#include "RestServer/ServerFeature.h"
#include "RestServer/TraverserEngineRegistryFeature.h"
@ -183,9 +184,7 @@ void GeneralServerFeature::validateOptions(std::shared_ptr<ProgramOptions>) {
}
static TRI_vocbase_t* LookupDatabaseFromRequest(GeneralRequest* request) {
auto databaseFeature =
application_features::ApplicationServer::getFeature<DatabaseFeature>(
"Database");
auto databaseFeature = FeatureCacheFeature::instance()->databaseFeature();
// get database name from request
std::string const& dbName = request->databaseName();
@ -208,8 +207,7 @@ static TRI_vocbase_t* LookupDatabaseFromRequest(GeneralRequest* request) {
}
static bool SetRequestContext(GeneralRequest* request, void* data) {
auto authentication = application_features::ApplicationServer::getFeature<
AuthenticationFeature>("Authentication");
auto authentication = FeatureCacheFeature::instance()->authenticationFeature();
TRI_ASSERT(authentication != nullptr);
TRI_vocbase_t* vocbase = LookupDatabaseFromRequest(request);
@ -254,8 +252,7 @@ void GeneralServerFeature::start() {
// populate the authentication cache. otherwise no one can access the new
// database
auto authentication = application_features::ApplicationServer::getFeature<
AuthenticationFeature>("Authentication");
auto authentication = FeatureCacheFeature::instance()->authenticationFeature();
TRI_ASSERT(authentication != nullptr);
if (authentication->isEnabled()) {
authentication->authInfo()->outdate();

2
arangod/GeneralServer/HttpCommTask.cpp
View File

@ -528,7 +528,7 @@ bool HttpCommTask::processRead(double startTime) {
LOG(DEBUG) << "no keep-alive, connection close requested by client";
_closeRequested = true;
} else if (!_useKeepAliveTimeout) {
} else if (!_useKeepAliveTimer) {
// if keepAliveTimeout was set to 0.0, we'll close even keep-alive
// connections immediately
LOG(DEBUG) << "keep-alive disabled by admin";

56
arangod/RestServer/FeatureCacheFeature.cpp Normal file
View File

@ -0,0 +1,56 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "FeatureCacheFeature.h"
#include "ApplicationFeatures/ApplicationServer.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "RestServer/DatabaseFeature.h"
using namespace arangodb;
using namespace arangodb::application_features;
FeatureCacheFeature* FeatureCacheFeature::Instance = nullptr;
FeatureCacheFeature::FeatureCacheFeature(application_features::ApplicationServer* server)
: ApplicationFeature(server, "FeatureCache"),
_authenticationFeature(nullptr),
_databaseFeature(nullptr) {
setOptional(false);
requiresElevatedPrivileges(false);
}
void FeatureCacheFeature::prepare() {
_authenticationFeature = application_features::ApplicationServer::getFeature<AuthenticationFeature>(
"Authentication");
_databaseFeature = application_features::ApplicationServer::getFeature<DatabaseFeature>(
"Database");
TRI_ASSERT(Instance == nullptr);
Instance = this;
}
void FeatureCacheFeature::unprepare() {
_authenticationFeature = nullptr;
_databaseFeature = nullptr;
// do not reset instance
}

63
arangod/RestServer/FeatureCacheFeature.h Normal file
View File

@ -0,0 +1,63 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef REST_SERVER_FEATURE_CACHE_FEATURE_H
#define REST_SERVER_FEATURE_CACHE_FEATURE_H 1
#include "ApplicationFeatures/ApplicationFeature.h"
namespace arangodb {
class AuthenticationFeature;
class DatabaseFeature;
class FeatureCacheFeature final : public application_features::ApplicationFeature {
public:
explicit FeatureCacheFeature(application_features::ApplicationServer*);
public:
void prepare() override final;
void unprepare() override final;
static inline FeatureCacheFeature* instance() {
TRI_ASSERT(Instance != nullptr);
return Instance;
}
inline AuthenticationFeature* authenticationFeature() const {
TRI_ASSERT(_authenticationFeature != nullptr);
return _authenticationFeature;
}
inline DatabaseFeature* databaseFeature() const {
TRI_ASSERT(_databaseFeature != nullptr);
return _databaseFeature;
}
private:
static FeatureCacheFeature* Instance;
AuthenticationFeature* _authenticationFeature;
DatabaseFeature* _databaseFeature;
};
}
#endif

10
arangod/RestServer/ServerFeature.h
View File

@ -33,6 +33,8 @@ class RestHandlerFactory;
class AsyncJobManager;
}
class FeatureCacheFeature;
class ServerFeature final : public application_features::ApplicationFeature {
public:
static std::string operationModeString(OperationMode mode);
@ -56,6 +58,9 @@ class ServerFeature final : public application_features::ApplicationFeature {
std::vector<std::string> const& scripts() const { return _scripts; }
std::vector<std::string> const& unitTests() const { return _unitTests; }
uint32_t const& vppMaxSize() const { return _vppMaxSize; }
private:
void waitForHeartbeat();
private:
bool _console = false;
@ -63,11 +68,6 @@ class ServerFeature final : public application_features::ApplicationFeature {
std::vector<std::string> _unitTests;
std::vector<std::string> _scripts;
uint32_t _vppMaxSize;
private:
void waitForHeartbeat();
private:
int* _result;
OperationMode _operationMode;
};

5
arangod/RestServer/VocbaseContext.cpp
View File

@ -34,6 +34,7 @@
#include "Endpoint/ConnectionInfo.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "Logger/Logger.h"
#include "RestServer/FeatureCacheFeature.h"
#include "Ssl/SslInterface.h"
#include "Utils/Events.h"
#include "VocBase/AuthInfo.h"
@ -51,9 +52,7 @@ VocbaseContext::VocbaseContext(GeneralRequest* request, TRI_vocbase_t* vocbase)
_vocbase(vocbase),
_authentication(nullptr) {
TRI_ASSERT(_vocbase != nullptr);
_authentication =
application_features::ApplicationServer::getFeature<AuthenticationFeature>(
"Authentication");
_authentication = FeatureCacheFeature::instance()->authenticationFeature();
TRI_ASSERT(_authentication != nullptr);
}

2
arangod/RestServer/arangod.cpp
View File

@ -54,6 +54,7 @@
#include "RestServer/DatabaseFeature.h"
#include "RestServer/DatabasePathFeature.h"
#include "RestServer/EndpointFeature.h"
#include "RestServer/FeatureCacheFeature.h"
#include "RestServer/FileDescriptorsFeature.h"
#include "RestServer/FrontendFeature.h"
#include "RestServer/InitDatabaseFeature.h"
@ -131,6 +132,7 @@ static int runServer(int argc, char** argv) {
server.addFeature(new DatabasePathFeature(&server));
server.addFeature(new EndpointFeature(&server));
server.addFeature(new EngineSelectorFeature(&server));
server.addFeature(new FeatureCacheFeature(&server));
server.addFeature(new FileDescriptorsFeature(&server));
server.addFeature(new FoxxQueuesFeature(&server));
server.addFeature(new FrontendFeature(&server));

17
arangod/Scheduler/SocketTask.cpp
View File

@ -50,8 +50,10 @@ SocketTask::SocketTask(arangodb::EventLoop loop,
_readBuffer(TRI_UNKNOWN_MEM_ZONE, READ_BLOCK_SIZE + 1, false),
_peer(std::move(socket)),
_keepAliveTimeout(static_cast<long>(keepAliveTimeout * 1000)),
_useKeepAliveTimeout(static_cast<long>(keepAliveTimeout * 1000) > 0),
_keepAliveTimer(_peer->_ioService, _keepAliveTimeout),
_useKeepAliveTimer(keepAliveTimeout > 0.0),
_keepAliveTimerActive(false),
_closeRequested(false),
_abandoned(false) {
ConnectionStatisticsAgent::acquire();
connectionStatisticsAgentSetStart();
@ -71,11 +73,13 @@ SocketTask::SocketTask(arangodb::EventLoop loop,
SocketTask::~SocketTask() {
boost::system::error_code err;
_keepAliveTimer.cancel(err);
if (_keepAliveTimerActive) {
_keepAliveTimer.cancel(err);
}
if (err) {
LOG_TOPIC(ERR, Logger::COMMUNICATION) << "unable to cancel _keepAliveTimer";
}
}
}
void SocketTask::start() {
@ -278,6 +282,7 @@ void SocketTask::closeStream() {
_closeRequested = false;
_keepAliveTimer.cancel();
_keepAliveTimerActive = false;
}
// -----------------------------------------------------------------------------
@ -289,7 +294,7 @@ void SocketTask::addToReadBuffer(char const* data, std::size_t len) {
}
void SocketTask::resetKeepAlive() {
if (_useKeepAliveTimeout) {
if (_useKeepAliveTimer) {
boost::system::error_code err;
_keepAliveTimer.expires_from_now(_keepAliveTimeout, err);
@ -298,6 +303,7 @@ void SocketTask::resetKeepAlive() {
return;
}
_keepAliveTimerActive = true;
auto self = shared_from_this();
_keepAliveTimer.async_wait(
@ -314,9 +320,10 @@ void SocketTask::resetKeepAlive() {
}
void SocketTask::cancelKeepAlive() {
if (_useKeepAliveTimeout) {
if (_useKeepAliveTimer && _keepAliveTimerActive) {
boost::system::error_code err;
_keepAliveTimer.cancel(err);
_keepAliveTimerActive = false;
}
}

6
arangod/Scheduler/SocketTask.h
View File

@ -101,10 +101,10 @@ class SocketTask : virtual public Task, public ConnectionStatisticsAgent {
std::unique_ptr<Socket> _peer;
boost::posix_time::milliseconds _keepAliveTimeout;
bool _useKeepAliveTimeout;
boost::asio::deadline_timer _keepAliveTimer;
bool _closeRequested = false;
bool const _useKeepAliveTimer;
bool _keepAliveTimerActive;
bool _closeRequested;
std::atomic_bool _abandoned;
private:

5
arangod/Utils/DatabaseGuard.h
View File

@ -27,6 +27,7 @@
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/Exceptions.h"
#include "RestServer/DatabaseFeature.h"
#include "RestServer/FeatureCacheFeature.h"
struct TRI_vocbase_t;
@ -44,7 +45,7 @@ class DatabaseGuard {
explicit DatabaseGuard(TRI_voc_tick_t id)
: _vocbase(nullptr) {
auto databaseFeature = application_features::ApplicationServer::getFeature<DatabaseFeature>("Database");
auto databaseFeature = FeatureCacheFeature::instance()->databaseFeature();
_vocbase = databaseFeature->useDatabase(id);
if (_vocbase == nullptr) {
@ -59,7 +60,7 @@ class DatabaseGuard {
explicit DatabaseGuard(std::string const& name)
: _vocbase(nullptr) {
auto databaseFeature = application_features::ApplicationServer::getFeature<DatabaseFeature>("Database");
auto databaseFeature = FeatureCacheFeature::instance()->databaseFeature();
_vocbase = databaseFeature->useDatabase(name);
if (_vocbase == nullptr) {

2
arangod/VocBase/vocbase.cpp
View File

@ -1298,7 +1298,7 @@ void TRI_SanitizeObject(VPackSlice const slice, VPackBuilder& builder) {
/// open object which will remain open. also excludes _from and _to
void TRI_SanitizeObjectWithEdges(VPackSlice const slice, VPackBuilder& builder) {
TRI_ASSERT(slice.isObject());
VPackObjectIterator it(slice);
VPackObjectIterator it(slice, true);
while (it.valid()) {
StringRef key(it.key());
if (key.empty() || key[0] != '_' ||

1
arangod/Wal/LogfileManager.cpp
View File

@ -112,6 +112,7 @@ LogfileManager::LogfileManager(ApplicationServer* server)
requiresElevatedPrivileges(false);
startsAfter("DatabasePath");
startsAfter("EngineSelector");
startsAfter("FeatureCache");
startsAfter("RevisionCache");
for (auto const& it : EngineSelectorFeature::availableEngines()) {

11
js/server/tests/resilience/moving-shards-cluster.js
View File

@ -305,9 +305,14 @@ function MovingShardsSuite () {
Object.keys(state.Pending).length === 0) {
return true;
}
console.info("Waiting for supervision jobs to finish:",
"ToDo jobs:", Object.keys(state.ToDo).length,
"Pending jobs:", Object.keys(state.Pending).length);
if (state.error) {
console.warn("Waiting for supervision jobs to finish:",
"Currently no agency communication possible.");
} else {
console.info("Waiting for supervision jobs to finish:",
"ToDo jobs:", Object.keys(state.ToDo).length,
"Pending jobs:", Object.keys(state.Pending).length);
}
wait(1.0);
}
return false;

4
lib/Rest/GeneralRequest.h
View File

@ -175,9 +175,7 @@ class GeneralRequest {
std::shared_ptr<VPackBuilder> toVelocyPackBuilderPtr(
arangodb::velocypack::Options const* options) {
auto rv = std::make_shared<VPackBuilder>();
rv->add(payload(options));
return rv;
return std::make_shared<VPackBuilder>(payload(options), options);
};
ContentType contentType() const { return _contentType; }

16
lib/Rest/HttpRequest.cpp
View File

@ -476,7 +476,7 @@ void HttpRequest::setValues(char* buffer, char* end) {
*(key - 2) = '\0';
setArrayValue(keyBegin, key - keyBegin - 2, valueBegin);
} else {
_values[std::string(keyBegin, key - keyBegin)] = valueBegin;
_values[std::string(keyBegin, key - keyBegin)] = std::string(valueBegin, value - valueBegin);
}
keyBegin = key = buffer + 1;
@ -535,7 +535,7 @@ void HttpRequest::setValues(char* buffer, char* end) {
*(key - 2) = '\0';
setArrayValue(keyBegin, key - keyBegin - 2, valueBegin);
} else {
_values[std::string(keyBegin, key - keyBegin)] = valueBegin;
_values[std::string(keyBegin, key - keyBegin)] = std::string(valueBegin, value - valueBegin);
}
}
}
@ -742,21 +742,19 @@ VPackSlice HttpRequest::payload(VPackOptions const* options) {
// check options for nullptr?
if (_contentType == ContentType::JSON) {
if (body().length() > 0) {
if (!_body.empty()) {
if (_vpackBuilder == nullptr) {
VPackParser parser(options);
parser.parse(body());
parser.parse(_body);
_vpackBuilder = parser.steal();
return VPackSlice(_vpackBuilder->slice());
} else {
return VPackSlice(_vpackBuilder->slice());
}
return VPackSlice(_vpackBuilder->slice());
}
return VPackSlice::noneSlice(); // no body
} else /*VPACK*/ {
VPackValidator validator;
validator.validate(body().c_str(), body().length());
return VPackSlice(body().c_str());
validator.validate(_body.c_str(), _body.length());
return VPackSlice(_body.c_str());
}
}

2
scripts/startStandAloneAgency.sh
View File

@ -158,7 +158,7 @@ if [ "$GOSSIP_MODE" = "0" ]; then
GOSSIP_PEERS=" --agency.endpoint $TRANSPORT://localhost:$BASE"
fi
#rm -rf agency
rm -rf agency
mkdir -p agency
PIDS=""