diff --git a/3rdParty/velocypack/include/velocypack/AttributeTranslator.h b/3rdParty/velocypack/include/velocypack/AttributeTranslator.h index 6b53be5e14..c463837c23 100644 --- a/3rdParty/velocypack/include/velocypack/AttributeTranslator.h +++ b/3rdParty/velocypack/include/velocypack/AttributeTranslator.h @@ -35,6 +35,7 @@ #include "velocypack/velocypack-common.h" #include "velocypack/Options.h" #include "velocypack/Slice.h" +#include "velocypack/StringRef.h" namespace arangodb { namespace velocypack { @@ -68,7 +69,8 @@ class AttributeTranslator { private: Builder* _builder; - std::unordered_map _keyToId; + std::unordered_map _keyToIdString; + std::unordered_map _keyToIdStringRef; std::unordered_map _idToKey; size_t _count; }; diff --git a/3rdParty/velocypack/include/velocypack/Builder.h b/3rdParty/velocypack/include/velocypack/Builder.h index c8bc9121c5..594dba862e 100644 --- a/3rdParty/velocypack/include/velocypack/Builder.h +++ b/3rdParty/velocypack/include/velocypack/Builder.h @@ -143,6 +143,11 @@ class Builder { throw Exception(Exception::InternalError, "Options cannot be a nullptr"); } } + + explicit Builder(Slice const& slice, Options const* options = &Options::Defaults) + : Builder(options) { + add(slice); + } explicit Builder(Options const* options = &Options::Defaults) : _buffer(new Buffer()), @@ -156,7 +161,7 @@ class Builder { throw Exception(Exception::InternalError, "Options cannot be a nullptr"); } } - + explicit Builder(Buffer& buffer, Options const* options = &Options::Defaults) : _pos(buffer.size()), _keyWritten(false), options(options) { diff --git a/3rdParty/velocypack/include/velocypack/Iterator.h b/3rdParty/velocypack/include/velocypack/Iterator.h index 25664584a4..49ce73f81d 100644 --- a/3rdParty/velocypack/include/velocypack/Iterator.h +++ b/3rdParty/velocypack/include/velocypack/Iterator.h @@ -180,10 +180,10 @@ class ObjectIterator { ObjectIterator() = delete; - ObjectIterator(Slice const& slice, bool allowRandomIteration = false) + explicit ObjectIterator(Slice const& slice, bool allowRandomIteration = false) : _slice(slice), _size(_slice.length()), _position(0), _current(nullptr), _allowRandomIteration(allowRandomIteration) { - if (slice.type() != ValueType::Object) { + if (!slice.isObject()) { throw Exception(Exception::InvalidValueType, "Expecting Object slice"); } @@ -195,7 +195,6 @@ class ObjectIterator { _current = slice.begin() + slice.findDataOffset(h); } } - } ObjectIterator(ObjectIterator const& other) diff --git a/3rdParty/velocypack/include/velocypack/StringRef.h b/3rdParty/velocypack/include/velocypack/StringRef.h new file mode 100644 index 0000000000..9b6315f79f --- /dev/null +++ b/3rdParty/velocypack/include/velocypack/StringRef.h @@ -0,0 +1,209 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief Library to build up VPack documents. +/// +/// DISCLAIMER +/// +/// Copyright 2015 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#ifndef VELOCYPACK_STRINGREF_H +#define VELOCYPACK_STRINGREF_H 1 + +#include +#include + +#include "velocypack/velocypack-common.h" +#include "velocypack/Slice.h" + +namespace arangodb { +namespace velocypack { + +class StringRef { + public: + /// @brief create an empty StringRef + constexpr StringRef() noexcept : _data(""), _length(0) {} + + /// @brief create a StringRef from an std::string + explicit StringRef(std::string const& str) : StringRef(str.c_str(), str.size()) {} + + /// @brief create a StringRef from a null-terminated C string + explicit StringRef(char const* data) : StringRef(data, strlen(data)) {} + + /// @brief create a StringRef from a VPack slice (must be of type String) + explicit StringRef(arangodb::velocypack::Slice const& slice) : StringRef() { + VELOCYPACK_ASSERT(slice.isString()); + arangodb::velocypack::ValueLength l; + _data = slice.getString(l); + _length = l; + } + + /// @brief create a StringRef from a C string plus length + StringRef(char const* data, size_t length) : _data(data), _length(length) {} + + /// @brief create a StringRef from another StringRef + StringRef(StringRef const& other) noexcept + : _data(other._data), _length(other._length) {} + + /// @brief create a StringRef from another StringRef + StringRef& operator=(StringRef const& other) { + _data = other._data; + _length = other._length; + return *this; + } + + /// @brief create a StringRef from an std::string + StringRef& operator=(std::string const& other) { + _data = other.c_str(); + _length = other.size(); + return *this; + } + + /// @brief create a StringRef from a null-terminated C string + StringRef& operator=(char const* other) { + _data = other; + _length = strlen(other); + return *this; + } + + /// @brief create a StringRef from a VPack slice of type String + StringRef& operator=(arangodb::velocypack::Slice const& slice) { + arangodb::velocypack::ValueLength l; + _data = slice.getString(l); + _length = l; + return *this; + } + + int compare(std::string const& other) const { + int res = memcmp(_data, other.c_str(), (std::min)(_length, other.size())); + if (res != 0) { + return res; + } + return static_cast(_length) - static_cast(other.size()); + } + + int compare(StringRef const& other) const { + int res = memcmp(_data, other._data, (std::min)(_length, other._length)); + if (res != 0) { + return res; + } + return static_cast(_length) - static_cast(other._length); + } + + inline std::string toString() const { + return std::string(_data, _length); + } + + inline bool empty() const { + return (_length == 0); + } + + inline char const* begin() const { + return _data; + } + + inline char const* end() const { + return _data + _length; + } + + inline char front() const { return _data[0]; } + + inline char back() const { return _data[_length - 1]; } + + inline char operator[](size_t index) const noexcept { + return _data[index]; + } + + inline char const* data() const noexcept { + return _data; + } + + inline size_t size() const noexcept { + return _length; + } + + inline size_t length() const noexcept { + return _length; + } + + private: + char const* _data; + size_t _length; +}; + +} +} + +/* +inline bool operator==(arangodb::velocypack::StringRef const& lhs, arangodb::velocypack::StringRef const& rhs) { + return (lhs.size() == rhs.size() && memcmp(lhs.data(), rhs.data(), lhs.size()) == 0); +} + +inline bool operator!=(arangodb::velocypack::StringRef const& lhs, arangodb::velocypack::StringRef const& rhs) { + return !(lhs == rhs); +} + +inline bool operator==(arangodb::velocypack::StringRef const& lhs, std::string const& rhs) { + return (lhs.size() == rhs.size() && memcmp(lhs.data(), rhs.c_str(), lhs.size()) == 0); +} + +inline bool operator!=(arangodb::velocypack::StringRef const& lhs, std::string const& rhs) { + return !(lhs == rhs); +} + +inline bool operator==(arangodb::velocypack::StringRef const& lhs, char const* rhs) { + size_t const len = strlen(rhs); + return (lhs.size() == len && memcmp(lhs.data(), rhs, lhs.size()) == 0); +} + +inline bool operator!=(arangodb::velocypack::StringRef const& lhs, char const* rhs) { + return !(lhs == rhs); +} + +inline bool operator<(arangodb::StringRef const& lhs, arangodb::StringRef const& rhs) { + return (lhs.compare(rhs) < 0); +} + +inline bool operator>(arangodb::StringRef const& lhs, arangodb::StringRef const& rhs) { + return (lhs.compare(rhs) > 0); +} +*/ + +namespace std { + +template <> +struct hash { + size_t operator()(arangodb::velocypack::StringRef const& value) const noexcept { + return VELOCYPACK_HASH(value.data(), value.size(), 0xdeadbeef); + } +}; + +template <> +struct equal_to { + bool operator()(arangodb::velocypack::StringRef const& lhs, + arangodb::velocypack::StringRef const& rhs) const noexcept { + return (lhs.size() == rhs.size() && + (memcmp(lhs.data(), rhs.data(), lhs.size()) == 0)); + } +}; + +} + +#endif diff --git a/3rdParty/velocypack/src/AttributeTranslator.cpp b/3rdParty/velocypack/src/AttributeTranslator.cpp index ae0ba93a53..6bcd548669 100644 --- a/3rdParty/velocypack/src/AttributeTranslator.cpp +++ b/3rdParty/velocypack/src/AttributeTranslator.cpp @@ -59,17 +59,26 @@ void AttributeTranslator::seal() { ObjectIterator it(s); while (it.valid()) { - _keyToId.emplace(it.key(false).copyString(), it.value().begin()); - _idToKey.emplace(it.value().getUInt(), it.key(false).begin()); + Slice const key(it.key(false)); + VELOCYPACK_ASSERT(key.isString()); + + // extract key value + ValueLength len; + char const* p = key.getString(len); + // insert into string and char lookup maps + _keyToIdString.emplace(key.copyString(), it.value().begin()); + _keyToIdStringRef.emplace(StringRef(p, len), it.value().begin()); + // insert into id to slice lookup map + _idToKey.emplace(it.value().getUInt(), key.begin()); it.next(); } } // translate from string to id uint8_t const* AttributeTranslator::translate(std::string const& key) const { - auto it = _keyToId.find(key); + auto it = _keyToIdString.find(key); - if (it == _keyToId.end()) { + if (it == _keyToIdString.end()) { return nullptr; } @@ -79,9 +88,9 @@ uint8_t const* AttributeTranslator::translate(std::string const& key) const { // translate from string to id uint8_t const* AttributeTranslator::translate(char const* key, ValueLength length) const { - auto it = _keyToId.find(std::string(key, checkOverflow(length))); + auto it = _keyToIdStringRef.find(StringRef(key, length)); - if (it == _keyToId.end()) { + if (it == _keyToIdStringRef.end()) { return nullptr; } diff --git a/3rdParty/velocypack/src/Builder.cpp b/3rdParty/velocypack/src/Builder.cpp index 3b8c78651f..ea395f0eb5 100644 --- a/3rdParty/velocypack/src/Builder.cpp +++ b/3rdParty/velocypack/src/Builder.cpp @@ -31,6 +31,7 @@ #include "velocypack/Dumper.h" #include "velocypack/Iterator.h" #include "velocypack/Sink.h" +#include "velocypack/StringRef.h" using namespace arangodb::velocypack; @@ -840,13 +841,14 @@ uint8_t* Builder::set(ValuePair const& pair) { void Builder::checkAttributeUniqueness(Slice const& obj) const { VELOCYPACK_ASSERT(options->checkAttributeUniqueness == true); - ValueLength const n = obj.length(); if (obj.isSorted()) { // object attributes are sorted Slice previous = obj.keyAt(0); ValueLength len; char const* p = previous.getString(len); + + ValueLength const n = obj.length(); // compare each two adjacent attribute names for (ValueLength i = 1; i < n; ++i) { @@ -866,17 +868,17 @@ void Builder::checkAttributeUniqueness(Slice const& obj) const { p = q; } } else { - std::unordered_set keys; + std::unordered_set keys; + ObjectIterator it(obj, true); - for (ValueLength i = 0; i < n; ++i) { - // note: keyAt() already translates integer attributes - Slice key = obj.keyAt(i); - // keyAt() guarantees a string as returned type + while (it.valid()) { + Slice const key = it.key(true); + // key() guarantees a string as returned type VELOCYPACK_ASSERT(key.isString()); - - if (!keys.emplace(key.copyString()).second) { + if (!keys.emplace(StringRef(key)).second) { throw Exception(Exception::DuplicateAttributeName); } + it.next(); } } } diff --git a/arangod/Agency/AgencyComm.cpp b/arangod/Agency/AgencyComm.cpp index 4a797eb08d..fd4d2c4e2e 100644 --- a/arangod/Agency/AgencyComm.cpp +++ b/arangod/Agency/AgencyComm.cpp @@ -45,6 +45,8 @@ #include "SimpleHttpClient/SimpleHttpClient.h" #include "SimpleHttpClient/SimpleHttpResult.h" +#include + using namespace arangodb; using namespace arangodb::application_features; using namespace arangodb::httpclient; @@ -297,7 +299,7 @@ int AgencyCommResult::httpCode() const { return _statusCode; } int AgencyCommResult::errorCode() const { try { std::shared_ptr bodyBuilder = - VPackParser::fromJson(_body.c_str()); + VPackParser::fromJson(_body); VPackSlice body = bodyBuilder->slice(); if (!body.isObject()) { return 0; @@ -322,7 +324,7 @@ std::string AgencyCommResult::errorMessage() const { try { std::shared_ptr bodyBuilder = - VPackParser::fromJson(_body.c_str()); + VPackParser::fromJson(_body); VPackSlice body = bodyBuilder->slice(); if (!body.isObject()) { @@ -367,7 +369,7 @@ VPackSlice AgencyCommResult::slice() const { return _vpack->slice(); } std::unique_ptr AgencyCommManager::MANAGER; AgencyConnectionOptions -AgencyCommManager::CONNECTION_OPTIONS (15.0, 120.0, 120.0, 10); +AgencyCommManager::CONNECTION_OPTIONS (2.0, 15.0, 15.0, 5); void AgencyCommManager::initialize(std::string const& prefix) { MANAGER.reset(new AgencyCommManager(prefix)); @@ -678,7 +680,7 @@ std::string AgencyComm::version() { AgencyCommResult result = sendWithFailover(arangodb::rest::RequestType::GET, AgencyCommManager::CONNECTION_OPTIONS._requestTimeout, - "/_api/version", "", false); + "/_api/version", ""); if (result.successful()) { return result._body; @@ -752,14 +754,14 @@ AgencyCommResult AgencyComm::getValues(std::string const& key) { AgencyCommResult result = sendWithFailover(arangodb::rest::RequestType::POST, AgencyCommManager::CONNECTION_OPTIONS._requestTimeout, - url, builder.toJson(), false); + url, builder.toJson()); if (!result.successful()) { return result; } try { - result.setVPack(VPackParser::fromJson(result.body().c_str())); + result.setVPack(VPackParser::fromJson(result.bodyRef())); if (!result.slice().isArray()) { result._statusCode = 500; @@ -774,7 +776,7 @@ AgencyCommResult AgencyComm::getValues(std::string const& key) { result._body.clear(); result._statusCode = 200; - } catch (std::exception& e) { + } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCYCOMM) << "Error transforming result. " << e.what(); result.clear(); @@ -997,10 +999,10 @@ AgencyCommResult AgencyComm::sendTransactionWithFailover( arangodb::rest::RequestType::POST, (timeout == 0.0 ? AgencyCommManager::CONNECTION_OPTIONS._requestTimeout : timeout), - url, builder.slice().toJson(), false); + url, builder.slice().toJson()); try { - result.setVPack(VPackParser::fromJson(result.body().c_str())); + result.setVPack(VPackParser::fromJson(result.bodyRef())); if (!transaction.validate(result)) { result._statusCode = 500; @@ -1009,7 +1011,7 @@ AgencyCommResult AgencyComm::sendTransactionWithFailover( result._body.clear(); - } catch (std::exception& e) { + } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCYCOMM) << "Error transforming result. " << e.what(); result.clear(); @@ -1180,57 +1182,64 @@ bool AgencyComm::unlock(std::string const& key, VPackSlice const& slice, AgencyCommResult AgencyComm::sendWithFailover( arangodb::rest::RequestType method, double const timeout, - std::string const& initialUrl, std::string const& body, bool isWatch) { + std::string const& initialUrl, std::string const& body) { + std::string endpoint; std::unique_ptr connection = - AgencyCommManager::MANAGER->acquire(endpoint); + AgencyCommManager::MANAGER->acquire(endpoint); AgencyCommResult result; std::string url = initialUrl; + std::chrono::duration waitInterval (.25); // seconds + auto timeOut = std::chrono::steady_clock::now() + + std::chrono::duration(timeout); + double conTimeout = 1.0; + for (uint64_t tries = 0; tries < MAX_TRIES; ++tries) { + + auto waitUntil = std::chrono::steady_clock::now() + waitInterval; + if (waitInterval.count() < 5.0) { + waitInterval *= 2; + } + if (connection == nullptr) { AgencyCommResult result(400, "No endpoints for agency found."); LOG_TOPIC(ERR, Logger::AGENCYCOMM) << result._message; return result; } - + if (1 < tries) { LOG_TOPIC(WARN, Logger::AGENCYCOMM) - << "Retrying agency communication at '" << endpoint - << "', tries: " << tries; + << "Retrying agency communication at '" << endpoint + << "', tries: " << tries; } - + // try to send; if we fail completely, do not retry try { - result = send(connection.get(), method, timeout, url, body); + result = send(connection.get(), method, conTimeout, url, body); } catch (...) { AgencyCommManager::MANAGER->failed(std::move(connection), endpoint); - - result = AgencyCommResult(0, "could not send request to agency"); - result._connected = false; - - break; + connection = AgencyCommManager::MANAGER->acquire(endpoint); + + continue; } - + // got a result, we are done if (result.successful()) { AgencyCommManager::MANAGER->release(std::move(connection), endpoint); break; } - + // break on a watch timeout (drop connection) if (result._statusCode == 0) { AgencyCommManager::MANAGER->failed(std::move(connection), endpoint); - - if (isWatch) { - break; - } else { - connection = AgencyCommManager::MANAGER->acquire(endpoint); - continue; - } + + connection = AgencyCommManager::MANAGER->acquire(endpoint); + continue; + } - + // sometimes the agency will return a 307 (temporary redirect) // in this case we have to pick it up and use the new location returned if (result._statusCode == @@ -1240,15 +1249,27 @@ AgencyCommResult AgencyComm::sendWithFailover( connection = AgencyCommManager::MANAGER->acquire(endpoint); continue; } - + // do not retry on client errors if (result._statusCode >= 400 && result._statusCode <= 499) { AgencyCommManager::MANAGER->release(std::move(connection), endpoint); break; } - + AgencyCommManager::MANAGER->failed(std::move(connection), endpoint); connection = AgencyCommManager::MANAGER->acquire(endpoint); + + if (std::chrono::steady_clock::now() < timeOut) { + std::this_thread::sleep_for(waitUntil-std::chrono::steady_clock::now()); + } else { + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) + << "Unsuccessful AgencyComm: Timeout" + << "errorCode: " << result.errorCode() + << " errorMessage: " << result.errorMessage() + << " errorDetails: " << result.errorDetails(); + return result; + } + } if (!result.successful() && result.httpCode() != 412) { diff --git a/arangod/Agency/AgencyComm.h b/arangod/Agency/AgencyComm.h index 7202f59749..7125d2be73 100644 --- a/arangod/Agency/AgencyComm.h +++ b/arangod/Agency/AgencyComm.h @@ -233,6 +233,7 @@ class AgencyCommResult { std::string const location() const { return _location; } std::string const body() const { return _body; } + std::string const& bodyRef() const { return _body; } void clear(); @@ -541,8 +542,7 @@ class AgencyComm { bool unlock(std::string const&, arangodb::velocypack::Slice const&, double); AgencyCommResult sendWithFailover(arangodb::rest::RequestType, double, - std::string const&, std::string const&, - bool); + std::string const&, std::string const&); AgencyCommResult send(httpclient::GeneralClientConnection*, rest::RequestType, double, std::string const&, std::string const&); diff --git a/arangod/Agency/FailedLeader.cpp b/arangod/Agency/FailedLeader.cpp index 232656dc1c..dc9caad1d6 100644 --- a/arangod/Agency/FailedLeader.cpp +++ b/arangod/Agency/FailedLeader.cpp @@ -61,7 +61,7 @@ FailedLeader::~FailedLeader() {} bool FailedLeader::create() { LOG_TOPIC(INFO, Logger::AGENCY) - << "Todo: failed Leader for " + _shard + " from " + _from + " to " + _to; + << "Handle failed Leader for " + _shard + " from " + _from + " to " + _to; std::string path = _agencyPrefix + toDoPrefix + _jobId; @@ -114,10 +114,9 @@ bool FailedLeader::start() { Node const& current = _snapshot(curPath); if (current.slice().length() == 1) { - LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to change leadership for shard " + - _shard + " from " + _from + " to " + - _to + ". No in-sync followers:" + - current.slice().toJson(); + LOG_TOPIC(ERR, Logger::AGENCY) + << "Failed to change leadership for shard " + _shard + " from " + _from + + " to " + _to + ". No in-sync followers:" + current.slice().toJson(); return false; } @@ -130,15 +129,15 @@ bool FailedLeader::start() { try { _snapshot(toDoPrefix + _jobId).toBuilder(todo); } catch (std::exception const&) { - LOG_TOPIC(INFO, Logger::AGENCY) << "Failed to get key " + toDoPrefix + - _jobId + " from agency snapshot"; + LOG_TOPIC(INFO, Logger::AGENCY) + << "Failed to get key " + toDoPrefix + _jobId + " from agency snapshot"; return false; } } else { todo.add(_jb->slice().get(_agencyPrefix + toDoPrefix + _jobId).valueAt(0)); } todo.close(); - + // Transaction pending.openArray(); @@ -201,11 +200,11 @@ bool FailedLeader::start() { write_ret_t res = transact(_agent, pending); if (res.accepted && res.indices.size() == 1 && res.indices[0]) { - LOG_TOPIC(INFO, Logger::AGENCY) << "Pending: Change leadership " + _shard + - " from " + _from + " to " + _to; + LOG_TOPIC(DEBUG, Logger::AGENCY) + << "Pending: Change leadership " + _shard + " from " + _from + " to " + _to; return true; } - + LOG_TOPIC(INFO, Logger::AGENCY) << "Precondition failed for starting job " + _jobId; return false; diff --git a/arangod/Agency/FailedServer.cpp b/arangod/Agency/FailedServer.cpp index d94156d4c2..107aedd89e 100644 --- a/arangod/Agency/FailedServer.cpp +++ b/arangod/Agency/FailedServer.cpp @@ -55,19 +55,19 @@ FailedServer::~FailedServer() {} bool FailedServer::start() { LOG_TOPIC(INFO, Logger::AGENCY) - << "Trying to start FailedServer job" + _jobId + " for server " + _server; + << "Start FailedServer job" + _jobId + " for server " + _server; // Copy todo to pending Builder todo, pending; - + // Get todo entry todo.openArray(); if (_jb == nullptr) { try { _snapshot(toDoPrefix + _jobId).toBuilder(todo); } catch (std::exception const&) { - LOG_TOPIC(INFO, Logger::AGENCY) << "Failed to get key " + toDoPrefix + - _jobId + " from agency snapshot"; + LOG_TOPIC(INFO, Logger::AGENCY) + << "Failed to get key " + toDoPrefix + _jobId + " from agency snapshot"; return false; } } else { @@ -118,7 +118,7 @@ bool FailedServer::start() { write_ret_t res = transact(_agent, pending); if (res.accepted && res.indices.size() == 1 && res.indices[0]) { - LOG_TOPIC(INFO, Logger::AGENCY) + LOG_TOPIC(DEBUG, Logger::AGENCY) << "Pending job for failed DB Server " << _server; auto const& databases = _snapshot("/Plan/Collections").children(); @@ -206,7 +206,7 @@ bool FailedServer::start() { } bool FailedServer::create() { - LOG_TOPIC(INFO, Logger::AGENCY) << "Todo: DB Server " + _server + " failed."; + LOG_TOPIC(DEBUG, Logger::AGENCY) << "Todo: Handle failed db server " + _server; std::string path = _agencyPrefix + toDoPrefix + _jobId; diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index a265b19ee9..911a054c8e 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -301,8 +301,10 @@ void Inception::reportIn(query_t const& query) { MUTEX_LOCKER(lock, _mLock); _measurements.push_back( std::vector( - {slice.get("mean").getNumber(), slice.get("stdev").getNumber(), - slice.get("max").getNumber(), slice.get("min").getNumber()} )); + {slice.get("mean").getNumber(), + slice.get("stdev").getNumber(), + slice.get("max").getNumber(), + slice.get("min").getNumber()} )); } @@ -445,12 +447,12 @@ bool Inception::estimateRAFTInterval() { double precision = 1.0e-2; mn = precision * - std::ceil((1. / precision)*(.5 + precision * (maxmean + 3.*maxstdev))); + std::ceil((1. / precision)*(1. + precision * (maxmean + 3.*maxstdev))); if (config.waitForSync()) { mn *= 4.; } - if (mn > 2.0) { - mn = 2.0; + if (mn > 5.0) { + mn = 5.0; } mx = 5. * mn; diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index d073943cb3..c716f2518d 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -257,6 +257,7 @@ SET(ARANGOD_SOURCES RestServer/DatabaseFeature.cpp RestServer/DatabasePathFeature.cpp RestServer/EndpointFeature.cpp + RestServer/FeatureCacheFeature.cpp RestServer/FileDescriptorsFeature.cpp RestServer/FrontendFeature.cpp RestServer/InitDatabaseFeature.cpp diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index 7a01ea0e61..b38206147d 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -169,6 +169,7 @@ void HeartbeatThread::runDBServer() { // we check Current/Version every few heartbeats: int const currentCountStart = 1; // set to 1 by Max to speed up discovery int currentCount = currentCountStart; +// size_t hearbeats = 0; while (!isStopping()) { try { diff --git a/arangod/GeneralServer/GeneralServerFeature.cpp b/arangod/GeneralServer/GeneralServerFeature.cpp index af44df07b4..dd1e643e0b 100644 --- a/arangod/GeneralServer/GeneralServerFeature.cpp +++ b/arangod/GeneralServer/GeneralServerFeature.cpp @@ -70,6 +70,7 @@ #include "RestHandler/WorkMonitorHandler.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/EndpointFeature.h" +#include "RestServer/FeatureCacheFeature.h" #include "RestServer/QueryRegistryFeature.h" #include "RestServer/ServerFeature.h" #include "RestServer/TraverserEngineRegistryFeature.h" @@ -183,9 +184,7 @@ void GeneralServerFeature::validateOptions(std::shared_ptr) { } static TRI_vocbase_t* LookupDatabaseFromRequest(GeneralRequest* request) { - auto databaseFeature = - application_features::ApplicationServer::getFeature( - "Database"); + auto databaseFeature = FeatureCacheFeature::instance()->databaseFeature(); // get database name from request std::string const& dbName = request->databaseName(); @@ -208,8 +207,7 @@ static TRI_vocbase_t* LookupDatabaseFromRequest(GeneralRequest* request) { } static bool SetRequestContext(GeneralRequest* request, void* data) { - auto authentication = application_features::ApplicationServer::getFeature< - AuthenticationFeature>("Authentication"); + auto authentication = FeatureCacheFeature::instance()->authenticationFeature(); TRI_ASSERT(authentication != nullptr); TRI_vocbase_t* vocbase = LookupDatabaseFromRequest(request); @@ -254,8 +252,7 @@ void GeneralServerFeature::start() { // populate the authentication cache. otherwise no one can access the new // database - auto authentication = application_features::ApplicationServer::getFeature< - AuthenticationFeature>("Authentication"); + auto authentication = FeatureCacheFeature::instance()->authenticationFeature(); TRI_ASSERT(authentication != nullptr); if (authentication->isEnabled()) { authentication->authInfo()->outdate(); diff --git a/arangod/GeneralServer/HttpCommTask.cpp b/arangod/GeneralServer/HttpCommTask.cpp index 65bc77fbef..51120d9490 100644 --- a/arangod/GeneralServer/HttpCommTask.cpp +++ b/arangod/GeneralServer/HttpCommTask.cpp @@ -528,7 +528,7 @@ bool HttpCommTask::processRead(double startTime) { LOG(DEBUG) << "no keep-alive, connection close requested by client"; _closeRequested = true; - } else if (!_useKeepAliveTimeout) { + } else if (!_useKeepAliveTimer) { // if keepAliveTimeout was set to 0.0, we'll close even keep-alive // connections immediately LOG(DEBUG) << "keep-alive disabled by admin"; diff --git a/arangod/RestServer/FeatureCacheFeature.cpp b/arangod/RestServer/FeatureCacheFeature.cpp new file mode 100644 index 0000000000..f89015bfe3 --- /dev/null +++ b/arangod/RestServer/FeatureCacheFeature.cpp @@ -0,0 +1,56 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2016 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +//////////////////////////////////////////////////////////////////////////////// + +#include "FeatureCacheFeature.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "GeneralServer/AuthenticationFeature.h" +#include "RestServer/DatabaseFeature.h" + +using namespace arangodb; +using namespace arangodb::application_features; + +FeatureCacheFeature* FeatureCacheFeature::Instance = nullptr; + +FeatureCacheFeature::FeatureCacheFeature(application_features::ApplicationServer* server) + : ApplicationFeature(server, "FeatureCache"), + _authenticationFeature(nullptr), + _databaseFeature(nullptr) { + setOptional(false); + requiresElevatedPrivileges(false); +} + +void FeatureCacheFeature::prepare() { + _authenticationFeature = application_features::ApplicationServer::getFeature( + "Authentication"); + _databaseFeature = application_features::ApplicationServer::getFeature( + "Database"); + + TRI_ASSERT(Instance == nullptr); + Instance = this; +} + +void FeatureCacheFeature::unprepare() { + _authenticationFeature = nullptr; + _databaseFeature = nullptr; + // do not reset instance +} diff --git a/arangod/RestServer/FeatureCacheFeature.h b/arangod/RestServer/FeatureCacheFeature.h new file mode 100644 index 0000000000..48f18da117 --- /dev/null +++ b/arangod/RestServer/FeatureCacheFeature.h @@ -0,0 +1,63 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2016 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +//////////////////////////////////////////////////////////////////////////////// + +#ifndef REST_SERVER_FEATURE_CACHE_FEATURE_H +#define REST_SERVER_FEATURE_CACHE_FEATURE_H 1 + +#include "ApplicationFeatures/ApplicationFeature.h" + +namespace arangodb { +class AuthenticationFeature; +class DatabaseFeature; + +class FeatureCacheFeature final : public application_features::ApplicationFeature { + public: + explicit FeatureCacheFeature(application_features::ApplicationServer*); + + public: + void prepare() override final; + void unprepare() override final; + + static inline FeatureCacheFeature* instance() { + TRI_ASSERT(Instance != nullptr); + return Instance; + } + + inline AuthenticationFeature* authenticationFeature() const { + TRI_ASSERT(_authenticationFeature != nullptr); + return _authenticationFeature; + } + + inline DatabaseFeature* databaseFeature() const { + TRI_ASSERT(_databaseFeature != nullptr); + return _databaseFeature; + } + + private: + static FeatureCacheFeature* Instance; + + AuthenticationFeature* _authenticationFeature; + DatabaseFeature* _databaseFeature; +}; +} + +#endif diff --git a/arangod/RestServer/ServerFeature.h b/arangod/RestServer/ServerFeature.h index b9dc65ef11..60a121a197 100644 --- a/arangod/RestServer/ServerFeature.h +++ b/arangod/RestServer/ServerFeature.h @@ -33,6 +33,8 @@ class RestHandlerFactory; class AsyncJobManager; } +class FeatureCacheFeature; + class ServerFeature final : public application_features::ApplicationFeature { public: static std::string operationModeString(OperationMode mode); @@ -56,6 +58,9 @@ class ServerFeature final : public application_features::ApplicationFeature { std::vector const& scripts() const { return _scripts; } std::vector const& unitTests() const { return _unitTests; } uint32_t const& vppMaxSize() const { return _vppMaxSize; } + + private: + void waitForHeartbeat(); private: bool _console = false; @@ -63,11 +68,6 @@ class ServerFeature final : public application_features::ApplicationFeature { std::vector _unitTests; std::vector _scripts; uint32_t _vppMaxSize; - - private: - void waitForHeartbeat(); - - private: int* _result; OperationMode _operationMode; }; diff --git a/arangod/RestServer/VocbaseContext.cpp b/arangod/RestServer/VocbaseContext.cpp index b3616942be..d3d7b8d213 100644 --- a/arangod/RestServer/VocbaseContext.cpp +++ b/arangod/RestServer/VocbaseContext.cpp @@ -34,6 +34,7 @@ #include "Endpoint/ConnectionInfo.h" #include "GeneralServer/AuthenticationFeature.h" #include "Logger/Logger.h" +#include "RestServer/FeatureCacheFeature.h" #include "Ssl/SslInterface.h" #include "Utils/Events.h" #include "VocBase/AuthInfo.h" @@ -51,9 +52,7 @@ VocbaseContext::VocbaseContext(GeneralRequest* request, TRI_vocbase_t* vocbase) _vocbase(vocbase), _authentication(nullptr) { TRI_ASSERT(_vocbase != nullptr); - _authentication = - application_features::ApplicationServer::getFeature( - "Authentication"); + _authentication = FeatureCacheFeature::instance()->authenticationFeature(); TRI_ASSERT(_authentication != nullptr); } diff --git a/arangod/RestServer/arangod.cpp b/arangod/RestServer/arangod.cpp index c13440ce88..4c5632e87e 100644 --- a/arangod/RestServer/arangod.cpp +++ b/arangod/RestServer/arangod.cpp @@ -54,6 +54,7 @@ #include "RestServer/DatabaseFeature.h" #include "RestServer/DatabasePathFeature.h" #include "RestServer/EndpointFeature.h" +#include "RestServer/FeatureCacheFeature.h" #include "RestServer/FileDescriptorsFeature.h" #include "RestServer/FrontendFeature.h" #include "RestServer/InitDatabaseFeature.h" @@ -131,6 +132,7 @@ static int runServer(int argc, char** argv) { server.addFeature(new DatabasePathFeature(&server)); server.addFeature(new EndpointFeature(&server)); server.addFeature(new EngineSelectorFeature(&server)); + server.addFeature(new FeatureCacheFeature(&server)); server.addFeature(new FileDescriptorsFeature(&server)); server.addFeature(new FoxxQueuesFeature(&server)); server.addFeature(new FrontendFeature(&server)); diff --git a/arangod/Scheduler/SocketTask.cpp b/arangod/Scheduler/SocketTask.cpp index 58b843d677..3c4116de90 100644 --- a/arangod/Scheduler/SocketTask.cpp +++ b/arangod/Scheduler/SocketTask.cpp @@ -50,8 +50,10 @@ SocketTask::SocketTask(arangodb::EventLoop loop, _readBuffer(TRI_UNKNOWN_MEM_ZONE, READ_BLOCK_SIZE + 1, false), _peer(std::move(socket)), _keepAliveTimeout(static_cast(keepAliveTimeout * 1000)), - _useKeepAliveTimeout(static_cast(keepAliveTimeout * 1000) > 0), _keepAliveTimer(_peer->_ioService, _keepAliveTimeout), + _useKeepAliveTimer(keepAliveTimeout > 0.0), + _keepAliveTimerActive(false), + _closeRequested(false), _abandoned(false) { ConnectionStatisticsAgent::acquire(); connectionStatisticsAgentSetStart(); @@ -71,11 +73,13 @@ SocketTask::SocketTask(arangodb::EventLoop loop, SocketTask::~SocketTask() { boost::system::error_code err; - _keepAliveTimer.cancel(err); + if (_keepAliveTimerActive) { + _keepAliveTimer.cancel(err); + } if (err) { LOG_TOPIC(ERR, Logger::COMMUNICATION) << "unable to cancel _keepAliveTimer"; - } + } } void SocketTask::start() { @@ -278,6 +282,7 @@ void SocketTask::closeStream() { _closeRequested = false; _keepAliveTimer.cancel(); + _keepAliveTimerActive = false; } // ----------------------------------------------------------------------------- @@ -289,7 +294,7 @@ void SocketTask::addToReadBuffer(char const* data, std::size_t len) { } void SocketTask::resetKeepAlive() { - if (_useKeepAliveTimeout) { + if (_useKeepAliveTimer) { boost::system::error_code err; _keepAliveTimer.expires_from_now(_keepAliveTimeout, err); @@ -298,6 +303,7 @@ void SocketTask::resetKeepAlive() { return; } + _keepAliveTimerActive = true; auto self = shared_from_this(); _keepAliveTimer.async_wait( @@ -314,9 +320,10 @@ void SocketTask::resetKeepAlive() { } void SocketTask::cancelKeepAlive() { - if (_useKeepAliveTimeout) { + if (_useKeepAliveTimer && _keepAliveTimerActive) { boost::system::error_code err; _keepAliveTimer.cancel(err); + _keepAliveTimerActive = false; } } diff --git a/arangod/Scheduler/SocketTask.h b/arangod/Scheduler/SocketTask.h index ce38070053..60eb422d66 100644 --- a/arangod/Scheduler/SocketTask.h +++ b/arangod/Scheduler/SocketTask.h @@ -101,10 +101,10 @@ class SocketTask : virtual public Task, public ConnectionStatisticsAgent { std::unique_ptr _peer; boost::posix_time::milliseconds _keepAliveTimeout; - bool _useKeepAliveTimeout; boost::asio::deadline_timer _keepAliveTimer; - - bool _closeRequested = false; + bool const _useKeepAliveTimer; + bool _keepAliveTimerActive; + bool _closeRequested; std::atomic_bool _abandoned; private: diff --git a/arangod/Utils/DatabaseGuard.h b/arangod/Utils/DatabaseGuard.h index cacac7640b..eafa5f8107 100644 --- a/arangod/Utils/DatabaseGuard.h +++ b/arangod/Utils/DatabaseGuard.h @@ -27,6 +27,7 @@ #include "ApplicationFeatures/ApplicationServer.h" #include "Basics/Exceptions.h" #include "RestServer/DatabaseFeature.h" +#include "RestServer/FeatureCacheFeature.h" struct TRI_vocbase_t; @@ -44,7 +45,7 @@ class DatabaseGuard { explicit DatabaseGuard(TRI_voc_tick_t id) : _vocbase(nullptr) { - auto databaseFeature = application_features::ApplicationServer::getFeature("Database"); + auto databaseFeature = FeatureCacheFeature::instance()->databaseFeature(); _vocbase = databaseFeature->useDatabase(id); if (_vocbase == nullptr) { @@ -59,7 +60,7 @@ class DatabaseGuard { explicit DatabaseGuard(std::string const& name) : _vocbase(nullptr) { - auto databaseFeature = application_features::ApplicationServer::getFeature("Database"); + auto databaseFeature = FeatureCacheFeature::instance()->databaseFeature(); _vocbase = databaseFeature->useDatabase(name); if (_vocbase == nullptr) { diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index 2417cc21b1..5954f5f48f 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -1298,7 +1298,7 @@ void TRI_SanitizeObject(VPackSlice const slice, VPackBuilder& builder) { /// open object which will remain open. also excludes _from and _to void TRI_SanitizeObjectWithEdges(VPackSlice const slice, VPackBuilder& builder) { TRI_ASSERT(slice.isObject()); - VPackObjectIterator it(slice); + VPackObjectIterator it(slice, true); while (it.valid()) { StringRef key(it.key()); if (key.empty() || key[0] != '_' || diff --git a/arangod/Wal/LogfileManager.cpp b/arangod/Wal/LogfileManager.cpp index f7cb6f9767..bde505a563 100644 --- a/arangod/Wal/LogfileManager.cpp +++ b/arangod/Wal/LogfileManager.cpp @@ -112,6 +112,7 @@ LogfileManager::LogfileManager(ApplicationServer* server) requiresElevatedPrivileges(false); startsAfter("DatabasePath"); startsAfter("EngineSelector"); + startsAfter("FeatureCache"); startsAfter("RevisionCache"); for (auto const& it : EngineSelectorFeature::availableEngines()) { diff --git a/js/server/tests/resilience/moving-shards-cluster.js b/js/server/tests/resilience/moving-shards-cluster.js index b42e8d720a..c70cc3a72f 100644 --- a/js/server/tests/resilience/moving-shards-cluster.js +++ b/js/server/tests/resilience/moving-shards-cluster.js @@ -305,9 +305,14 @@ function MovingShardsSuite () { Object.keys(state.Pending).length === 0) { return true; } - console.info("Waiting for supervision jobs to finish:", - "ToDo jobs:", Object.keys(state.ToDo).length, - "Pending jobs:", Object.keys(state.Pending).length); + if (state.error) { + console.warn("Waiting for supervision jobs to finish:", + "Currently no agency communication possible."); + } else { + console.info("Waiting for supervision jobs to finish:", + "ToDo jobs:", Object.keys(state.ToDo).length, + "Pending jobs:", Object.keys(state.Pending).length); + } wait(1.0); } return false; diff --git a/lib/Rest/GeneralRequest.h b/lib/Rest/GeneralRequest.h index f6b07c5799..a275483e55 100644 --- a/lib/Rest/GeneralRequest.h +++ b/lib/Rest/GeneralRequest.h @@ -175,9 +175,7 @@ class GeneralRequest { std::shared_ptr toVelocyPackBuilderPtr( arangodb::velocypack::Options const* options) { - auto rv = std::make_shared(); - rv->add(payload(options)); - return rv; + return std::make_shared(payload(options), options); }; ContentType contentType() const { return _contentType; } diff --git a/lib/Rest/HttpRequest.cpp b/lib/Rest/HttpRequest.cpp index c4b77baa01..2bc628a164 100644 --- a/lib/Rest/HttpRequest.cpp +++ b/lib/Rest/HttpRequest.cpp @@ -476,7 +476,7 @@ void HttpRequest::setValues(char* buffer, char* end) { *(key - 2) = '\0'; setArrayValue(keyBegin, key - keyBegin - 2, valueBegin); } else { - _values[std::string(keyBegin, key - keyBegin)] = valueBegin; + _values[std::string(keyBegin, key - keyBegin)] = std::string(valueBegin, value - valueBegin); } keyBegin = key = buffer + 1; @@ -535,7 +535,7 @@ void HttpRequest::setValues(char* buffer, char* end) { *(key - 2) = '\0'; setArrayValue(keyBegin, key - keyBegin - 2, valueBegin); } else { - _values[std::string(keyBegin, key - keyBegin)] = valueBegin; + _values[std::string(keyBegin, key - keyBegin)] = std::string(valueBegin, value - valueBegin); } } } @@ -742,21 +742,19 @@ VPackSlice HttpRequest::payload(VPackOptions const* options) { // check options for nullptr? if (_contentType == ContentType::JSON) { - if (body().length() > 0) { + if (!_body.empty()) { if (_vpackBuilder == nullptr) { VPackParser parser(options); - parser.parse(body()); + parser.parse(_body); _vpackBuilder = parser.steal(); - return VPackSlice(_vpackBuilder->slice()); - } else { - return VPackSlice(_vpackBuilder->slice()); } + return VPackSlice(_vpackBuilder->slice()); } return VPackSlice::noneSlice(); // no body } else /*VPACK*/ { VPackValidator validator; - validator.validate(body().c_str(), body().length()); - return VPackSlice(body().c_str()); + validator.validate(_body.c_str(), _body.length()); + return VPackSlice(_body.c_str()); } } diff --git a/scripts/startStandAloneAgency.sh b/scripts/startStandAloneAgency.sh index 2b7c090ba2..74e5150670 100755 --- a/scripts/startStandAloneAgency.sh +++ b/scripts/startStandAloneAgency.sh @@ -158,7 +158,7 @@ if [ "$GOSSIP_MODE" = "0" ]; then GOSSIP_PEERS=" --agency.endpoint $TRANSPORT://localhost:$BASE" fi -#rm -rf agency +rm -rf agency mkdir -p agency PIDS=""