diff --git a/arangod/Agency/AddFollower.cpp b/arangod/Agency/AddFollower.cpp index 765232b347..4470c9d67e 100644 --- a/arangod/Agency/AddFollower.cpp +++ b/arangod/Agency/AddFollower.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2018-2018 ArangoDB GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. diff --git a/arangod/Agency/AddFollower.h b/arangod/Agency/AddFollower.h index 907ef466a4..5d993ede6c 100644 --- a/arangod/Agency/AddFollower.h +++ b/arangod/Agency/AddFollower.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgencyComm.cpp b/arangod/Agency/AgencyComm.cpp index 1920fb31ee..9699e9d324 100644 --- a/arangod/Agency/AgencyComm.cpp +++ b/arangod/Agency/AgencyComm.cpp @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgencyComm.h b/arangod/Agency/AgencyComm.h index 9005abcb73..3b9f566260 100644 --- a/arangod/Agency/AgencyComm.h +++ b/arangod/Agency/AgencyComm.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgencyCommon.h b/arangod/Agency/AgencyCommon.h index db23c279c9..82fcf81a52 100644 --- a/arangod/Agency/AgencyCommon.h +++ b/arangod/Agency/AgencyCommon.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. diff --git a/arangod/Agency/AgencyFeature.cpp b/arangod/Agency/AgencyFeature.cpp index 9f8ce6b4f1..a9d08b91c5 100644 --- a/arangod/Agency/AgencyFeature.cpp +++ b/arangod/Agency/AgencyFeature.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgencyFeature.h b/arangod/Agency/AgencyFeature.h index 8b990ed04a..5a8b61242a 100644 --- a/arangod/Agency/AgencyFeature.h +++ b/arangod/Agency/AgencyFeature.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. diff --git a/arangod/Agency/Agent.h b/arangod/Agency/Agent.h index d0b27358cc..d2e8150c88 100644 --- a/arangod/Agency/Agent.h +++ b/arangod/Agency/Agent.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgentCallback.cpp b/arangod/Agency/AgentCallback.cpp index 611b5d1cdf..179c7ac76b 100644 --- a/arangod/Agency/AgentCallback.cpp +++ b/arangod/Agency/AgentCallback.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgentCallback.h b/arangod/Agency/AgentCallback.h index 594c0ec00c..0ea8ebcfc1 100644 --- a/arangod/Agency/AgentCallback.h +++ b/arangod/Agency/AgentCallback.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/AgentConfiguration.h b/arangod/Agency/AgentConfiguration.h index 63e9683ee2..52cfebe746 100644 --- a/arangod/Agency/AgentConfiguration.h +++ b/arangod/Agency/AgentConfiguration.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/CleanOutServer.h b/arangod/Agency/CleanOutServer.h index 3f46802342..70caa24697 100644 --- a/arangod/Agency/CleanOutServer.h +++ b/arangod/Agency/CleanOutServer.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Compactor.cpp b/arangod/Agency/Compactor.cpp index 3bb6de8149..348f52df42 100644 --- a/arangod/Agency/Compactor.cpp +++ b/arangod/Agency/Compactor.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Compactor.h b/arangod/Agency/Compactor.h index 618cb85c5b..ba72240de4 100644 --- a/arangod/Agency/Compactor.h +++ b/arangod/Agency/Compactor.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Constituent.cpp b/arangod/Agency/Constituent.cpp index 733015a6c7..97d538cc53 100644 --- a/arangod/Agency/Constituent.cpp +++ b/arangod/Agency/Constituent.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Constituent.h b/arangod/Agency/Constituent.h index 90fbd186d5..acff171262 100644 --- a/arangod/Agency/Constituent.h +++ b/arangod/Agency/Constituent.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/FailedFollower.h b/arangod/Agency/FailedFollower.h index df6265cb7f..b1ca424f1c 100644 --- a/arangod/Agency/FailedFollower.h +++ b/arangod/Agency/FailedFollower.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/FailedLeader.h b/arangod/Agency/FailedLeader.h index 4cb6b1437a..4b60739a2f 100644 --- a/arangod/Agency/FailedLeader.h +++ b/arangod/Agency/FailedLeader.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/FailedServer.h b/arangod/Agency/FailedServer.h index 832f0eacd6..0768cbd821 100644 --- a/arangod/Agency/FailedServer.h +++ b/arangod/Agency/FailedServer.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/GossipCallback.cpp b/arangod/Agency/GossipCallback.cpp index f36159d0ea..9c53fcc42e 100644 --- a/arangod/Agency/GossipCallback.cpp +++ b/arangod/Agency/GossipCallback.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/GossipCallback.h b/arangod/Agency/GossipCallback.h index 2ff0d3a032..120043698c 100644 --- a/arangod/Agency/GossipCallback.h +++ b/arangod/Agency/GossipCallback.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 2c31acbeaf..98867c0fea 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -239,9 +239,12 @@ bool Inception::restartingActiveAgent() { auto comres = cc->syncRequest( clientId, 1, p, rest::RequestType::POST, path, greetstr, std::unordered_map(), 2.0); - if (comres->status == CL_COMM_SENT) { + // FIXMEMAINTENANCE: handle case of result not 200 + if (comres->status == CL_COMM_SENT) { // WARN: What if result not 200? auto const theirConfigVP = comres->result->getBodyVelocyPack(); auto const& theirConfig = theirConfigVP->slice(); + // FIXMEMAINTENANCE: handle the case that tcc is not an object such + // that the next command would throw. auto const& tcc = theirConfig.get("configuration"); auto const& theirId = tcc.get("id").copyString(); diff --git a/arangod/Agency/JobContext.cpp b/arangod/Agency/JobContext.cpp index 9ac9245d88..c3747797c2 100644 --- a/arangod/Agency/JobContext.cpp +++ b/arangod/Agency/JobContext.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/JobContext.h b/arangod/Agency/JobContext.h index 4f0c3bbdef..c1753a57c9 100644 --- a/arangod/Agency/JobContext.h +++ b/arangod/Agency/JobContext.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/MoveShard.h b/arangod/Agency/MoveShard.h index a90204461f..872fc80972 100644 --- a/arangod/Agency/MoveShard.h +++ b/arangod/Agency/MoveShard.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Node.cpp b/arangod/Agency/Node.cpp index 96afcdecd4..6b841e8578 100644 --- a/arangod/Agency/Node.cpp +++ b/arangod/Agency/Node.cpp @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Node.h b/arangod/Agency/Node.h index 5fc1edf668..995d2745a2 100644 --- a/arangod/Agency/Node.h +++ b/arangod/Agency/Node.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -276,13 +276,15 @@ class Node { // These two operator() functions could be "protected" once // unit tests updated. // -public: /// @brief Get node specified by path string Node& operator()(std::string const& path); /// @brief Get node specified by path string Node const& operator()(std::string const& path) const; + /// @brief Get string value (throws if type NODE or if conversion fails) + std::string getString() const; + // // The protected accessors are the "old" interface. They throw. // Please use the hasAsXXX replacements. @@ -303,9 +305,6 @@ protected: /// @brief Get double value (throws if type NODE or if conversion fails) double getDouble() const; - /// @brief Get string value (throws if type NODE or if conversion fails) - std::string getString() const; - /// @brief Get array value Slice getArray() const; diff --git a/arangod/Agency/NotifyCallback.cpp b/arangod/Agency/NotifyCallback.cpp index 364ff9b14f..53ec183ec4 100644 --- a/arangod/Agency/NotifyCallback.cpp +++ b/arangod/Agency/NotifyCallback.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/NotifyCallback.h b/arangod/Agency/NotifyCallback.h index ff0fa0bce0..b8192d1916 100644 --- a/arangod/Agency/NotifyCallback.h +++ b/arangod/Agency/NotifyCallback.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/RemoveFollower.cpp b/arangod/Agency/RemoveFollower.cpp index 8c94e9b3eb..9e545e6baf 100644 --- a/arangod/Agency/RemoveFollower.cpp +++ b/arangod/Agency/RemoveFollower.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2018-2018 ArangoDB GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. @@ -73,7 +73,7 @@ void RemoveFollower::run() { } bool RemoveFollower::create(std::shared_ptr envelope) { - LOG_TOPIC(INFO, Logger::SUPERVISION) << "Todo: RemoveFollower(s) " + LOG_TOPIC(DEBUG, Logger::SUPERVISION) << "Todo: RemoveFollower(s) " << " to shard " << _shard << " in collection " << _collection; bool selfCreate = (envelope == nullptr); // Do we create ourselves? @@ -373,7 +373,7 @@ bool RemoveFollower::start() { if (res.accepted && res.indices.size() == 1 && res.indices[0]) { _status = FINISHED; - LOG_TOPIC(INFO, Logger::SUPERVISION) + LOG_TOPIC(DEBUG, Logger::SUPERVISION) << "Pending: RemoveFollower(s) to shard " << _shard << " in collection " << _collection; return true; diff --git a/arangod/Agency/RemoveFollower.h b/arangod/Agency/RemoveFollower.h index 595cf8dff0..6c6a2ca791 100644 --- a/arangod/Agency/RemoveFollower.h +++ b/arangod/Agency/RemoveFollower.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/RestAgencyHandler.cpp b/arangod/Agency/RestAgencyHandler.cpp index 25a475907f..1f21862930 100644 --- a/arangod/Agency/RestAgencyHandler.cpp +++ b/arangod/Agency/RestAgencyHandler.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/RestAgencyHandler.h b/arangod/Agency/RestAgencyHandler.h index a374d3c803..682a9106c2 100644 --- a/arangod/Agency/RestAgencyHandler.h +++ b/arangod/Agency/RestAgencyHandler.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,7 +39,9 @@ class RestAgencyHandler : public RestBaseHandler { public: char const* name() const override final { return "RestAgencyHandler"; } + RequestLane lane() const override final { return RequestLane::AGENCY_CLUSTER; } + RestStatus execute() override; private: diff --git a/arangod/Agency/RestAgencyPrivHandler.cpp b/arangod/Agency/RestAgencyPrivHandler.cpp index 25a4895f3f..0b29b472e7 100644 --- a/arangod/Agency/RestAgencyPrivHandler.cpp +++ b/arangod/Agency/RestAgencyPrivHandler.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/RestAgencyPrivHandler.h b/arangod/Agency/RestAgencyPrivHandler.h index 4c6d9d6b4b..5bfa76f704 100644 --- a/arangod/Agency/RestAgencyPrivHandler.h +++ b/arangod/Agency/RestAgencyPrivHandler.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -66,7 +66,9 @@ class RestAgencyPrivHandler : public arangodb::RestBaseHandler { public: char const* name() const override final { return "RestAgencyPrivHandler"; } + RequestLane lane() const override final { return RequestLane::AGENCY_INTERNAL; } + RestStatus execute() override; private: diff --git a/arangod/Agency/State.cpp b/arangod/Agency/State.cpp index 6a2b0b22fa..f19ba08126 100644 --- a/arangod/Agency/State.cpp +++ b/arangod/Agency/State.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/State.h b/arangod/Agency/State.h index 5d220899c3..0446e20a1c 100644 --- a/arangod/Agency/State.h +++ b/arangod/Agency/State.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Store.cpp b/arangod/Agency/Store.cpp index 2f24188725..19c4d50ec0 100644 --- a/arangod/Agency/Store.cpp +++ b/arangod/Agency/Store.cpp @@ -2,7 +2,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Store.h b/arangod/Agency/Store.h index d3fb16410a..5d82ec15c8 100644 --- a/arangod/Agency/Store.h +++ b/arangod/Agency/Store.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/StoreCallback.cpp b/arangod/Agency/StoreCallback.cpp index a432edefd5..20d1af5bce 100644 --- a/arangod/Agency/StoreCallback.cpp +++ b/arangod/Agency/StoreCallback.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/StoreCallback.h b/arangod/Agency/StoreCallback.h index 98591ea78c..c825a451c5 100644 --- a/arangod/Agency/StoreCallback.h +++ b/arangod/Agency/StoreCallback.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 41c816f791..6773707208 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -154,8 +154,8 @@ Supervision::Supervision() _snapshot("Supervision"), _transient("Transient"), _frequency(1.), - _gracePeriod(5.), - _okThreshold(1.5), + _gracePeriod(10.), + _okThreshold(5.), _jobId(0), _jobIdMax(0), _selfShutdown(false), @@ -556,6 +556,54 @@ std::vector Supervision::check(std::string const& type) { return ret; } +bool Supervision::earlyBird() const { + + std::vector tpath {"Sync","ServerStates"}; + std::vector pdbpath {"Plan","DBServers"}; + std::vector pcpath {"Plan","Coordinators"}; + + if (!_snapshot.has(pdbpath)) { + LOG_TOPIC(DEBUG, Logger::SUPERVISION) + << "No Plan/DBServers key in persistent store"; + return false; + } + VPackBuilder dbserversB = _snapshot(pdbpath).toBuilder(); + VPackSlice dbservers = dbserversB.slice(); + + if (!_snapshot.has(pcpath)) { + LOG_TOPIC(DEBUG, Logger::SUPERVISION) + << "No Plan/Coordinators key in persistent store"; + return false; + } + VPackBuilder coordinatorsB = _snapshot(pcpath).toBuilder(); + VPackSlice coordinators = coordinatorsB.slice(); + + if (!_transient.has(tpath)) { + LOG_TOPIC(DEBUG, Logger::SUPERVISION) + << "No Sync/ServerStates key in transient store"; + return false; + } + VPackBuilder serverStatesB = _transient(tpath).toBuilder(); + VPackSlice serverStates = serverStatesB.slice(); + + // every db server in plan accounted for in transient store? + for (auto const& server : VPackObjectIterator(dbservers)) { + auto serverId = server.key.copyString(); + if (!serverStates.hasKey(serverId)) { + return false; + } + } + // every db server in plan accounted for in transient store? + for (auto const& server : VPackObjectIterator(coordinators)) { + auto serverId = server.key.copyString(); + if (!serverStates.hasKey(serverId)) { + return false; + } + } + + return true; +} + // Update local agency snapshot, guarded by callers bool Supervision::updateSnapshot() { @@ -698,7 +746,9 @@ void Supervision::run() { upgradeAgency(); } - if (_agent->leaderFor() > 10) { + if (_agent->leaderFor() > 55 || earlyBird()) { + // 55 seconds is less than a minute, which fits to the + // 60 seconds timeout in /_admin/cluster/health try { doChecks(); } catch (std::exception const& e) { @@ -708,6 +758,10 @@ void Supervision::run() { LOG_TOPIC(ERR, Logger::SUPERVISION) << "Supervision::doChecks() generated an uncaught exception."; } + } else { + LOG_TOPIC(INFO, Logger::SUPERVISION) + << "Postponing supervision for now, waiting for incoming " + "heartbeats: " << _agent->leaderFor(); } handleJobs(); diff --git a/arangod/Agency/Supervision.h b/arangod/Agency/Supervision.h index 7e7e8aade1..707f38cf87 100644 --- a/arangod/Agency/Supervision.h +++ b/arangod/Agency/Supervision.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,14 +24,13 @@ #ifndef ARANGOD_CONSENSUS_SUPERVISION_H #define ARANGOD_CONSENSUS_SUPERVISION_H 1 +#include "Agency/AgencyCommon.h" #include "Agency/Node.h" -#include "AgencyCommon.h" +#include "Agency/TimeString.h" #include "Basics/ConditionVariable.h" #include "Basics/Mutex.h" #include "Cluster/CriticalThread.h" -#include - namespace arangodb { namespace consensus { @@ -124,6 +123,9 @@ class Supervision : public arangodb::CriticalThread { private: + /// @brief decide, if we can start supervision ahead of armageddon delay + bool earlyBird() const; + /// @brief Upgrade agency with FailedServers an object from array void upgradeZero(VPackBuilder&); @@ -219,34 +221,6 @@ class Supervision : public arangodb::CriticalThread { */ query_t removeTransactionBuilder(std::vector const&); -inline std::string timepointToString(Supervision::TimePoint const& t) { - time_t tt = std::chrono::system_clock::to_time_t(t); - struct tm tb; - size_t const len(21); - char buffer[len]; - TRI_gmtime(tt, &tb); - ::strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%SZ", &tb); - return std::string(buffer, len - 1); -} - -inline Supervision::TimePoint stringToTimepoint(std::string const& s) { - if (!s.empty()) { - try { - std::tm tt; - tt.tm_year = std::stoi(s.substr(0, 4)) - 1900; - tt.tm_mon = std::stoi(s.substr(5, 2)) - 1; - tt.tm_mday = std::stoi(s.substr(8, 2)); - tt.tm_hour = std::stoi(s.substr(11, 2)); - tt.tm_min = std::stoi(s.substr(14, 2)); - tt.tm_sec = std::stoi(s.substr(17, 2)); - tt.tm_isdst = 0; - auto time_c = TRI_timegm(&tt); - return std::chrono::system_clock::from_time_t(time_c); - } catch (...) {} - } - return std::chrono::time_point(); -} - }} // Name spaces #endif diff --git a/arangod/Agency/TimeString.h b/arangod/Agency/TimeString.h new file mode 100644 index 0000000000..dc0bbc3358 --- /dev/null +++ b/arangod/Agency/TimeString.h @@ -0,0 +1,61 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_CONSENSUS_TIMESTRING_H +#define ARANGOD_CONSENSUS_TIMESTRING_H 1 + +#include + +inline std::string timepointToString(std::chrono::system_clock::time_point const& t) { + time_t tt = std::chrono::system_clock::to_time_t(t); + struct tm tb; + size_t const len(21); + char buffer[len]; + TRI_gmtime(tt, &tb); + ::strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%SZ", &tb); + return std::string(buffer, len - 1); +} + +inline std::string timepointToString(std::chrono::system_clock::duration const& d) { + return timepointToString(std::chrono::system_clock::time_point() + d); +} + + + inline std::chrono::system_clock::time_point stringToTimepoint(std::string const& s) { + if (!s.empty()) { + try { + std::tm tt; + tt.tm_year = std::stoi(s.substr(0, 4)) - 1900; + tt.tm_mon = std::stoi(s.substr(5, 2)) - 1; + tt.tm_mday = std::stoi(s.substr(8, 2)); + tt.tm_hour = std::stoi(s.substr(11, 2)); + tt.tm_min = std::stoi(s.substr(14, 2)); + tt.tm_sec = std::stoi(s.substr(17, 2)); + tt.tm_isdst = 0; + auto time_c = TRI_timegm(&tt); + return std::chrono::system_clock::from_time_t(time_c); + } catch (...) {} + } + return std::chrono::time_point(); +} + +#endif diff --git a/arangod/Agency/v8-agency.cpp b/arangod/Agency/v8-agency.cpp index e1f4b3d259..78c47132a9 100644 --- a/arangod/Agency/v8-agency.cpp +++ b/arangod/Agency/v8-agency.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Agency/v8-agency.h b/arangod/Agency/v8-agency.h index 250fdb2bf7..eb08a85a95 100644 --- a/arangod/Agency/v8-agency.h +++ b/arangod/Agency/v8-agency.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index befbf35ac4..41a5cd02d1 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -283,6 +283,9 @@ SET(ARANGOD_SOURCES Cache/TransactionalBucket.cpp Cache/TransactionalCache.cpp Cache/TransactionManager.cpp + Cluster/ActionBase.cpp + Cluster/Action.cpp + Cluster/ActionDescription.cpp Cluster/AgencyCallback.cpp Cluster/AgencyCallbackRegistry.cpp Cluster/ClusterComm.cpp @@ -294,17 +297,32 @@ SET(ARANGOD_SOURCES Cluster/ClusterRepairDistributeShardsLike.cpp Cluster/ClusterRepairOperations.cpp Cluster/ClusterTraverser.cpp + Cluster/CreateCollection.cpp + Cluster/CreateDatabase.cpp Cluster/CriticalThread.cpp Cluster/EngineEqualityCheckFeature.cpp Cluster/FollowerInfo.cpp Cluster/DBServerAgencySync.cpp + Cluster/DropCollection.cpp + Cluster/DropDatabase.cpp + Cluster/DropIndex.cpp + Cluster/EnsureIndex.cpp + Cluster/FollowerInfo.cpp Cluster/HeartbeatThread.cpp + Cluster/Maintenance.cpp + Cluster/MaintenanceFeature.cpp + Cluster/MaintenanceRestHandler.cpp + Cluster/MaintenanceWorker.cpp + Cluster/NonAction.cpp Cluster/ReplicationTimeoutFeature.cpp + Cluster/ResignShardLeadership.cpp Cluster/RestAgencyCallbacksHandler.cpp Cluster/RestClusterHandler.cpp Cluster/ServerState.cpp + Cluster/SynchronizeShard.cpp Cluster/TraverserEngine.cpp Cluster/TraverserEngineRegistry.cpp + Cluster/UpdateCollection.cpp Cluster/v8-cluster.cpp GeneralServer/AsyncJobManager.cpp GeneralServer/AuthenticationFeature.cpp diff --git a/arangod/Cluster/Action.cpp b/arangod/Cluster/Action.cpp new file mode 100644 index 0000000000..ced51eb25a --- /dev/null +++ b/arangod/Cluster/Action.cpp @@ -0,0 +1,161 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "Action.h" + +#include "Cluster/CreateCollection.h" +#include "Cluster/CreateDatabase.h" +#include "Cluster/DropCollection.h" +#include "Cluster/DropDatabase.h" +#include "Cluster/DropIndex.h" +#include "Cluster/EnsureIndex.h" +#include "Cluster/NonAction.h" +#include "Cluster/ResignShardLeadership.h" +#include "Cluster/SynchronizeShard.h" +#include "Cluster/UpdateCollection.h" + +#include "Logger/Logger.h" + +using namespace arangodb; +using namespace arangodb::maintenance; + +Action::Action( + MaintenanceFeature& feature, + ActionDescription const& description) : _action(nullptr) { + TRI_ASSERT(description.has("name")); + create(feature, description); +} + +Action::Action( + MaintenanceFeature& feature, + ActionDescription&& description) : _action(nullptr) { + TRI_ASSERT(description.has("name")); + create(feature, std::move(description)); +} + +Action::Action( + MaintenanceFeature& feature, + std::shared_ptr const description) + : _action(nullptr) { + TRI_ASSERT(description->has("name")); + create(feature, *description); +} + +Action::Action(std::unique_ptr action) + : _action(std::move(action)) {} + +Action::~Action() {} + +void Action::create( + MaintenanceFeature& feature, ActionDescription const& description) { + std::string name = description.name(); + if (name == "CreateCollection") { + _action.reset(new CreateCollection(feature, description)); + } else if (name == "CreateDatabase") { + _action.reset(new CreateDatabase(feature, description)); + } else if (name == "DropCollection") { + _action.reset(new DropCollection(feature, description)); + } else if (name == "DropDatabase") { + _action.reset(new DropDatabase(feature, description)); + } else if (name == "DropIndex") { + _action.reset(new DropIndex(feature, description)); + } else if (name == "EnsureIndex") { + _action.reset(new EnsureIndex(feature, description)); + } else if (name == "ResignShardLeadership") { + _action.reset(new ResignShardLeadership(feature, description)); + } else if (name == "SynchronizeShard") { + _action.reset(new SynchronizeShard(feature, description)); + } else if (name == "UpdateCollection") { + _action.reset(new UpdateCollection(feature, description)); + } else { + _action.reset(new NonAction(feature, description)); + } +} + +ActionDescription const& Action::describe() const { + TRI_ASSERT(_action != nullptr); + return _action->describe(); +} + +arangodb::MaintenanceFeature& Action::feature() const { + TRI_ASSERT(_action != nullptr); + return _action->feature(); +} + +std::shared_ptr const Action::properties() const { + return describe().properties(); +} + +bool Action::first() { + TRI_ASSERT(_action != nullptr); + return _action->first(); +} + +bool Action::next() { + TRI_ASSERT(_action != nullptr); + return _action->next(); +} + +arangodb::Result Action::result() { + TRI_ASSERT(_action != nullptr); + return _action->result(); +} + +arangodb::Result Action::kill(Signal const& signal) { + TRI_ASSERT(_action != nullptr); + return _action->kill(signal); +} + +arangodb::Result Action::progress(double& p) { + TRI_ASSERT(_action != nullptr); + return _action->progress(p); +} + +ActionState Action::getState() const { + return _action->getState(); +} + +void Action::startStats() { + _action->startStats(); +} + +void Action::incStats() { + _action->incStats(); +} + +void Action::endStats() { + _action->endStats(); +} + +void Action::toVelocyPack(arangodb::velocypack::Builder& builder) const { + TRI_ASSERT(_action != nullptr); + _action->toVelocyPack(builder); +} + +namespace std { +ostream& operator<< ( + ostream& out, arangodb::maintenance::Action const& d) { + out << d.toVelocyPack().toJson(); + return out; +}} diff --git a/arangod/Cluster/Action.h b/arangod/Cluster/Action.h new file mode 100644 index 0000000000..f3439c4a8b --- /dev/null +++ b/arangod/Cluster/Action.h @@ -0,0 +1,193 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_CLUSTER_MAINTENANCE_ACTION_H +#define ARANGODB_CLUSTER_MAINTENANCE_ACTION_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include "lib/Basics/Result.h" + +#include + +namespace arangodb { + +class MaintenanceFeature; + +namespace maintenance { + +class Action { + +public: + + /// @brief construct with description + Action(MaintenanceFeature&, ActionDescription const&); + + /// @brief construct with description + Action(MaintenanceFeature&, ActionDescription&&); + + /// @brief construct with description + Action(MaintenanceFeature&, std::shared_ptr const); + + /** + * @brief construct with concrete action base + * @param feature Maintenance feature + * @param action Concrete action + */ + Action(std::unique_ptr action); + + /// @brief clean up + virtual ~Action(); + + /// @brief run for some time and tell, if need more time or done + bool next(); + + /// @brief run for some time and tell, if need more time or done + arangodb::Result result(); + + /// @brief run for some time and tell, if need more time or done + bool first (); + + /// @brief run for some time and tell, if need more time or done + ActionState state() const; + + /// @brief is object in a usable condition + bool ok() const { return (nullptr != _action.get() && _action->ok()); }; + + /// @brief kill action with signal + arangodb::Result kill(Signal const& signal); + + /// @brief check progress + arangodb::Result progress(double& progress); + + /// @brief describe action + ActionDescription const& describe() const; + + /// @brief describe action + MaintenanceFeature& feature() const; + + // @brief get properties + std::shared_ptr const properties() const; + + ActionState getState() const; + + void setState(ActionState state) { + _action->setState(state); + } + + /// @brief update incremental statistics + void startStats(); + + /// @brief update incremental statistics + void incStats(); + + /// @brief finalize statistics + void endStats(); + + /// @brief return progress statistic + uint64_t getProgress() const { return _action->getProgress(); } + + /// @brief Once PreAction completes, remove its pointer + void clearPreAction() { _action->clearPreAction(); } + + /// @brief Retrieve pointer to action that should run before this one + std::shared_ptr getPreAction() { return _action->getPreAction(); } + + /// @brief Initiate a pre action + void createPreAction(ActionDescription const& description); + + /// @brief Initiate a post action + void createPostAction(ActionDescription const& description); + + /// @brief Retrieve pointer to action that should run directly after this one + std::shared_ptr getPostAction() { return _action->getPostAction(); } + + /// @brief Save pointer to successor action + void setPostAction(std::shared_ptr post) { + _action->setPostAction(post); + } + + /// @brief hash value of ActionDescription + /// @return uint64_t hash + uint64_t hash() const { return _action->hash(); } + + /// @brief hash value of ActionDescription + /// @return uint64_t hash + uint64_t id() const { return _action->id(); } + + /// @brief add VPackObject to supplied builder with info about this action + void toVelocyPack(VPackBuilder & builder) const; + + /// @brief add VPackObject to supplied builder with info about this action + VPackBuilder toVelocyPack() const { return _action->toVelocyPack(); } + + /// @brief Returns json array of object contents for status reports + /// Thread safety of this function is questionable for some member objects +// virtual Result toJson(/* builder */) {return Result;}; + + /// @brief Return Result object contain action specific status + Result result() const { return _action->result(); } + + /// @brief execution finished successfully or failed ... and race timer expired + bool done() const { return _action->done(); } + + /// @brief waiting for a worker to grab it and go! + bool runable() const {return _action->runable();} + + /// @brief When object was constructed + std::chrono::system_clock::time_point getCreateTime() const + {return _action->getCreateTime();} + + /// @brief When object was first started + std::chrono::system_clock::time_point getStartTime() const + {return _action->getStartTime();} + + + /// @brief When object most recently iterated + std::chrono::system_clock::time_point getLastStatTime() const + {return _action->getLastStatTime();} + + + /// @brief When object finished executing + std::chrono::system_clock::time_point getDoneTime() const + {return _action->getDoneTime();} + +private: + + /// @brief actually create the concrete action + void create(MaintenanceFeature&, ActionDescription const&); + + /// @brief concrete action + std::unique_ptr _action; + +}; + +}} + +namespace std { +ostream& operator<< ( + ostream& o, arangodb::maintenance::Action const& d); +} +#endif diff --git a/arangod/Cluster/ActionBase.cpp b/arangod/Cluster/ActionBase.cpp new file mode 100644 index 0000000000..b1c584ecd3 --- /dev/null +++ b/arangod/Cluster/ActionBase.cpp @@ -0,0 +1,259 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "Cluster/ActionBase.h" + +#include "Agency/TimeString.h" +#include "ApplicationFeatures/ApplicationServer.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/MaintenanceFeature.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; + +// FIXMAINTENANCE: These strings appear again in ActionDescription.h, +// we should remove this duplication. +const char ActionBase::KEY[]="key"; +const char ActionBase::FIELDS[]="fields"; +const char ActionBase::TYPE[]="type"; +const char ActionBase::INDEXES[]="indexes"; +const char ActionBase::INDEX[]="index"; +const char ActionBase::SHARDS[]="shards"; +const char ActionBase::DATABASE[]="database"; +const char ActionBase::COLLECTION[]="collection"; +const char ActionBase::EDGE[]="edge"; +const char ActionBase::NAME[]="name"; +const char ActionBase::ID[]="id"; +const char ActionBase::LEADER[]="theLeader"; +const char ActionBase::LOCAL_LEADER[]="localLeader"; +const char ActionBase::GLOB_UID[]="globallyUniqueId"; +const char ActionBase::OBJECT_ID[]="objectId"; + +inline static std::chrono::system_clock::duration secs_since_epoch() { + return std::chrono::system_clock::now().time_since_epoch(); +} + +ActionBase::ActionBase(MaintenanceFeature& feature, ActionDescription const& desc) + : _feature(feature), _description(desc), _state(READY), + _progress(0) { + + init(); + +} + +ActionBase::ActionBase(MaintenanceFeature& feature, ActionDescription&& desc) + : _feature(feature), _description(std::move(desc)), _state(READY), + _progress(0) { + + init(); +} + +void ActionBase::init() { + _hash = _description.hash(); + _clientId = std::to_string(_hash); + _id = _feature.nextActionId(); + + // initialization of duration struct is not guaranteed in atomic form + _actionCreated = secs_since_epoch(); + _actionStarted = std::chrono::system_clock::duration::zero(); + _actionLastStat = std::chrono::system_clock::duration::zero(); + _actionDone = std::chrono::system_clock::duration::zero(); +} + + +ActionBase::~ActionBase() { +} + + +void ActionBase::notify() { + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Job " << _description << " calling syncDBServerStatusQuo"; + auto cf = ApplicationServer::getFeature("Cluster"); + if (cf != nullptr) { + cf->syncDBServerStatusQuo(); + } +} + + +/// @brief execution finished successfully or failed ... and race timer expired +bool ActionBase::done() const { + + return (COMPLETE==_state || FAILED==_state) && + _actionDone.load() + std::chrono::seconds(_feature.getSecondsActionsBlock()) <= secs_since_epoch(); + +} // ActionBase::done + +ActionDescription const& ActionBase::describe() const { + return _description; +} + +MaintenanceFeature& ActionBase::feature() const { + return _feature; +} + +VPackSlice const ActionBase::properties() const { + return _description.properties()->slice(); +} + + +/// @brief Initiate a new action that will start immediately, pausing this action +void ActionBase::createPreAction(std::shared_ptr const & description) { + + _preAction = description; + std::shared_ptr new_action = _feature.preAction(description); + + // shift from EXECUTING to WAITINGPRE ... EXECUTING is set to block other + // workers from picking it up + if (_preAction && new_action->ok()) { + setState(WAITINGPRE); + } else { + _result.reset(TRI_ERROR_BAD_PARAMETER, "preAction rejected parameters."); + } // else + +} // ActionBase::createPreAction + + +/// @brief Retrieve pointer to action that should run before this one +std::shared_ptr ActionBase::getPreAction() { + return (_preAction != nullptr) ? _feature.findAction(_preAction) : nullptr; +} + + +/// @brief Retrieve pointer to action that should run after this one +std::shared_ptr ActionBase::getPostAction() { + return (_postAction != nullptr) ? _feature.findAction(_postAction) : nullptr; +} + + +// FIXMEMAINTENANCE: Code path could corrupt registry object because +// this does not hold lock. Also, current implementation is a race condition +// where another thread could pick this up. + +/// @brief Create a new action that will start after this action successfully completes +void ActionBase::createPostAction(std::shared_ptr const& description) { + + // preAction() sets up what we need + _postAction = description; + std::shared_ptr new_action = _feature.postAction(description); + + // shift from EXECUTING to WAITINGPOST ... EXECUTING is set to block other + // workers from picking it up + if (_postAction && new_action->ok()) { + new_action->setState(WAITINGPOST); + } else { + _result.reset(TRI_ERROR_BAD_PARAMETER, "preAction rejected parameters for _postAction."); + } // else + +} // ActionBase::createPostAction + + +void ActionBase::startStats() { + + _actionStarted = secs_since_epoch(); + +} // ActionBase::startStats + + +/// @brief show progress on Action, and when that progress occurred +void ActionBase::incStats() { + + ++_progress; + _actionLastStat = secs_since_epoch(); + +} // ActionBase::incStats + + +void ActionBase::endStats() { + + _actionDone = secs_since_epoch(); + +} // ActionBase::endStats + + +Result arangodb::actionError(int errorCode, std::string const& errorMessage) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << errorMessage; + return Result(errorCode, errorMessage); +} + +Result arangodb::actionWarn(int errorCode, std::string const& errorMessage) { + LOG_TOPIC(WARN, Logger::MAINTENANCE) << errorMessage; + return Result(errorCode, errorMessage); +} + +void ActionBase::toVelocyPack(VPackBuilder & builder) const { + VPackObjectBuilder ob(&builder); + + builder.add("id", VPackValue(_id)); + builder.add("state", VPackValue(_state)); + builder.add("progress", VPackValue(_progress)); + + builder.add("created", VPackValue(timepointToString(_actionCreated.load()))); + builder.add("started", VPackValue(timepointToString(_actionStarted.load()))); + builder.add("lastStat", VPackValue(timepointToString(_actionLastStat.load()))); + builder.add("done", VPackValue(timepointToString(_actionDone.load()))); + + builder.add("result", VPackValue(_result.errorNumber())); + + builder.add(VPackValue("description")); + { VPackObjectBuilder desc(&builder); + _description.toVelocyPack(builder); } + +} // MaintanceAction::toVelocityPack + +VPackBuilder ActionBase::toVelocyPack() const { + VPackBuilder builder; + toVelocyPack(builder); + return builder; +} + + + +/** + * kill() operation is an expected future feature. Not supported in the + * original ActionBase derivatives + */ +arangodb::Result ActionBase::kill(Signal const& signal) { + return actionError( + TRI_ERROR_ACTION_OPERATION_UNABORTABLE, "Kill operation not supported on this action."); +} + + +/** + * progress() operation is an expected future feature. Not supported in the + * original ActionBase derivatives + */ +arangodb::Result ActionBase::progress(double& progress) { + progress = 0.5; + return arangodb::Result(TRI_ERROR_NO_ERROR); +} + + +namespace std { +ostream& operator<< ( + ostream& out, arangodb::maintenance::ActionBase const& d) { + out << d.toVelocyPack().toJson(); + return out; +}} diff --git a/arangod/Cluster/ActionBase.h b/arangod/Cluster/ActionBase.h new file mode 100644 index 0000000000..59ead5cdde --- /dev/null +++ b/arangod/Cluster/ActionBase.h @@ -0,0 +1,251 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_ACTION_BASE_H +#define ARANGODB_MAINTENANCE_ACTION_BASE_H + +#include "ActionDescription.h" + +#include "Basics/Common.h" +#include "Basics/Result.h" + +#include + +namespace arangodb { + +class MaintenanceFeature; + +namespace maintenance { + + +class Action; + +class ActionBase { + + public: + ActionBase (MaintenanceFeature&, ActionDescription const&); + + ActionBase (MaintenanceFeature&, ActionDescription&&); + + ActionBase() = delete; + + virtual ~ActionBase(); + + // + // MaintenanceWork entry points + // + + + /// @brief initial call to object to perform a unit of work. + /// really short tasks could do all work here and return false + /// @return true to continue processing, false done (result() set) + virtual bool first() = 0; + + /// @brief iterative call to perform a unit of work + /// @return true to continue processing, false done (result() set) + virtual bool next() { return false; } + + // + // common property or decription names + // + + static const char KEY[]; + static const char FIELDS[]; + static const char TYPE[]; + static const char INDEXES[]; + static const char INDEX[]; + static const char SHARDS[]; + static const char DATABASE[]; + static const char COLLECTION[]; + static const char EDGE[]; + static const char NAME[]; + static const char ID[]; + static const char LEADER[]; + static const char LOCAL_LEADER[]; + static const char GLOB_UID[]; + static const char OBJECT_ID[]; + + /// @brief execution finished successfully or failed ... and race timer expired + virtual bool done() const; + + /// @brief waiting for a worker to grab it and go! + bool runable() const {return READY==_state;} + + /// @brief did initialization have issues? + bool ok() const {return FAILED!=_state;} + + /// @brief adjust state of object, assumes WRITE lock on _actionRegistryLock + ActionState state() const { + return _state; + } + + void notify(); + + virtual arangodb::Result kill(Signal const& signal); + + virtual arangodb::Result progress(double& progress); + + ActionDescription const& describe() const; + + MaintenanceFeature& feature() const; + + std::string const& get(std::string const&) const; + + VPackSlice const properties() const; + + /// @brief adjust state of object, assumes WRITE lock on _actionRegistryLock + ActionState getState() const { + return _state; + } + + /// @brief adjust state of object, assumes WRITE lock on _actionRegistryLock + void setState(ActionState state) { + _state = state; + } + + /// @brief update incremental statistics + void startStats(); + + /// @brief update incremental statistics + void incStats(); + + /// @brief finalize statistics + void endStats(); + + /// @brief return progress statistic + uint64_t getProgress() const {return _progress.load();} + + /// @brief Once PreAction completes, remove its pointer + void clearPreAction() {_preAction.reset();} + + /// @brief Retrieve pointer to action that should run before this one + std::shared_ptr getPreAction(); + + /// @brief Initiate a pre action + void createPreAction(std::shared_ptr const& description); + + /// @brief Initiate a post action + void createPostAction(std::shared_ptr const& description); + + /// @brief Retrieve pointer to action that should run directly after this one + std::shared_ptr getPostAction(); + + /// @brief Save pointer to successor action + void setPostAction(std::shared_ptr post) { + _postAction=post; + } + + /// @brief hash value of ActionDescription + /// @return uint64_t hash + std::string clientId() const { return _clientId; } + + /// @brief hash value of ActionDescription + /// @return uint64_t hash + uint64_t hash() const {return _hash;} + + /// @brief hash value of ActionDescription + /// @return uint64_t hash + uint64_t id() const {return _id;} + + /// @brief add VPackObject to supplied builder with info about this action + virtual void toVelocyPack(VPackBuilder & builder) const; + + /// @brief add VPackObject to supplied builder with info about this action + VPackBuilder toVelocyPack() const; + + /// @brief Returns json array of object contents for status reports + /// Thread safety of this function is questionable for some member objects + // virtual Result toJson(/* builder */) {return Result;} + + /// @brief Return Result object contain action specific status + Result result() const {return _result;} + + /// @brief When object was constructed + std::chrono::system_clock::time_point getCreateTime() const + {return std::chrono::system_clock::time_point() + _actionCreated.load(); } + + /// @brief When object was first started + std::chrono::system_clock::time_point getStartTime() const + {return std::chrono::system_clock::time_point() + _actionStarted.load(); } + + /// @brief When object most recently iterated + std::chrono::system_clock::time_point getLastStatTime() const + {return std::chrono::system_clock::time_point() + _actionLastStat.load(); } + + /// @brief When object finished executing + std::chrono::system_clock::time_point getDoneTime() const + {return std::chrono::system_clock::time_point() + _actionDone.load(); } + + +protected: + + /// @brief common initialization for all constructors + void init(); + + + arangodb::MaintenanceFeature& _feature; + + ActionDescription _description; + + ActionModel _model; + + uint64_t _hash; + std::string _clientId; + + uint64_t _id; + + std::atomic _state; + + // NOTE: preAction should only be set within first() or post(), not construction + std::shared_ptr _preAction; + std::shared_ptr _postAction; + + // times for user reporting (and _actionDone used by done() to prevent + // race conditions of same task executing twice + std::atomic _actionCreated; + std::atomic _actionStarted; + std::atomic _actionLastStat; + std::atomic _actionDone; + + std::atomic _progress; + + Result _result; + + + +}; // class ActionBase + +} // namespace maintenance + +Result actionError(int errorCode, std::string const& errorMessage); +Result actionWarn(int errorCode, std::string const& errorMessage); + +} // namespace arangodb + + +namespace std { +ostream& operator<< ( + ostream& o, arangodb::maintenance::ActionBase const& d); +} +#endif diff --git a/arangod/Cluster/ActionDescription.cpp b/arangod/Cluster/ActionDescription.cpp new file mode 100644 index 0000000000..4f337bde90 --- /dev/null +++ b/arangod/Cluster/ActionDescription.cpp @@ -0,0 +1,147 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "ActionDescription.h" + +#include + +using namespace arangodb; +using namespace arangodb::maintenance; + +/// @brief ctor +ActionDescription::ActionDescription( + std::map const& d, + std::shared_ptr const p) : + _description(d), _properties(p) { + TRI_ASSERT(d.find(NAME) != d.end()); + TRI_ASSERT(p == nullptr || p->isEmpty() || p->slice().isObject()); +} + +/// @brief Default dtor +ActionDescription::~ActionDescription() {} + +/// @brief Does this description have a "p" parameter? +bool ActionDescription::has(std::string const& p) const { + return _description.find(p) != _description.end(); +} + +/// @brief Does this description have a "p" parameter? +std::string ActionDescription::operator()(std::string const& p) const { + return _description.at(p); +} + +/// @brief Does this description have a "p" parameter? +std::string ActionDescription::get(std::string const& p) const { + return _description.at(p); +} + +/// @brief Get parameter +Result ActionDescription::get(std::string const& p, std::string& r) const { + Result result; + auto const& it = _description.find(p); + if (it == _description.end()) { + result.reset(TRI_ERROR_FAILED); + } else { + r = it->second; + } + return result; +} + +/// @brief Hash function +std::size_t ActionDescription::hash() const { + std::string propstr; + for (auto const& i : _description) { + propstr += i.first + i.second; + } + return std::hash{}(propstr); +} + +std::size_t ActionDescription::hash(std::map desc) { + std::string propstr; + for (auto const& i : desc) { + propstr += i.first + i.second; + } + return std::hash{}(propstr); +} + +/// @brief Equality operator +bool ActionDescription::operator==( + ActionDescription const& other) const { + return _description== other._description; +} + +/// @brief Get action name. Cannot throw. See constructor +std::string const& ActionDescription::name() const { + static const std::string EMPTY_STRING; + auto const& it = _description.find(NAME); + return (it != _description.end()) ? it->second : EMPTY_STRING; +} + +/// @brief summary to velocypack +VPackBuilder ActionDescription::toVelocyPack() const { + VPackBuilder b; + { VPackObjectBuilder bb(&b); + toVelocyPack(b); } + return b; +} + + +/// @brief summary to velocypack +void ActionDescription::toVelocyPack(VPackBuilder& b) const { + TRI_ASSERT(b.isOpenObject()); + for (auto const& i : _description) { + b.add(i.first, VPackValue(i.second)); + } + if (_properties != nullptr && !_properties->isEmpty()) { + b.add("properties", _properties->slice()); + } +} + + +/// @brief summary to JSON string +std::string ActionDescription::toJson() const { + return toVelocyPack().toJson(); +} + + +/// @brief non discrimantory properties. +std::shared_ptr const ActionDescription::properties() const { + return _properties; +} + + +/// @brief hash implementation for ActionRegistry +namespace std { +std::size_t hash::operator()( + ActionDescription const& a) const noexcept { + return a.hash(); +} + +ostream& operator<< ( + ostream& out, arangodb::maintenance::ActionDescription const& d) { + out << d.toJson(); + return out; +} +} + diff --git a/arangod/Cluster/ActionDescription.h b/arangod/Cluster/ActionDescription.h new file mode 100644 index 0000000000..627456ace9 --- /dev/null +++ b/arangod/Cluster/ActionDescription.h @@ -0,0 +1,205 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_CLUSTER_MAINTENANCE_ACTION_DESCRIPTION_H +#define ARANGODB_CLUSTER_MAINTENANCE_ACTION_DESCRIPTION_H + +#include "Basics/VelocyPackHelper.h" + +#include +#include + +namespace arangodb { +namespace maintenance { + +enum Signal { GRACEFUL, IMMEDIATE }; + +enum ActionModel { BACKGROUND, FOREGROUND }; + + // + // state accessor and set functions + // (some require time checks and/or combination tests) + // + enum ActionState { + READY = 1, // waiting for a worker on the deque + EXECUTING = 2, // user or worker thread currently executing + WAITING = 3, // initiated a pre-task, waiting for its completion + WAITINGPRE = 4,// parent task created, about to execute on parent's thread + WAITINGPOST = 5,// parent task created, will execute after parent's success + PAUSED = 6, // (not implemented) user paused task + COMPLETE = 7, // task completed successfully + FAILED = 8, // task failed, no longer executing + }; + +static std::string const KEY("key"); +static std::string const FIELDS("fields"); +static std::string const TYPE("type"); +static std::string const INDEXES("indexes"); +static std::string const SHARDS("shards"); +static std::string const DATABASE("database"); +static std::string const EDGE("edge"); +static std::string const COLLECTION("collection"); +static std::string const SHARD("shard"); +static std::string const NAME("name"); +static std::string const ID("id"); +static std::string const LEADER("theLeader"); +static std::string const LOCAL_LEADER("localLeader"); +static std::string const GLOB_UID("globallyUniqueId"); +static std::string const OBJECT_ID("objectId"); +static std::string const SERVER_ID("serverId"); + +/** + * @brief Action description for mainenance actions + * + * This structure holds once initialized constant parameters of a maintenance + * action. Members are declared const, thus thread safety guards are ommited. + */ +struct ActionDescription { + +public: + + /** + * @brief Construct with properties + * @param desc Descriminatory properties, which are considered for hash + * @param supp Non discriminatory properties + */ + ActionDescription( + std::map const& desc, + std::shared_ptr const suppl = std::make_shared()); + + /** + * @brief Clean up + */ + virtual ~ActionDescription(); + + /** + * @brief Check equality (only _description considered) + * @param other Other descriptor + */ + bool operator== (ActionDescription const& other) const; + + /** + * @brief Calculate hash of _description as concatenation + * @param other Other descriptor + */ + std::size_t hash() const; + static std::size_t hash(std::map desc); + + /// @brief Name of action + std::string const& name() const; + + /** + * @brief Check if key exists in discrimantory container + * @param key Key to lookup + * @return true if key is found + */ + bool has(std::string const& keu) const; + + /** + * @brief Get a string value from description + * @param key Key to get + * @exception std::out_of_range if the we do not have this key in discrimatory container + * @return Value to specified key + */ + std::string get(std::string const& key) const; + + /** + * @brief Get a string value from description + * @param key Key to get + * @exception std::out_of_range if the we do not have this key in discrimatory container + * @return Value to specified key + */ + std::string operator()(std::string const& key) const; + + /** + * @brief Get a string value from description + * + * @param key Key to get + * @param value If key is found the value is assigned to this variable + * @return Success (key found?) + */ + Result get(std::string const& key, std::string& value) const; + + /** + * @brief Dump to JSON(vpack) + * @return JSON Velocypack of all paramters + */ + VPackBuilder toVelocyPack() const; + + /** + * @brief Dump to JSON(vpack) + * @return JSON Velocypack of all paramters + */ + void toVelocyPack(VPackBuilder&) const; + + /** + * @brief Dump to JSON(string) + * @return JSON string of all paramters + */ + std::string toJson() const; + + /** + * @brief Dump to JSON(output stream) + * @param os Output stream reference + * @return Output stream reference + */ + std::ostream& print(std::ostream& os) const; + + /** + * @brief Get non discrimantory properties + * Get non discrimantory properties. + * This function does not throw as builder is always when here. + * @return Non discriminatory properties + */ + std::shared_ptr const properties() const; + +private: + + /// Note: members are const. No thread safety guards needed. + + /** @brief discriminatory properties */ + std::map const _description; + + /** @brief non-discriminatory properties */ + std::shared_ptr const _properties; + +}; + +}} + + + +namespace std { +ostream& operator<< ( + ostream& o, arangodb::maintenance::ActionDescription const& d); +/// @brief Hash function used by std::unordered_map +template<> struct hash { + typedef arangodb::maintenance::ActionDescription argument_t; + typedef std::size_t result_t; + result_t operator()(argument_t const& a) const noexcept; +}; + +} + +#endif diff --git a/arangod/Cluster/AgencyCallback.cpp b/arangod/Cluster/AgencyCallback.cpp index 06831de463..4f89b1ed77 100644 --- a/arangod/Cluster/AgencyCallback.cpp +++ b/arangod/Cluster/AgencyCallback.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,6 +36,7 @@ #include "Logger/Logger.h" using namespace arangodb; +using namespace arangodb::application_features; AgencyCallback::AgencyCallback(AgencyComm& agency, std::string const& key, std::function const& cb, diff --git a/arangod/Cluster/AgencyCallback.h b/arangod/Cluster/AgencyCallback.h index 1390e02d15..349704233f 100644 --- a/arangod/Cluster/AgencyCallback.h +++ b/arangod/Cluster/AgencyCallback.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/AgencyCallbackRegistry.cpp b/arangod/Cluster/AgencyCallbackRegistry.cpp index 180779882b..6135d852bb 100644 --- a/arangod/Cluster/AgencyCallbackRegistry.cpp +++ b/arangod/Cluster/AgencyCallbackRegistry.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/AgencyCallbackRegistry.h b/arangod/Cluster/AgencyCallbackRegistry.h index 93b9c902d9..8384c1ebe4 100644 --- a/arangod/Cluster/AgencyCallbackRegistry.h +++ b/arangod/Cluster/AgencyCallbackRegistry.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterComm.cpp b/arangod/Cluster/ClusterComm.cpp index d2391ed3dc..1317afd80f 100644 --- a/arangod/Cluster/ClusterComm.cpp +++ b/arangod/Cluster/ClusterComm.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterComm.h b/arangod/Cluster/ClusterComm.h index e063185248..7c667163e6 100644 --- a/arangod/Cluster/ClusterComm.h +++ b/arangod/Cluster/ClusterComm.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterEdgeCursor.cpp b/arangod/Cluster/ClusterEdgeCursor.cpp index bb6d1ed277..be4c202576 100644 --- a/arangod/Cluster/ClusterEdgeCursor.cpp +++ b/arangod/Cluster/ClusterEdgeCursor.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterEdgeCursor.h b/arangod/Cluster/ClusterEdgeCursor.h index 7cc4e65df6..df15c00f7a 100644 --- a/arangod/Cluster/ClusterEdgeCursor.h +++ b/arangod/Cluster/ClusterEdgeCursor.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterFeature.cpp b/arangod/Cluster/ClusterFeature.cpp index cf0de36f47..7913086019 100644 --- a/arangod/Cluster/ClusterFeature.cpp +++ b/arangod/Cluster/ClusterFeature.cpp @@ -41,21 +41,20 @@ #include "SimpleHttpClient/ConnectionManager.h" #include "StorageEngine/EngineSelectorFeature.h" +using namespace arangodb; using namespace arangodb::application_features; using namespace arangodb::basics; using namespace arangodb::options; -namespace arangodb { - ClusterFeature::ClusterFeature(application_features::ApplicationServer& server) - : ApplicationFeature(server, "Cluster"), - _unregisterOnShutdown(false), - _enableCluster(false), - _requirePersistedId(false), - _heartbeatThread(nullptr), - _heartbeatInterval(0), - _agencyCallbackRegistry(nullptr), - _requestedRole(ServerState::RoleEnum::ROLE_UNDEFINED) { + : ApplicationFeature(server, "Cluster"), + _unregisterOnShutdown(false), + _enableCluster(false), + _requirePersistedId(false), + _heartbeatThread(nullptr), + _heartbeatInterval(0), + _agencyCallbackRegistry(nullptr), + _requestedRole(ServerState::RoleEnum::ROLE_UNDEFINED) { setOptional(true); startsAfter("DatabasePhase"); } @@ -110,7 +109,7 @@ void ClusterFeature::collectOptions(std::shared_ptr options) { new VectorParameter(&_agencyEndpoints)); options->addHiddenOption("--cluster.agency-prefix", "agency prefix", - new StringParameter(&_agencyPrefix)); + new StringParameter(&_agencyPrefix)); options->addObsoleteOption("--cluster.my-local-info", "this server's local info", false); options->addObsoleteOption("--cluster.my-id", "this server's id", false); @@ -126,12 +125,12 @@ void ClusterFeature::collectOptions(std::shared_ptr options) { new UInt32Parameter(&_systemReplicationFactor)); options->addHiddenOption("--cluster.create-waits-for-sync-replication", - "active coordinator will wait for all replicas to create collection", - new BooleanParameter(&_createWaitsForSyncReplication)); + "active coordinator will wait for all replicas to create collection", + new BooleanParameter(&_createWaitsForSyncReplication)); options->addHiddenOption("--cluster.index-create-timeout", - "amount of time (in seconds) the coordinator will wait for an index to be created before giving up", - new DoubleParameter(&_indexCreationTimeout)); + "amount of time (in seconds) the coordinator will wait for an index to be created before giving up", + new DoubleParameter(&_indexCreationTimeout)); } void ClusterFeature::validateOptions(std::shared_ptr options) { @@ -156,7 +155,7 @@ void ClusterFeature::validateOptions(std::shared_ptr options) { // validate --cluster.agency-endpoint (currently a noop) if (_agencyEndpoints.empty()) { LOG_TOPIC(FATAL, Logger::CLUSTER) - << "must at least specify one endpoint in --cluster.agency-endpoint"; + << "must at least specify one endpoint in --cluster.agency-endpoint"; FATAL_ERROR_EXIT(); } @@ -167,7 +166,7 @@ void ClusterFeature::validateOptions(std::shared_ptr options) { // validate --cluster.agency-prefix size_t found = _agencyPrefix.find_first_not_of( - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/"); + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/"); if (found != std::string::npos || _agencyPrefix.empty()) { LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "invalid value specified for --cluster.agency-prefix"; @@ -226,7 +225,7 @@ void ClusterFeature::prepare() { // create callback registery _agencyCallbackRegistry.reset( - new AgencyCallbackRegistry(agencyCallbacksPath())); + new AgencyCallbackRegistry(agencyCallbacksPath())); // Initialize ClusterInfo library: ClusterInfo::createInstance(_agencyCallbackRegistry.get()); @@ -249,7 +248,7 @@ void ClusterFeature::prepare() { // nullptr happens only during shutdown if (af->isActive() && !af->hasUserdefinedJwt()) { LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "Cluster authentication enabled but JWT not set via command line. Please" - << " provide --server.jwt-secret which is used throughout the cluster."; + << " provide --server.jwt-secret which is used throughout the cluster."; FATAL_ERROR_EXIT(); } } @@ -276,7 +275,7 @@ void ClusterFeature::prepare() { if (unified.empty()) { LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "invalid endpoint '" << _agencyEndpoints[i] - << "' specified for --cluster.agency-endpoint"; + << "' specified for --cluster.agency-endpoint"; FATAL_ERROR_EXIT(); } @@ -293,7 +292,7 @@ void ClusterFeature::prepare() { // perform an initial connect to the agency if (!AgencyCommManager::MANAGER->start()) { LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "Could not connect to any agency endpoints (" - << AgencyCommManager::MANAGER->endpointsString() << ")"; + << AgencyCommManager::MANAGER->endpointsString() << ")"; FATAL_ERROR_EXIT(); } @@ -309,8 +308,8 @@ void ClusterFeature::prepare() { if (role == ServerState::ROLE_UNDEFINED) { // no role found LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "unable to determine unambiguous role for server '" - << ServerState::instance()->getId() - << "'. No role configured in agency (" << endpoints << ")"; + << ServerState::instance()->getId() + << "'. No role configured in agency (" << endpoints << ")"; FATAL_ERROR_EXIT(); } @@ -346,16 +345,16 @@ void ClusterFeature::prepare() { if (_myAddress.empty()) { LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "unable to determine internal address for server '" - << ServerState::instance()->getId() - << "'. Please specify --cluster.my-address or configure the " - "address for this server in the agency."; + << ServerState::instance()->getId() + << "'. Please specify --cluster.my-address or configure the " + "address for this server in the agency."; FATAL_ERROR_EXIT(); } // now we can validate --cluster.my-address if (Endpoint::unifiedForm(_myAddress).empty()) { LOG_TOPIC(FATAL, arangodb::Logger::CLUSTER) << "invalid endpoint '" << _myAddress - << "' specified for --cluster.my-address"; + << "' specified for --cluster.my-address"; FATAL_ERROR_EXIT(); } @@ -384,22 +383,22 @@ void ClusterFeature::start() { std::string myId = ServerState::instance()->getId(); LOG_TOPIC(INFO, arangodb::Logger::CLUSTER) << "Cluster feature is turned on. Agency version: " << version - << ", Agency endpoints: " << endpoints << ", server id: '" << myId - << "', internal address: " << _myAddress - << ", role: " << role; + << ", Agency endpoints: " << endpoints << ", server id: '" << myId + << "', internal address: " << _myAddress + << ", role: " << role; AgencyCommResult result = comm.getValues("Sync/HeartbeatIntervalMs"); if (result.successful()) { velocypack::Slice HeartbeatIntervalMs = - result.slice()[0].get(std::vector( - {AgencyCommManager::path(), "Sync", "HeartbeatIntervalMs"})); + result.slice()[0].get(std::vector( + {AgencyCommManager::path(), "Sync", "HeartbeatIntervalMs"})); if (HeartbeatIntervalMs.isInteger()) { try { _heartbeatInterval = HeartbeatIntervalMs.getUInt(); LOG_TOPIC(INFO, arangodb::Logger::CLUSTER) << "using heartbeat interval value '" << _heartbeatInterval - << " ms' from agency"; + << " ms' from agency"; } catch (...) { // Ignore if it is not a small int or uint } @@ -410,7 +409,7 @@ void ClusterFeature::start() { if (_heartbeatInterval == 0) { _heartbeatInterval = 5000; // 1/s LOG_TOPIC(WARN, arangodb::Logger::CLUSTER) << "unable to read heartbeat interval from agency. Using " - << "default value '" << _heartbeatInterval << " ms'"; + << "default value '" << _heartbeatInterval << " ms'"; } startHeartbeatThread(_agencyCallbackRegistry.get(), _heartbeatInterval, 5, endpoints); @@ -440,6 +439,7 @@ void ClusterFeature::start() { } std::this_thread::sleep_for(std::chrono::seconds(1)); + } comm.increment("Current/Version"); @@ -523,10 +523,10 @@ void ClusterFeature::unprepare() { AgencySimpleOperationType::DELETE_OP)); // Unregister unreg.operations.push_back( - AgencyOperation("Current/ServersRegistered/" + me, - AgencySimpleOperationType::DELETE_OP)); + AgencyOperation("Current/ServersRegistered/" + me, + AgencySimpleOperationType::DELETE_OP)); unreg.operations.push_back( - AgencyOperation("Current/Version", AgencySimpleOperationType::INCREMENT_OP)); + AgencyOperation("Current/Version", AgencySimpleOperationType::INCREMENT_OP)); comm.sendTransactionWithFailover(unreg, 120.0); while (_heartbeatThread->isRunning()) { @@ -565,4 +565,10 @@ void ClusterFeature::startHeartbeatThread(AgencyCallbackRegistry* agencyCallback } } -} // arangodb +void ClusterFeature::syncDBServerStatusQuo() { + if (_heartbeatThread != nullptr) { + _heartbeatThread->syncDBServerStatusQuo(true); + } +} + + diff --git a/arangod/Cluster/ClusterFeature.h b/arangod/Cluster/ClusterFeature.h index b0deec8e64..67c997fbc7 100644 --- a/arangod/Cluster/ClusterFeature.h +++ b/arangod/Cluster/ClusterFeature.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,6 +54,8 @@ class ClusterFeature : public application_features::ApplicationFeature { return _agencyPrefix; } + void syncDBServerStatusQuo(); + protected: void startHeartbeatThread(AgencyCallbackRegistry* agencyCallbackRegistry, uint64_t interval_ms, diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index b851dcb04c..f45e32a6ef 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -3612,3 +3612,26 @@ std::unordered_map ClusterInfo::getServerAliases() { } return ret; } + +arangodb::Result ClusterInfo::getShardServers( + ShardID const& shardId, std::vector& servers) { + + READ_LOCKER(readLocker, _planProt.lock); + + auto it = _shardServers.find(shardId); + if (it != _shardServers.end()) { + servers = (*it).second; + return arangodb::Result(); + } + + LOG_TOPIC(DEBUG, Logger::CLUSTER) + << "Strange, did not find shard in _shardServers: " << shardId; + return arangodb::Result(TRI_ERROR_FAILED); + +} + + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + diff --git a/arangod/Cluster/ClusterInfo.h b/arangod/Cluster/ClusterInfo.h index f25a037eb3..206846b573 100644 --- a/arangod/Cluster/ClusterInfo.h +++ b/arangod/Cluster/ClusterInfo.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -605,6 +605,14 @@ class ClusterInfo { return _currentVersion; } + /** + * @brief Get sorted list of DB server, which serve a shard + * + * @param shardId The id of said shard + * @return List of DB servers serving the shard + */ + arangodb::Result getShardServers(ShardID const& shardId, std::vector&); + private: void loadClusterId(); diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 6306dddce2..582ab01a42 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterMethods.h b/arangod/Cluster/ClusterMethods.h index 6703ff7432..35e76b77af 100644 --- a/arangod/Cluster/ClusterMethods.h +++ b/arangod/Cluster/ClusterMethods.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterTraverser.cpp b/arangod/Cluster/ClusterTraverser.cpp index 53551fc86e..70c8c99bb4 100644 --- a/arangod/Cluster/ClusterTraverser.cpp +++ b/arangod/Cluster/ClusterTraverser.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ClusterTraverser.h b/arangod/Cluster/ClusterTraverser.h index f54c3c78e1..0c0eae2b42 100644 --- a/arangod/Cluster/ClusterTraverser.h +++ b/arangod/Cluster/ClusterTraverser.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/CreateCollection.cpp b/arangod/Cluster/CreateCollection.cpp new file mode 100644 index 0000000000..7a119b1bcf --- /dev/null +++ b/arangod/Cluster/CreateCollection.cpp @@ -0,0 +1,204 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "CreateCollection.h" +#include "MaintenanceFeature.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/FollowerInfo.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Databases.h" + +#include +#include +#include +#include + + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; + +constexpr auto WAIT_FOR_SYNC_REPL = "waitForSyncReplication"; +constexpr auto ENF_REPL_FACT = "enforceReplicationFactor"; + +CreateCollection::CreateCollection( + MaintenanceFeature& feature, ActionDescription const& desc) + : ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(DATABASE)) { + error << "database must be specified. "; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!desc.has(COLLECTION)) { + error << "cluster-wide collection must be specified. "; + } + TRI_ASSERT(desc.has(COLLECTION)); + + if (!desc.has(SHARD)) { + error << "shard must be specified. "; + } + TRI_ASSERT(desc.has(SHARD)); + + if (!desc.has(LEADER)) { + error << "shard leader must be specified. "; + } + TRI_ASSERT(desc.has(LEADER)); + + if (!desc.has(SERVER_ID)) { + error << "own server id must be specified. "; + } + TRI_ASSERT(desc.has(SERVER_ID)); + + if (!properties().hasKey(TYPE) || !properties().get(TYPE).isNumber()) { + error << "properties slice must specify collection type. "; + } + TRI_ASSERT(properties().hasKey(TYPE) && properties().get(TYPE).isNumber()); + + uint32_t const type = properties().get(TYPE).getNumber(); + if (type != TRI_COL_TYPE_DOCUMENT && type != TRI_COL_TYPE_EDGE) { + error << "invalid collection type number. " << type; + } + TRI_ASSERT(type == TRI_COL_TYPE_DOCUMENT || type == TRI_COL_TYPE_EDGE); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "CreateCollection: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + + +CreateCollection::~CreateCollection() {}; + + +bool CreateCollection::first() { + + auto const& database = _description.get(DATABASE); + auto const& collection = _description.get(COLLECTION); + auto const& shard = _description.get(SHARD); + auto const& leader = _description.get(LEADER); + auto const& props = properties(); + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "CreateCollection: creating local shard '" << database << "/" << shard + << "' for central '" << database << "/" << collection << "'"; + + try { // now try to guard the vocbase + + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto cluster = + ApplicationServer::getFeature("Cluster"); + + bool waitForRepl = + (props.hasKey(WAIT_FOR_SYNC_REPL) && + props.get(WAIT_FOR_SYNC_REPL).isBool()) ? + props.get(WAIT_FOR_SYNC_REPL).getBool() : + cluster->createWaitsForSyncReplication(); + + bool enforceReplFact = + (props.hasKey(ENF_REPL_FACT) && + props.get(ENF_REPL_FACT).isBool()) ? + props.get(ENF_REPL_FACT).getBool() : true; + + TRI_col_type_e type = static_cast(props.get(TYPE).getNumber()); + + VPackBuilder docket; + { VPackObjectBuilder d(&docket); + for (auto const& i : VPackObjectIterator(props)) { + auto const& key = i.key.copyString(); + if (key == ID || key == NAME || key == GLOB_UID || key == OBJECT_ID) { + if (key == GLOB_UID || key == OBJECT_ID) { + LOG_TOPIC(WARN, Logger::MAINTENANCE) + << "unexpected " << key << " in " << props.toJson(); + } + continue; + } + docket.add(key, i.value); + } + docket.add("planId", VPackValue(collection)); + } + + _result = Collections::create( + vocbase, shard, type, docket.slice(), waitForRepl, enforceReplFact, + [=](LogicalCollection& col) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "local collection " << database + << "/" << shard << " successfully created"; + col.followers()->setTheLeader(leader); + if (leader.empty()) { + col.followers()->clear(); + } + }); + + if (_result.fail()) { + std::stringstream error; + error << "creating local shard '" << database << "/" << shard + << "' for central '" << database << "/" << collection << "' failed: " + << _result; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error.str(); + + // Error report for phaseTwo + VPackBuilder eb; + { VPackObjectBuilder o(&eb); + eb.add("error", VPackValue(true)); + eb.add("errorMessage", VPackValue(_result.errorMessage())); + eb.add("errorNum", VPackValue(_result.errorNumber())); + eb.add(VPackValue("indexes")); + { VPackArrayBuilder a(&eb); } // [] + eb.add(VPackValue("servers")); + {VPackArrayBuilder a(&eb); // [serverId] + eb.add(VPackValue(_description.get(SERVER_ID))); }} + + // Steal buffer for maintenance feature + _feature.storeShardError(database, collection, shard, eb.steal()); + + _result.reset(TRI_ERROR_FAILED, error.str()); + // FIXMEMAINTENANCE: notify here? + return false; + } + + } catch (std::exception const& e) { // Guard failed? + std::stringstream error; + error << "action " << _description << " failed with exception " << e.what(); + LOG_TOPIC(WARN, Logger::MAINTENANCE) << error.str(); + _result.reset(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, error.str()); + // FIXMEMAINTENANCE: notify here? + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/CreateCollection.h b/arangod/Cluster/CreateCollection.h new file mode 100644 index 0000000000..91a45c7e48 --- /dev/null +++ b/arangod/Cluster/CreateCollection.h @@ -0,0 +1,50 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_CREATE_COLLECTION_H +#define ARANGODB_MAINTENANCE_CREATE_COLLECTION_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +class CreateCollection : public ActionBase { + +public: + + CreateCollection(MaintenanceFeature&, ActionDescription const& d); + + virtual ~CreateCollection(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/CreateDatabase.cpp b/arangod/Cluster/CreateDatabase.cpp new file mode 100644 index 0000000000..ec7df3e0c0 --- /dev/null +++ b/arangod/Cluster/CreateDatabase.cpp @@ -0,0 +1,106 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "ActionBase.h" +#include "CreateDatabase.h" +#include "MaintenanceFeature.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "RestServer/DatabaseFeature.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/Methods/Databases.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; + +CreateDatabase::CreateDatabase( + MaintenanceFeature& feature, ActionDescription const& desc) + : ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(DATABASE)) { + error << "database must be specified"; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "CreateDatabase: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +CreateDatabase::~CreateDatabase() {}; + +bool CreateDatabase::first() { + + VPackSlice users; + auto database = _description.get(DATABASE); + + LOG_TOPIC(INFO, Logger::MAINTENANCE) + << "CreateDatabase: creating database " << database; + + try { + + DatabaseGuard guard("_system"); + + // Assertion in constructor makes sure that we have DATABASE. + _result = Databases::create(_description.get(DATABASE), users, properties()); + if (!_result.ok()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "CreateDatabase: failed to create database " << database << ": " << _result; + + VPackBuilder eb; + { VPackObjectBuilder b(&eb); + eb.add(NAME, VPackValue(database)); + eb.add("error", VPackValue(true)); + eb.add("errorNum", VPackValue(_result.errorNumber())); + eb.add("errorMessage", VPackValue(_result.errorMessage())); } + + _feature.storeDBError(database, eb.steal()); + // FIXMEMAINTENANCE: notify here? + return false; + } + + LOG_TOPIC(INFO, Logger::MAINTENANCE) + << "CreateDatabase: database " << database << " created"; + + } catch (std::exception const& e) { + std::stringstream error; + error << "action " << _description << " failed with exception " << e.what(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "CreateDatabase: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + // FIXMEMAINTENANCE: notify here? + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/CreateDatabase.h b/arangod/Cluster/CreateDatabase.h new file mode 100644 index 0000000000..b530a81116 --- /dev/null +++ b/arangod/Cluster/CreateDatabase.h @@ -0,0 +1,49 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_CREATE_DATABASE_H +#define ARANGODB_MAINTENANCE_CREATE_DATABASE_H + +#include "ActionBase.h" + +namespace arangodb { + +namespace maintenance { + +class CreateDatabase : public ActionBase { + +public: + + CreateDatabase(MaintenanceFeature& feature, + ActionDescription const& description); + + virtual ~CreateDatabase(); + + virtual bool first() override; + +}; + +}} + +#endif diff --git a/arangod/Cluster/DBServerAgencySync.cpp b/arangod/Cluster/DBServerAgencySync.cpp index cc84fb96cf..0a97254697 100644 --- a/arangod/Cluster/DBServerAgencySync.cpp +++ b/arangod/Cluster/DBServerAgencySync.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,8 +24,13 @@ #include "DBServerAgencySync.h" #include "Basics/MutexLocker.h" +#include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" +#include "Cluster/FollowerInfo.h" #include "Cluster/HeartbeatThread.h" +#include "Cluster/Maintenance.h" +#include "Cluster/MaintenanceFeature.h" +#include "Cluster/ServerState.h" #include "Logger/Logger.h" #include "RestServer/DatabaseFeature.h" #include "Utils/DatabaseGuard.h" @@ -35,9 +40,12 @@ #include "V8Server/V8Context.h" #include "V8Server/V8DealerFeature.h" #include "VocBase/vocbase.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/Methods/Databases.h" using namespace arangodb; using namespace arangodb::application_features; +using namespace arangodb::methods; using namespace arangodb::rest; DBServerAgencySync::DBServerAgencySync(HeartbeatThread* heartbeat) @@ -52,125 +60,188 @@ void DBServerAgencySync::work() { _heartbeat->dispatchedJobResult(result); } +Result getLocalCollections(VPackBuilder& collections) { + + using namespace arangodb::basics; + Result result; + DatabaseFeature* dbfeature = nullptr; + + try { + dbfeature = ApplicationServer::getFeature("Database"); + } catch (...) {} + + if (dbfeature == nullptr) { + LOG_TOPIC(ERR, Logger::HEARTBEAT) << "Failed to get feature database"; + return Result(TRI_ERROR_INTERNAL, "Failed to get feature database"); + } + + collections.clear(); + VPackObjectBuilder c(&collections); + for (auto const& database : Databases::list()) { + + try { + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + collections.add(VPackValue(database)); + VPackObjectBuilder db(&collections); + auto cols = vocbase->collections(false); + + for (auto const& collection : cols) { + collections.add(VPackValue(collection->name())); + VPackObjectBuilder col(&collections); + collection->toVelocyPack(collections,true,false); + collections.add( + "theLeader", VPackValue(collection->followers()->getLeader())); + } + } catch (std::exception const& e) { + return Result( + TRI_ERROR_INTERNAL, + std::string("Failed to guard database ") + database + ": " + e.what()); + } + + } + + return Result(); + +} + DBServerAgencySyncResult DBServerAgencySync::execute() { // default to system database - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "DBServerAgencySync::execute starting"; - DatabaseFeature* database = + TRI_ASSERT(AgencyCommManager::isEnabled()); + AgencyComm comm; + + using namespace std::chrono; + using clock = std::chrono::steady_clock; + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::execute starting"; + DatabaseFeature* dbfeature = ApplicationServer::getFeature("Database"); - TRI_vocbase_t* const vocbase = database->systemDatabase(); + MaintenanceFeature* mfeature = + ApplicationServer::getFeature("Maintenance"); + TRI_vocbase_t* const vocbase = dbfeature->systemDatabase(); + DBServerAgencySyncResult result; if (vocbase == nullptr) { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::execute no vocbase"; return result; } + Result tmp; + VPackBuilder rb; auto clusterInfo = ClusterInfo::instance(); auto plan = clusterInfo->getPlan(); - auto current = clusterInfo->getCurrent(); - DatabaseGuard guard(*vocbase); - double startTime = TRI_microtime(); - V8Context* context = V8DealerFeature::DEALER->enterContext(vocbase, true, V8DealerFeature::ANY_CONTEXT_OR_PRIORITY); + auto serverId = arangodb::ServerState::instance()->getId(); - if (context == nullptr) { - LOG_TOPIC(WARN, arangodb::Logger::HEARTBEAT) << "DBServerAgencySync::execute: no V8 context"; + VPackBuilder local; + Result glc = getLocalCollections(local); + if (!glc.ok()) { + // FIXMEMAINTENANCE: if this fails here, then result is empty, is this + // intended? I also notice that there is another Result object "tmp" + // that is going to eat bad results in few lines later. Again, is + // that the correct action? If so, how about supporting comments in + // the code for both. return result; } - TRI_DEFER(V8DealerFeature::DEALER->exitContext(context)); + auto start = clock::now(); + try { + // in previous life handlePlanChange + + VPackObjectBuilder o(&rb); - double now = TRI_microtime(); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::phaseOne"; + tmp = arangodb::maintenance::phaseOne( + plan->slice(), local.slice(), serverId, *mfeature, rb); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::phaseOne done"; - if (now - startTime > 5.0) { - LOG_TOPIC(WARN, arangodb::Logger::HEARTBEAT) << "DBServerAgencySync::execute took " << Logger::FIXED(now - startTime) << " to get free V8 context, starting handlePlanChange now"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo"; + glc = getLocalCollections(local); + // We intentionally refetch local collections here, such that phase 2 + // can already see potential changes introduced by phase 1. The two + // phases are sufficiently independent that this is OK. + LOG_TOPIC(TRACE, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo - local state: " << local.toJson(); + if (!glc.ok()) { + return result; + } + + auto current = clusterInfo->getCurrent(); + LOG_TOPIC(TRACE, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo - current state: " << current->toJson(); + + tmp = arangodb::maintenance::phaseTwo( + plan->slice(), current->slice(), local.slice(), serverId, *mfeature, rb); + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo done"; + + } catch (std::exception const& e) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Failed to handle plan change: " << e.what(); } - auto isolate = context->_isolate; + if (rb.isClosed()) { + // FIXMEMAINTENANCE: when would rb not be closed? and if "catch" + // just happened, would you want to be doing this anyway? - try { - v8::HandleScope scope(isolate); + auto report = rb.slice(); + if (report.isObject()) { - // execute script inside the context - auto file = TRI_V8_ASCII_STRING(isolate, "handlePlanChange"); - auto content = - TRI_V8_ASCII_STRING(isolate, "require('@arangodb/cluster').handlePlanChange"); - - v8::TryCatch tryCatch; - v8::Handle handlePlanChange = TRI_ExecuteJavaScriptString( - isolate, isolate->GetCurrentContext(), content, file, false); - - if (tryCatch.HasCaught()) { - TRI_LogV8Exception(isolate, &tryCatch); - return result; - } - - if (!handlePlanChange->IsFunction()) { - LOG_TOPIC(ERR, Logger::HEARTBEAT) << "handlePlanChange is not a function"; - return result; - } - - v8::Handle func = - v8::Handle::Cast(handlePlanChange); - v8::Handle args[2]; - // Keep the shared_ptr to the builder while we run TRI_VPackToV8 on the - // slice(), just to be on the safe side: - auto builder = clusterInfo->getPlan(); - args[0] = TRI_VPackToV8(isolate, builder->slice()); - builder = clusterInfo->getCurrent(); - args[1] = TRI_VPackToV8(isolate, builder->slice()); - - v8::Handle res = - func->Call(isolate->GetCurrentContext()->Global(), 2, args); - - if (tryCatch.HasCaught()) { - TRI_LogV8Exception(isolate, &tryCatch); - return result; - } - - result.success = true; // unless overwritten by actual result - - if (res->IsObject()) { - v8::Handle o = res->ToObject(); - - v8::Handle names = o->GetOwnPropertyNames(); - uint32_t const n = names->Length(); + std::vector agency = {"phaseTwo", "agency"}; + if (report.hasKey(agency) && report.get(agency).isObject()) { - for (uint32_t i = 0; i < n; ++i) { - v8::Handle key = names->Get(i); - v8::String::Utf8Value str(key); - - v8::Handle value = o->Get(key); + auto phaseTwo = report.get(agency); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "DBServerAgencySync reporting to Current: " << phaseTwo.toJson(); + + // Report to current + if (!phaseTwo.isEmptyObject()) { - if (value->IsNumber()) { - if (strcmp(*str, "plan") == 0) { - result.planVersion = - static_cast(value->ToInteger()->Value()); - } else if (strcmp(*str, "current") == 0) { - result.currentVersion = - static_cast(value->ToInteger()->Value()); + std::vector operations; + for (auto const& ao : VPackObjectIterator(phaseTwo)) { + auto const key = ao.key.copyString(); + auto const op = ao.value.get("op").copyString(); + if (op == "set") { + auto const value = ao.value.get("payload"); + operations.push_back( + AgencyOperation(key, AgencyValueOperationType::SET, value)); + } else if (op == "delete") { + operations.push_back( + AgencyOperation(key, AgencySimpleOperationType::DELETE_OP)); + } + } + operations.push_back( + AgencyOperation( + "Current/Version", AgencySimpleOperationType::INCREMENT_OP)); + AgencyWriteTransaction currentTransaction(operations); + AgencyCommResult r = comm.sendTransactionWithFailover(currentTransaction); + if (!r.successful()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "Error reporting to agency"; + } else { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "Invalidating current in ClusterInfo"; + clusterInfo->invalidateCurrent(); } - } else if (value->IsBoolean() && strcmp(*str, "success")) { - result.success = TRI_ObjectToBoolean(value); } } - } else { - LOG_TOPIC(ERR, Logger::HEARTBEAT) - << "handlePlanChange returned a non-object"; - return result; + + // FIXMEMAINTENANCE: If comm.sendTransactionWithFailover() + // fails, the result is ok() based upon phaseTwo()'s execution? + result = DBServerAgencySyncResult( + tmp.ok(), + report.hasKey("Plan") ? + report.get("Plan").get("Version").getNumber() : 0, + report.hasKey("Current") ? + report.get("Current").get("Version").getNumber() : 0); + } - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) - << "DBServerAgencySync::execute back from JS"; - // invalidate our local cache, even if an error occurred - clusterInfo->flush(); - } catch (...) { + } + + auto took = duration(clock::now()-start).count(); + if (took > 30.0) { + LOG_TOPIC(WARN, Logger::MAINTENANCE) << "DBServerAgencySync::execute " + "took " << took << " s to execute handlePlanChange"; } - now = TRI_microtime(); - if (now - startTime > 30.0) { - LOG_TOPIC(WARN, Logger::HEARTBEAT) << "DBServerAgencySync::execute " - "took " << Logger::FIXED(now - startTime) << " s to execute handlePlanChange"; - } return result; } diff --git a/arangod/Cluster/DBServerAgencySync.h b/arangod/Cluster/DBServerAgencySync.h index c21fc4544b..6ef0787aa2 100644 --- a/arangod/Cluster/DBServerAgencySync.h +++ b/arangod/Cluster/DBServerAgencySync.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,6 +37,9 @@ struct DBServerAgencySyncResult { DBServerAgencySyncResult() : success(false), planVersion(0), currentVersion(0) {} + DBServerAgencySyncResult(bool s, uint64_t p, uint64_t c) + : success(s), planVersion(p), currentVersion(c) {} + DBServerAgencySyncResult(const DBServerAgencySyncResult& other) : success(other.success), planVersion(other.planVersion), diff --git a/arangod/Cluster/DropCollection.cpp b/arangod/Cluster/DropCollection.cpp new file mode 100644 index 0000000000..eb45b71038 --- /dev/null +++ b/arangod/Cluster/DropCollection.cpp @@ -0,0 +1,104 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "DropCollection.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Databases.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; + +DropCollection::DropCollection( + MaintenanceFeature& feature, ActionDescription const& d) : + ActionBase(feature, d) { + + std::stringstream error; + + if (!d.has(COLLECTION)) { + error << "collection must be specified. "; + } + TRI_ASSERT(d.has(COLLECTION)); + + if (!d.has(DATABASE)) { + error << "database must be specified. "; + } + TRI_ASSERT(d.has(DATABASE)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropCollectio: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +DropCollection::~DropCollection() {}; + +bool DropCollection::first() { + + auto const& database = _description.get(DATABASE); + auto const& collection = _description.get(COLLECTION); + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "DropCollection: dropping local shard '" << database << "/" << collection; + + try { + + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + Result found = methods::Collections::lookup( + vocbase, collection, [&](LogicalCollection& coll) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Dropping local collection " + collection; + _result = Collections::drop(vocbase, &coll, false, 120); + }); + + if (found.fail()) { + std::stringstream error; + error << "failed to lookup local collection " << database << "/" << collection; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropCollection: " << error.str(); + _result.reset(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, error.str()); + return false; + } + + } catch (std::exception const& e) { + std::stringstream error; + error << " action " << _description << " failed with exception " << e.what(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/DropCollection.h b/arangod/Cluster/DropCollection.h new file mode 100644 index 0000000000..df5ddb3f1c --- /dev/null +++ b/arangod/Cluster/DropCollection.h @@ -0,0 +1,50 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_DROP_COLLECTION_H +#define ARANGODB_MAINTENANCE_DROP_COLLECTION_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +class DropCollection : public ActionBase { + +public: + + DropCollection(MaintenanceFeature&, ActionDescription const&); + + virtual ~DropCollection(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/DropDatabase.cpp b/arangod/Cluster/DropDatabase.cpp new file mode 100644 index 0000000000..75b8258bc5 --- /dev/null +++ b/arangod/Cluster/DropDatabase.cpp @@ -0,0 +1,85 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "DropDatabase.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "RestServer/DatabaseFeature.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/Methods/Databases.h" + +using namespace arangodb::application_features; +using namespace arangodb::methods; +using namespace arangodb::maintenance; +using namespace arangodb; + +DropDatabase::DropDatabase( + MaintenanceFeature& feature, ActionDescription const& desc) + : ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(DATABASE)) { + error << "database must be specified"; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropDatabase: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +DropDatabase::~DropDatabase() {}; + +bool DropDatabase::first() { + + std::string const database = _description.get(DATABASE); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DropDatabase: dropping " << database; + + try { + DatabaseGuard guard("_system"); + auto vocbase = &guard.database(); + + _result = Databases::drop(vocbase, database); + if (!_result.ok()) { + LOG_TOPIC(ERR, Logger::AGENCY) + << "DropDatabase: dropping database " << database << " failed: " + << _result.errorMessage(); + return false; + } + } catch (std::exception const& e) { + std::stringstream error; + error << "action " << _description << " failed with exception " << e.what(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/DropDatabase.h b/arangod/Cluster/DropDatabase.h new file mode 100644 index 0000000000..760572cfbe --- /dev/null +++ b/arangod/Cluster/DropDatabase.h @@ -0,0 +1,51 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_DROP_DATABASE_H +#define ARANGODB_MAINTENANCE_DROP_DATABASE_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { + +namespace maintenance { + +class DropDatabase : public ActionBase { + +public: + + DropDatabase(MaintenanceFeature&, ActionDescription const&); + + virtual ~DropDatabase(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/DropIndex.cpp b/arangod/Cluster/DropIndex.cpp new file mode 100644 index 0000000000..81b1cc6b0f --- /dev/null +++ b/arangod/Cluster/DropIndex.cpp @@ -0,0 +1,123 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "DropIndex.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Indexes.h" +#include "VocBase/Methods/Databases.h" + +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; +using namespace arangodb; + +DropIndex::DropIndex( + MaintenanceFeature& feature, ActionDescription const& d) : + ActionBase(feature, d) { + + std::stringstream error; + + if (!d.has(COLLECTION)) { + error << "collection must be specified. "; + } + TRI_ASSERT(d.has(COLLECTION)); + + if (!d.has(DATABASE)) { + error << "database must be specified. "; + } + TRI_ASSERT(d.has(DATABASE)); + + if (!d.has(INDEX)) { + error << "index id must be stecified. "; + } + TRI_ASSERT(d.has(INDEX)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropIndex: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +DropIndex::~DropIndex() {}; + +bool DropIndex::first() { + + auto const& database = _description.get(DATABASE); + auto const& collection = _description.get(COLLECTION); + auto const& id = _description.get(INDEX); + + VPackBuilder index; + index.add(VPackValue(_description.get(INDEX))); + + try { + + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto col = vocbase->lookupCollection(collection); + if (col == nullptr) { + std::stringstream error; + error << "failed to lookup local collection " << collection + << " in database " << database; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropIndex: " << error.str(); + _result.reset(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, error.str()); + return false; + } + + // FIXMEMAINTENANCE: Why doing the actual work in a callback? + Result found = methods::Collections::lookup( + vocbase, collection, [&](LogicalCollection& coll) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Dropping local index " + collection + "/" + id; + _result = Indexes::drop(&coll, index.slice()); + }); + + if (found.fail()) { + std::stringstream error; + error << "failed to lookup local collection " << collection + << "in database " + database; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropIndex: " << error.str(); + _result.reset(TRI_ERROR_ARANGO_INDEX_NOT_FOUND, error.str()); + return false; + } + + } catch (std::exception const& e) { + std::stringstream error; + error << "action " << _description << " failed with exception " << e.what(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "DropIndex " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/DropIndex.h b/arangod/Cluster/DropIndex.h new file mode 100644 index 0000000000..0987961242 --- /dev/null +++ b/arangod/Cluster/DropIndex.h @@ -0,0 +1,50 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_DROP_INDEX_H +#define ARANGODB_MAINTENANCE_DROP_INDEX_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +class DropIndex : public ActionBase { + +public: + + DropIndex(MaintenanceFeature&, ActionDescription const&); + + virtual ~DropIndex(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/EnsureIndex.cpp b/arangod/Cluster/EnsureIndex.cpp new file mode 100644 index 0000000000..61fcb7663e --- /dev/null +++ b/arangod/Cluster/EnsureIndex.cpp @@ -0,0 +1,169 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include + +#include "EnsureIndex.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/MaintenanceFeature.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Databases.h" +#include "VocBase/Methods/Indexes.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; + +EnsureIndex::EnsureIndex( + MaintenanceFeature& feature, ActionDescription const& desc) : + ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(DATABASE)) { + error << "database must be specified. "; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!desc.has(COLLECTION)) { + error << "cluster-wide collection must be specified. "; + } + TRI_ASSERT(desc.has(COLLECTION)); + + if (!desc.has(SHARD)) { + error << "shard must be specified. "; + } + TRI_ASSERT(desc.has(SHARD)); + + if (!properties().hasKey(ID)) { + error << "index properties must include id. "; + } + TRI_ASSERT(properties().hasKey(ID)); + + if (!desc.has(TYPE)) { + error << "index type must be specified - discriminatory. "; + } + TRI_ASSERT(desc.has(TYPE)); + + if (!desc.has(FIELDS)) { + error << "index fields must be specified - discriminatory. "; + } + TRI_ASSERT(desc.has(FIELDS)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "EnsureIndex: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +EnsureIndex::~EnsureIndex() {}; + +bool EnsureIndex::first() { + + arangodb::Result res; + + auto const& database = _description.get(DATABASE); + auto const& collection = _description.get(COLLECTION); + auto const& shard = _description.get(SHARD); + auto const& id = properties().get(ID).copyString(); + + VPackBuilder body; + + try { // now try to guard the database + + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto col = vocbase->lookupCollection(shard); + if (col == nullptr) { + std::stringstream error; + error << "failed to lookup local collection " << shard + << " in database " + database; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "EnsureIndex: " << error.str(); + _result.reset(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, error.str()); + return false; + } + + auto const props = properties(); + { VPackObjectBuilder b(&body); + body.add(COLLECTION, VPackValue(shard)); + for (auto const& i : VPackObjectIterator(props)) { + body.add(i.key.copyString(), i.value); + }} + + VPackBuilder index; + _result = methods::Indexes::ensureIndex(col.get(), body.slice(), true, index); + + if (_result.ok()) { + VPackSlice created = index.slice().get("isNewlyCreated"); + std::string log = std::string("Index ") + id; + log += (created.isBool() && created.getBool() ? std::string(" created") + : std::string(" updated")); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << log; + } else { + std::stringstream error; + error << "failed to ensure index " << body.slice().toJson() << " " + << _result.errorMessage(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "EnsureIndex: " << error.str(); + + VPackBuilder eb; + { VPackObjectBuilder o(&eb); + eb.add("error", VPackValue(true)); + eb.add("errorMessage", VPackValue(_result.errorMessage())); + eb.add("errorNum", VPackValue(_result.errorNumber())); + eb.add(ID, VPackValue(id)); } + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Reporting error " << eb.toJson(); + + // FIXMEMAINTENANCE: If this action is refused due to missing + // components in description, no IndexError gets produced. But + // then, if you are missing components, such as database name, will + // you be able to produce an IndexError? + + _feature.storeIndexError(database, collection, shard, id, eb.steal()); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + notify(); + return false; + } + + } catch (std::exception const& e) { // Guard failed? + std::stringstream error; + error << "action " << _description << " failed with exception " << e.what(); + LOG_TOPIC(WARN, Logger::MAINTENANCE) << "EnsureIndex: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/EnsureIndex.h b/arangod/Cluster/EnsureIndex.h new file mode 100644 index 0000000000..98e555230f --- /dev/null +++ b/arangod/Cluster/EnsureIndex.h @@ -0,0 +1,50 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_ENSURE_INDEX_H +#define ARANGODB_MAINTENANCE_ENSURE_INDEX_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +class EnsureIndex : public ActionBase { + +public: + + EnsureIndex(MaintenanceFeature&, ActionDescription const& d); + + virtual ~EnsureIndex(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/FollowerInfo.cpp b/arangod/Cluster/FollowerInfo.cpp index 719295be81..f69a96df77 100644 --- a/arangod/Cluster/FollowerInfo.cpp +++ b/arangod/Cluster/FollowerInfo.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -190,6 +190,10 @@ bool FollowerInfo::remove(ServerID const& sid) { // such an important decision like dropping a follower. return false; } + + LOG_TOPIC(DEBUG, Logger::CLUSTER) + << "Removing follower " << sid << " from " << _docColl->name(); + MUTEX_LOCKER(locker, _mutex); // First check if there is anything to do: @@ -280,6 +284,10 @@ bool FollowerInfo::remove(ServerID const& sid) { LOG_TOPIC(ERR, Logger::CLUSTER) << "FollowerInfo::remove, timeout in agency operation for key " << path; } + + LOG_TOPIC(DEBUG, Logger::CLUSTER) + << "Removing follower " << sid << " from " << _docColl->name() << "succeeded: " << success; + return success; } diff --git a/arangod/Cluster/FollowerInfo.h b/arangod/Cluster/FollowerInfo.h index 5d429a1b70..48fd5b2a94 100644 --- a/arangod/Cluster/FollowerInfo.h +++ b/arangod/Cluster/FollowerInfo.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index 36eb152966..c0becb6e3c 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -265,7 +265,7 @@ void HeartbeatThread::runDBServer() { if (version > _desiredVersions->plan) { _desiredVersions->plan = version; LOG_TOPIC(DEBUG, Logger::HEARTBEAT) - << "Desired Current Version is now " << _desiredVersions->plan; + << "Desired Plan Version is now " << _desiredVersions->plan; doSync = true; } } @@ -291,6 +291,49 @@ void HeartbeatThread::runDBServer() { } } + std::function updateCurrent = + [=](VPackSlice const& result) { + + if (!result.isNumber()) { + LOG_TOPIC(ERR, Logger::HEARTBEAT) + << "Plan Version is not a number! " << result.toJson(); + return false; + } + + uint64_t version = result.getNumber(); + bool doSync = false; + + { + MUTEX_LOCKER(mutexLocker, *_statusLock); + if (version > _desiredVersions->current) { + _desiredVersions->current = version; + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) + << "Desired Current Version is now " << _desiredVersions->plan; + doSync = true; + } + } + + if (doSync) { + syncDBServerStatusQuo(true); + } + + return true; + }; + + auto currentAgencyCallback = std::make_shared( + _agency, "Current/Version", updateCurrent, true); + + registered = false; + while (!registered) { + registered = + _agencyCallbackRegistry->registerCallback(currentAgencyCallback); + if (!registered) { + LOG_TOPIC(ERR, Logger::HEARTBEAT) + << "Couldn't register current change in agency!"; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } + // we check Current/Version every few heartbeats: int const currentCountStart = 1; // set to 1 by Max to speed up discovery int currentCount = currentCountStart; @@ -414,8 +457,9 @@ void HeartbeatThread::runDBServer() { } if (!wasNotified) { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Lock reached timeout"; + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Heart beating..."; planAgencyCallback->refetchAndUpdate(true, false); + currentAgencyCallback->refetchAndUpdate(true, false); } else { // mop: a plan change returned successfully... // recheck and redispatch in case our desired versions increased @@ -434,6 +478,7 @@ void HeartbeatThread::runDBServer() { } } + _agencyCallbackRegistry->unregisterCallback(currentAgencyCallback); _agencyCallbackRegistry->unregisterCallback(planAgencyCallback); } @@ -991,29 +1036,15 @@ void HeartbeatThread::beginShutdown() { void HeartbeatThread::dispatchedJobResult(DBServerAgencySyncResult result) { LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Dispatched job returned!"; - bool doSleep = false; - { - MUTEX_LOCKER(mutexLocker, *_statusLock); - if (result.success) { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) - << "Sync request successful. Now have Plan " << result.planVersion - << ", Current " << result.currentVersion; - _currentVersions = AgencyVersions(result); - } else { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Sync request failed!"; - // mop: we will retry immediately so wait at least a LITTLE bit - doSleep = true; - } + MUTEX_LOCKER(mutexLocker, *_statusLock); + if (result.success) { + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) + << "Sync request successful. Now have Plan " << result.planVersion + << ", Current " << result.currentVersion; + _currentVersions = AgencyVersions(result); + } else { + LOG_TOPIC(ERR, Logger::HEARTBEAT) << "Sync request failed!"; } - if (doSleep) { - // Sleep a little longer, since this might be due to some synchronization - // of shards going on in the background - std::this_thread::sleep_for(std::chrono::microseconds(500000)); - std::this_thread::sleep_for(std::chrono::microseconds(500000)); - } - CONDITION_LOCKER(guard, _condition); - _wasNotified = true; - _condition.signal(); } //////////////////////////////////////////////////////////////////////////////// @@ -1134,34 +1165,34 @@ bool HeartbeatThread::handlePlanChangeCoordinator(uint64_t currentPlanVersion) { //////////////////////////////////////////////////////////////////////////////// void HeartbeatThread::syncDBServerStatusQuo(bool asyncPush) { - bool shouldUpdate = false; MUTEX_LOCKER(mutexLocker, *_statusLock); - + bool shouldUpdate = false; + if (_desiredVersions->plan > _currentVersions.plan) { LOG_TOPIC(DEBUG, Logger::HEARTBEAT) - << "Plan version " << _currentVersions.plan - << " is lower than desired version " << _desiredVersions->plan; + << "Plan version " << _currentVersions.plan + << " is lower than desired version " << _desiredVersions->plan; shouldUpdate = true; } if (_desiredVersions->current > _currentVersions.current) { LOG_TOPIC(DEBUG, Logger::HEARTBEAT) - << "Current version " << _currentVersions.current - << " is lower than desired version " << _desiredVersions->current; + << "Current version " << _currentVersions.current + << " is lower than desired version " << _desiredVersions->current; shouldUpdate = true; } - + // 7.4 seconds is just less than half the 15 seconds agency uses to declare dead server, // perform a safety execution of job in case other plan changes somehow incomplete or undetected double now = TRI_microtime(); if (now > _lastSyncTime + 7.4 || asyncPush) { shouldUpdate = true; } - + if (!shouldUpdate) { return; } - + // First invalidate the caches in ClusterInfo: auto ci = ClusterInfo::instance(); if (_desiredVersions->plan > ci->getPlanVersion()) { @@ -1170,21 +1201,22 @@ void HeartbeatThread::syncDBServerStatusQuo(bool asyncPush) { if (_desiredVersions->current > ci->getCurrentVersion()) { ci->invalidateCurrent(); } - + if (_backgroundJobScheduledOrRunning) { _launchAnotherBackgroundJob = true; return; } - + // schedule a job for the change: uint64_t jobNr = ++_backgroundJobsPosted; LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " << jobNr; _backgroundJobScheduledOrRunning = true; - + // the JobGuard is in the operator() of HeartbeatBackgroundJob _lastSyncTime = TRI_microtime(); SchedulerFeature::SCHEDULER->post( - HeartbeatBackgroundJob(shared_from_this(), _lastSyncTime), false); + HeartbeatBackgroundJob(shared_from_this(), _lastSyncTime), false); + } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Cluster/HeartbeatThread.h b/arangod/Cluster/HeartbeatThread.h index a97ee2c114..99d3ca1a1a 100644 --- a/arangod/Cluster/HeartbeatThread.h +++ b/arangod/Cluster/HeartbeatThread.h @@ -162,12 +162,14 @@ class HeartbeatThread : public CriticalThread, /// @brief bring the db server in sync with the desired state ////////////////////////////////////////////////////////////////////////////// +public: void syncDBServerStatusQuo(bool asyncPush = false); ////////////////////////////////////////////////////////////////////////////// /// @brief update the local agent pool from the slice ////////////////////////////////////////////////////////////////////////////// + private: void updateAgentPool(arangodb::velocypack::Slice const& agentPool); ////////////////////////////////////////////////////////////////////////////// @@ -176,7 +178,6 @@ class HeartbeatThread : public CriticalThread, void updateServerMode(arangodb::velocypack::Slice const& readOnlySlice); - private: ////////////////////////////////////////////////////////////////////////////// /// @brief AgencyCallbackRegistry ////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Cluster/Maintenance.cpp b/arangod/Cluster/Maintenance.cpp new file mode 100644 index 0000000000..091a14de7b --- /dev/null +++ b/arangod/Cluster/Maintenance.cpp @@ -0,0 +1,1127 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/StringUtils.h" +#include "Basics/VelocyPackHelper.h" +#include "Logger/Logger.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" +#include "Cluster/FollowerInfo.h" +#include "Cluster/Maintenance.h" +#include "Indexes/Index.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/Methods/Databases.h" + +#include +#include +#include +#include +#include + +#include +#include + +using namespace arangodb; +using namespace arangodb::basics; +using namespace arangodb::maintenance; +using namespace arangodb::methods; +using namespace arangodb::basics::StringUtils; + +static std::vector const cmp { + "journalSize", "waitForSync", "doCompact", "indexBuckets"}; +static std::string const CURRENT_COLLECTIONS("Current/Collections/"); +static std::string const CURRENT_DATABASES("Current/Databases/"); +static std::string const DATABASES("Databases"); +static std::string const ERROR_MESSAGE("errorMessage"); +static std::string const ERROR_NUM("errorNum"); +static std::string const ERROR_STR("error"); +static std::string const PLAN_ID("planId"); +static std::string const PRIMARY("primary"); +static std::string const SERVERS("servers"); +static std::string const SELECTIVITY_ESTIMATE("selectivityEstimate"); +static std::string const COLLECTIONS("Collections"); +static std::string const DB ("/_db/"); +static std::string const FOLLOWER_ID("followerId"); +static VPackValue const VP_DELETE("delete"); +static VPackValue const VP_SET("set"); +static std::string const OP("op"); +static std::string const UNDERSCORE("_"); + +static int indexOf(VPackSlice const& slice, std::string const& val) { + size_t counter = 0; + if (slice.isArray()) { + for (auto const& entry : VPackArrayIterator(slice)) { + if (entry.isString()) { + if (entry.copyString() == val) { + return counter; + } + } + counter++; + } + } + return -1; +} + +static std::shared_ptr createProps(VPackSlice const& s) { + TRI_ASSERT(s.isObject()); + return std::make_shared( + arangodb::velocypack::Collection::remove(s, + std::unordered_set({ID, NAME}))); +} + +static std::shared_ptr compareRelevantProps ( + VPackSlice const& first, VPackSlice const& second) { + auto result = std::make_shared(); + { VPackObjectBuilder b(result.get()); + for (auto const& property : cmp) { + auto const& planned = first.get(property); + if (planned != second.get(property)) { // Register any change + result->add(property,planned); + } + } + } + return result; +} + + +static VPackBuilder compareIndexes( + std::string const& dbname, std::string const& collname, + std::string const& shname, + VPackSlice const& plan, VPackSlice const& local, + MaintenanceFeature::errors_t const& errors, + std::unordered_set& indis) { + + VPackBuilder builder; + { VPackArrayBuilder a(&builder); + if (plan.isArray()) { + for (auto const& pindex : VPackArrayIterator(plan)) { + + // Skip primary and edge indexes + auto const& ptype = pindex.get(TYPE).copyString(); + if (ptype == PRIMARY || ptype == EDGE) { + continue; + } + VPackSlice planId = pindex.get(ID); + TRI_ASSERT(planId.isString()); + std::string planIdS = planId.copyString(); + std::string planIdWithColl = shname + "/" + planIdS; + indis.emplace(planIdWithColl); + + // See, if we already have an index with the id given in the Plan: + bool found = false; + if (local.isArray()) { + for (auto const& lindex : VPackArrayIterator(local)) { + + // Skip primary and edge indexes + auto const& ltype = lindex.get(TYPE).copyString(); + if (ltype == PRIMARY || ltype == EDGE) { + continue; + } + + VPackSlice localId = lindex.get(ID); + TRI_ASSERT(localId.isString()); + // The local ID has the form /, to compare, + // we need to extract the local ID: + std::string localIdS = localId.copyString(); + auto pos = localIdS.find('/'); + if (pos != std::string::npos) { + localIdS = localIdS.substr(pos+1); + } + + if (localIdS == planIdS) { + // Already have this id, so abort search: + found = true; + // We should be done now, this index already exists, and since + // one cannot legally change the properties of an index, we + // should be fine. However, for robustness sake, we compare, + // if the local index found actually has the right properties, + // if not, we schedule a dropIndex action: + if (!arangodb::Index::Compare(pindex, lindex)) { + // To achieve this, we remove the long version of the ID + // from the indis set. This way, the local index will be + // dropped further down in handleLocalShard: + indis.erase(planIdWithColl); + } + break; + } + } + } + if (!found) { + // Finally check if we have an error for this index: + bool haveError = false; + std::string errorKey = dbname + "/" + collname + "/" + shname; + auto it1 = errors.indexes.find(errorKey); + if (it1 != errors.indexes.end()) { + auto it2 = it1->second.find(planIdS); + if (it2 != it1->second.end()) { + haveError = true; + } + } + if (!haveError) { + builder.add(pindex); + } + } + } + } + } + + return builder; +} + + +void handlePlanShard( + VPackSlice const& cprops, VPackSlice const& ldb, + std::string const& dbname, std::string const& colname, std::string const& shname, + std::string const& serverId, std::string const& leaderId, + std::unordered_set& commonShrds, std::unordered_set& indis, + MaintenanceFeature::errors_t& errors, std::vector& actions) { + + bool shouldBeLeading = serverId == leaderId; + + commonShrds.emplace(shname); + auto props = createProps(cprops); // Only once might need often! + + if (ldb.hasKey(shname)) { // Have local collection with that name + auto const lcol = ldb.get(shname); + bool leading = lcol.get(LEADER).copyString().empty(); + auto const properties = compareRelevantProps(cprops, lcol); + + // If comparison has brought any updates + if (properties->slice() != VPackSlice::emptyObjectSlice() + || leading != shouldBeLeading) { + + if (errors.shards.find(dbname + "/" + colname + "/" + shname) == + errors.shards.end()) { + actions.emplace_back( + ActionDescription( + {{NAME, "UpdateCollection"}, {DATABASE, dbname}, {COLLECTION, shname}, + {LEADER, shouldBeLeading ? std::string() : leaderId}, + {LOCAL_LEADER, lcol.get(LEADER).copyString()}}, + properties)); + } else { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Previous failure exists for local shard " << dbname + << "/" << shname << "for central " << dbname << "/" << colname + <<"- skipping"; + } + } + + // Indexes + if (cprops.hasKey(INDEXES)) { + auto const& pindexes = cprops.get(INDEXES); + auto const& lindexes = lcol.get(INDEXES); + auto difference = compareIndexes(dbname, colname, shname, + pindexes, lindexes, errors, indis); + + if (difference.slice().isArray()) { + for (auto const& index : VPackArrayIterator(difference.slice())) { + actions.emplace_back( + ActionDescription({{NAME, "EnsureIndex"}, {DATABASE, dbname}, + {COLLECTION, colname}, {TYPE, index.get(TYPE).copyString()}, + {FIELDS, index.get(FIELDS).toJson()}, {SHARD, shname}, {ID, index.get(ID).copyString()}}, + std::make_shared(index))); + } + } + } + } else { // Create the sucker, if not a previous error stops us + if (errors.shards.find(dbname + "/" + colname + "/" + shname) == + errors.shards.end()) { + actions.emplace_back( + ActionDescription( + {{NAME, "CreateCollection"}, {COLLECTION, colname}, {SHARD, shname}, + {DATABASE, dbname}, {SERVER_ID, serverId}, {LEADER, shouldBeLeading ? std::string() : leaderId}}, + props)); + } else { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Previous failure exists for creating local shard " << dbname + << "/" << shname << "for central " << dbname << "/" << colname + <<"- skipping"; + } + } +} + + +void handleLocalShard( + std::string const& dbname, std::string const& colname, VPackSlice const& cprops, + VPackSlice const& shardMap, std::unordered_set& commonShrds, + std::unordered_set& indis, std::string serverId, + std::vector& actions) { + + bool drop = false; + std::unordered_set::const_iterator it; + + std::string plannedLeader; + if (shardMap.hasKey(colname) && shardMap.get(colname).isArray()) { + plannedLeader = shardMap.get(colname)[0].copyString(); + } + bool localLeader = cprops.get(LEADER).copyString().empty(); + if (plannedLeader == UNDERSCORE + serverId && localLeader) { + actions.emplace_back( + ActionDescription ( + {{NAME, "ResignShardLeadership"}, {DATABASE, dbname}, {SHARD, colname}})); + } else { + + // check if shard is in plan, if not drop it + if (commonShrds.empty()) { + drop = true; + } else { + it = std::find(commonShrds.begin(), commonShrds.end(), colname); + if (it == commonShrds.end()) { + drop = true; + } + } + + if (drop) { + actions.emplace_back( + ActionDescription({{NAME, "DropCollection"}, + {DATABASE, dbname}, {COLLECTION, colname}})); + } else { + // The shard exists in both Plan and Local + commonShrds.erase(it); // it not a common shard? + + // We only drop indexes, when collection is not being dropped already + if (cprops.hasKey(INDEXES)) { + if (cprops.get(INDEXES).isArray()) { + for (auto const& index : + VPackArrayIterator(cprops.get(INDEXES))) { + auto const& type = index.get(TYPE).copyString(); + if (type != PRIMARY && type != EDGE) { + std::string const id = index.get(ID).copyString(); + + // check if index is in plan + if (indis.find(colname + "/" + id) != indis.end() || + indis.find(id) != indis.end()) { + indis.erase(id); + } else { + actions.emplace_back( + ActionDescription( + {{NAME, "DropIndex"}, {DATABASE, dbname}, {COLLECTION, colname}, {"index", id}})); + } + } + } + } + } + } + } +} + +/// @brief Get a map shardName -> servers +VPackBuilder getShardMap (VPackSlice const& plan) { + VPackBuilder shardMap; + { VPackObjectBuilder o(&shardMap); + for (auto database : VPackObjectIterator(plan)) { + for (auto collection : VPackObjectIterator(database.value)) { + for (auto shard : VPackObjectIterator(collection.value.get(SHARDS))) { + std::string const shName = shard.key.copyString(); + shardMap.add(shName, shard.value); + } + } + }} + return shardMap; +} + + +struct NotEmpty { + bool operator()(const std::string& s) { return !s.empty(); } +}; + +/* + Replaced by StringUtils::split +inline static std::vector split(std::string const& key) { + + std::vector result; + if (key.empty()) { + return result; + } + + std::string::size_type p = 0; + std::string::size_type q; + while ((q = key.find('/', p)) != std::string::npos) { + result.emplace_back(key, p, q - p); + p = q + 1; + } + result.emplace_back(key, p); + result.erase(std::find_if(result.rbegin(), result.rend(), NotEmpty()).base(), + result.end()); + return result; +}*/ + + + +/// @brief calculate difference between plan and local for for databases +arangodb::Result arangodb::maintenance::diffPlanLocal ( + VPackSlice const& plan, VPackSlice const& local, std::string const& serverId, + MaintenanceFeature::errors_t& errors, std::vector& actions) { + + arangodb::Result result; + std::unordered_set commonShrds; // Intersection collections plan&local + std::unordered_set indis; // Intersection indexes plan&local + + // Plan to local mismatch ---------------------------------------------------- + // Create or modify if local databases are affected + auto pdbs = plan.get("Databases"); + for (auto const& pdb : VPackObjectIterator(pdbs)) { + auto const& dbname = pdb.key.copyString(); + if (!local.hasKey(dbname)) { + if (errors.databases.find(dbname) == errors.databases.end()) { + actions.emplace_back( + ActionDescription({{NAME, "CreateDatabase"}, {DATABASE, dbname}})); + } else { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Previous failure exists for creating database " << dbname + << "skipping"; + } + } + } + + // Drop databases, which are no longer in plan + for (auto const& ldb : VPackObjectIterator(local)) { + auto const& dbname = ldb.key.copyString(); + if (!plan.hasKey(std::vector {"Databases", dbname})) { + actions.emplace_back( + ActionDescription({{NAME, "DropDatabase"}, {DATABASE, dbname}})); + } + } + + // Check errors for databases, which are no longer in plan and remove from errors + for (auto& database : errors.databases) { + if (!plan.hasKey(std::vector{"Databases", database.first})) { + database.second.reset(); + } + } + + // Create or modify if local collections are affected + pdbs = plan.get(COLLECTIONS); + for (auto const& pdb : VPackObjectIterator(pdbs)) { // for each db in Plan + auto const& dbname = pdb.key.copyString(); + if (local.hasKey(dbname)) { // have database in both + auto const& ldb = local.get(dbname); + for (auto const& pcol : VPackObjectIterator(pdb.value)) { // for each plan collection + auto const& cprops = pcol.value; + for (auto const& shard : VPackObjectIterator(cprops.get(SHARDS))) { // for each shard in plan collection + if (shard.value.isArray()) { + for (auto const& dbs : VPackArrayIterator(shard.value)) { // for each db server with that shard + // We only care for shards, where we find our own ID + if (dbs.isEqualString(serverId)) { + // at this point a shard is in plan, we have the db for it + handlePlanShard( + cprops, ldb, dbname, pcol.key.copyString(), + shard.key.copyString(), serverId, shard.value[0].copyString(), + commonShrds, indis, errors, actions); + break ; + } + } + } // else if(!shard.value.isArray()) - intentionally do nothing + } + } + } + } + + // At this point commonShrds contains all shards that eventually reside on + // this server, are in Plan and their database is present + + // Compare local to plan ----------------------------------------------------- + auto const shardMap = getShardMap(pdbs); // plan shards -> servers + for (auto const& db : VPackObjectIterator(local)) { // for each local databases + auto const& dbname = db.key.copyString(); + if (pdbs.hasKey(dbname)) { // if in plan + for (auto const& sh : VPackObjectIterator(db.value)) { // for each local shard + std::string shName = sh.key.copyString(); + if (shName.front() != '_') { // exclude local system shards/collections + handleLocalShard(dbname, shName, sh.value, shardMap.slice(), commonShrds, + indis, serverId, actions); + } + } + } + } + + + + // See if shard errors can be thrown out: + for (auto& shard : errors.shards) { + std::vector path = split(shard.first, '/'); + path.pop_back(); // Get rid of shard + if (!pdbs.hasKey(path)) { // we can drop the local error + shard.second.reset(); + } + } + + // See if index errors can be thrown out: + for (auto& shard : errors.indexes) { + std::vector path = split(shard.first, '/'); // dbname, collection, shardid + path.pop_back(); // dbname, collection + path.emplace_back(INDEXES); // dbname, collection, indexes + VPackSlice indexes = pdbs.get(path); + if (!indexes.isArray()) { // collection gone, can drop errors + for (auto& index : shard.second) { + index.second.reset(); + } + } else { // need to look at individual errors and indexes: + for (auto& p : shard.second) { + std::string const& id = p.first; + bool found = false; + for (auto const& ind : VPackArrayIterator(indexes)) { + if (ind.get(ID).copyString() == id) { + found = true; + break; + } + } + if (!found) { + p.second.reset(); + } + } + } + } + + return result; + +} + +/// @brief handle plan for local databases +arangodb::Result arangodb::maintenance::executePlan ( + VPackSlice const& plan, VPackSlice const& local, + std::string const& serverId, MaintenanceFeature& feature, + VPackBuilder& report) { + + arangodb::Result result; + + // Errors from maintenance feature + MaintenanceFeature::errors_t errors; + result = feature.copyAllErrors(errors); + if (!result.ok()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << + "phaseOne: failed to acquire copy of errors from maintenance feature."; + return result; + } + + // build difference between plan and local + std::vector actions; + report.add(VPackValue("agency")); + { VPackArrayBuilder a(&report); + diffPlanLocal(plan, local, serverId, errors, actions); } + + for (auto const& i : errors.databases) { + if (i.second == nullptr) { + feature.removeDBError(i.first); + } + } + for (auto const& i : errors.shards) { + if (i.second == nullptr) { + feature.removeShardError(i.first); + } + } + for (auto const& i : errors.indexes) { + std::unordered_set tmp; + for (auto const& index : i.second) { + if (index.second == nullptr) { + tmp.emplace(index.first); + } + } + if (!tmp.empty()) { + feature.removeIndexErrors(i.first, tmp); + } + } + + TRI_ASSERT(report.isOpenObject()); + report.add(VPackValue("actions")); + { VPackArrayBuilder a(&report); + // enact all + for (auto const& action : actions) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "adding action " << action << " to feature "; + { VPackObjectBuilder b(&report); + action.toVelocyPack(report); + } + feature.addAction(std::make_shared(action), false); + } + } + + return result; +} + +/// @brief add new database to current +void addDatabaseToTransactions( + std::string const& name, Transactions& transactions) { + + // [ {"dbPath":{}}, {"dbPath":{"oldEmpty":true}} ] + + std::string dbPath = CURRENT_COLLECTIONS + name; + VPackBuilder operation; // create database in current + { VPackObjectBuilder b(&operation); + operation.add(dbPath, VPackSlice::emptyObjectSlice()); } + VPackBuilder precondition; + { VPackObjectBuilder b(&precondition); + precondition.add(VPackValue(dbPath)); + { VPackObjectBuilder bb(&precondition); + precondition.add("oldEmpty", VPackValue(true)); }} + transactions.push_back({operation, precondition}); + +} + +/// @brief report local to current +arangodb::Result arangodb::maintenance::diffLocalCurrent ( + VPackSlice const& local, VPackSlice const& current, + std::string const& serverId, Transactions& transactions) { + + arangodb::Result result; + auto const& cdbs = current; + + // Iterate over local databases + for (auto const& ldbo : VPackObjectIterator(local)) { + std::string dbname = ldbo.key.copyString(); + //VPackSlice ldb = ldbo.value; + + // Current has this database + if (cdbs.hasKey(dbname)) { + + } else { + // Create new database in current + addDatabaseToTransactions(dbname, transactions); + } + } + + return result; +} + + +/// @brief Phase one: Compare plan and local and create descriptions +arangodb::Result arangodb::maintenance::phaseOne ( + VPackSlice const& plan, VPackSlice const& local, std::string const& serverId, + MaintenanceFeature& feature, VPackBuilder& report) { + + arangodb::Result result; + + report.add(VPackValue("phaseOne")); + { VPackObjectBuilder por(&report); + + // Execute database changes + try { + result = executePlan(plan, local, serverId, feature, report); + } catch (std::exception const& e) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Error executing plan: " << e.what() + << ". " << __FILE__ << ":" << __LINE__; + }} + + report.add(VPackValue("Plan")); + { VPackObjectBuilder p(&report); + report.add("Version", plan.get("Version"));} + + return result; + +} + +static VPackBuilder removeSelectivityEstimate(VPackSlice const& index) { + TRI_ASSERT(index.isObject()); + return arangodb::velocypack::Collection::remove(index, + std::unordered_set({SELECTIVITY_ESTIMATE})); +} + +static VPackBuilder assembleLocalCollectionInfo( + VPackSlice const& info, VPackSlice const& planServers, + std::string const& database, std::string const& shard, + std::string const& ourselves, MaintenanceFeature::errors_t const& allErrors) { + + VPackBuilder ret; + + try { + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto collection = vocbase->lookupCollection(shard); + if (collection == nullptr) { + std::string errorMsg( + "Maintenance::assembleLocalCollectionInfo: Failed to lookup collection "); + errorMsg += shard; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << errorMsg; + { VPackObjectBuilder o(&ret); } + return ret; + } + + std::string errorKey + = database + "/" + std::to_string(collection->planId()) + "/" + shard; + { VPackObjectBuilder r(&ret); + auto it = allErrors.shards.find(errorKey); + if (it == allErrors.shards.end()) { + ret.add(ERROR_STR, VPackValue(false)); + ret.add(ERROR_MESSAGE, VPackValue(std::string())); + ret.add(ERROR_NUM, VPackValue(0)); + } else { + VPackSlice errs(static_cast(it->second->data())); + ret.add(ERROR_STR, errs.get(ERROR_STR)); + ret.add(ERROR_NUM, errs.get(ERROR_NUM)); + ret.add(ERROR_MESSAGE, errs.get(ERROR_MESSAGE)); + } + ret.add(VPackValue(INDEXES)); + { VPackArrayBuilder ixs(&ret); + if (info.get(INDEXES).isArray()) { + auto it1 = allErrors.indexes.find(errorKey); + std::unordered_set indexesDone; + // First the indexes as they are in Local, potentially replaced + // by an error: + for (auto const& index : VPackArrayIterator(info.get(INDEXES))) { + std::string id = index.get(ID).copyString(); + indexesDone.insert(id); + if (it1 != allErrors.indexes.end()) { + auto it2 = it1->second.find(id); + if (it2 != it1->second.end()) { + // Add the error instead: + ret.add(VPackSlice( + static_cast(it2->second->data()))); + continue; + } + } + ret.add(removeSelectivityEstimate(index).slice()); + } + // Now all the errors for this shard, for which there is no index: + if (it1 != allErrors.indexes.end()) { + for (auto const& p : it1->second) { + if (indexesDone.find(p.first) == indexesDone.end()) { + ret.add(VPackSlice( + static_cast(p.second->data()))); + } + } + } + } + } + ret.add(VPackValue(SERVERS)); + { VPackArrayBuilder a(&ret); + ret.add(VPackValue(ourselves)); + auto current = *(collection->followers()->get()); + for (auto const& server : VPackArrayIterator(planServers)) { + if (std::find(current.begin(), current.end(), server.copyString()) + != current.end()) { + ret.add(server); + } + }}} + + return ret; + } catch (std::exception const& e) { + std::string errorMsg( + "Maintenance::assembleLocalCollectionInfo: Failed to lookup database "); + errorMsg += database; + errorMsg += " exception: "; + errorMsg += e.what(); + LOG_TOPIC(WARN, Logger::MAINTENANCE) << errorMsg; + { VPackObjectBuilder o(&ret); } + return ret; + } +} + +bool equivalent(VPackSlice const& local, VPackSlice const& current) { + for (auto const& i : VPackObjectIterator(local)) { + if (!VPackNormalizedCompare::equals(i.value,current.get(i.key.copyString()))) { + return false; + } + } + return true; +} + +static VPackBuilder assembleLocalDatabaseInfo (std::string const& database, + MaintenanceFeature::errors_t const& allErrors) { + // This creates the VelocyPack that is put into + // /Current/Databases// for a database. + + VPackBuilder ret; + + try { + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + { VPackObjectBuilder o(&ret); + auto it = allErrors.databases.find(database); + if (it == allErrors.databases.end()) { + ret.add(ERROR_STR, VPackValue(false)); + ret.add(ERROR_NUM, VPackValue(0)); + ret.add(ERROR_MESSAGE, VPackValue("")); + } else { + VPackSlice errs(static_cast(it->second->data())); + ret.add(ERROR_STR, errs.get(ERROR_STR)); + ret.add(ERROR_NUM, errs.get(ERROR_NUM)); + ret.add(ERROR_MESSAGE, errs.get(ERROR_MESSAGE)); + } + ret.add(ID, VPackValue(std::to_string(vocbase->id()))); + ret.add("name", VPackValue(vocbase->name())); } + + return ret; + } catch(std::exception const& e) { + std::string errorMsg( + "Maintenance::assembleLocalDatabaseInfo: Failed to lookup database "); + errorMsg += database; + errorMsg += " exception: "; + errorMsg += e.what(); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << errorMsg; + { VPackObjectBuilder o(&ret); } + return ret; + } +} + + +// updateCurrentForCollections +// diff current and local and prepare agency transactions or whatever +// to update current. Will report the errors created locally to the agency +arangodb::Result arangodb::maintenance::reportInCurrent( + VPackSlice const& plan, VPackSlice const& cur, VPackSlice const& local, + MaintenanceFeature::errors_t const& allErrors, + std::string const& serverId, VPackBuilder& report) { + arangodb::Result result; + + auto shardMap = getShardMap(plan.get(COLLECTIONS)); + auto pdbs = plan.get(COLLECTIONS); + + for (auto const& database : VPackObjectIterator(local)) { + auto const dbName = database.key.copyString(); + + std::vector const cdbpath {"Databases", dbName, serverId}; + + if (!cur.hasKey(cdbpath)) { + auto const localDatabaseInfo = assembleLocalDatabaseInfo(dbName, allErrors); + if (!localDatabaseInfo.slice().isEmptyObject()) { + report.add(VPackValue(CURRENT_DATABASES + dbName + "/" + serverId)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_SET); + report.add("payload", localDatabaseInfo.slice()); } + } + } + + + for (auto const& shard : VPackObjectIterator(database.value)) { + + auto const shName = shard.key.copyString(); + if (shName.at(0) == '_') { // local system collection + continue; + } + auto const shSlice = shard.value; + auto const colName = shSlice.get(PLAN_ID).copyString(); + + VPackBuilder error; + if (shSlice.get(LEADER).copyString().empty()) { // Leader + + auto const localCollectionInfo = + assembleLocalCollectionInfo( + shSlice, shardMap.slice().get(shName), dbName, shName, serverId, + allErrors); + // Collection no longer exists + if (localCollectionInfo.slice().isEmptyObject()) { + continue; + } + + auto cp = std::vector {COLLECTIONS, dbName, colName, shName}; + + + auto inCurrent = cur.hasKey(cp); + if (!inCurrent || + (inCurrent && + !equivalent(localCollectionInfo.slice(), cur.get(cp)))) { + + + report.add( + VPackValue(CURRENT_COLLECTIONS+dbName+"/"+colName+"/"+shName)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_SET); + report.add("payload", localCollectionInfo.slice()); } + } + } else { // Follower + + auto servers = std::vector + {COLLECTIONS, dbName, colName, shName, SERVERS}; + if (cur.hasKey(servers)) { + auto s = cur.get(servers); + if (s.isArray() && cur.get(servers)[0].copyString() == serverId) { + // we were previously leader and we are done resigning. + // update current and let supervision handle the rest + VPackBuilder ns; + { VPackArrayBuilder a(&ns); + bool front = true; + if (s.isArray()) { + for (auto const& i : VPackArrayIterator(s)) { + ns.add(VPackValue((!front) ? i.copyString() : UNDERSCORE + i.copyString())); + front = false; + }}} + report.add( + VPackValue( + CURRENT_COLLECTIONS + dbName + "/" + colName + "/" + shName + + "/" + SERVERS)); + + { VPackObjectBuilder o(&report); + report.add(OP, VP_SET); + report.add("payload", ns.slice()); } + } + } + } + } + } + + // UpdateCurrentForDatabases + auto cdbs = cur.get(DATABASES); + for (auto const& database : VPackObjectIterator(cdbs)) { + auto const dbName = database.key.copyString(); + if (!database.value.isObject()) { + continue; + } + VPackSlice myEntry = database.value.get(serverId); + if (!myEntry.isNone()) { + + // Database no longer in Plan and local + if (!local.hasKey(dbName) && !pdbs.hasKey(dbName)) { + // This covers the case that the database is neither in Local nor in Plan. + // It remains to make sure an error is reported to Current if there is + // a database in the Plan but not in Local + report.add(VPackValue(CURRENT_DATABASES + dbName + "/" + serverId)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_DELETE); } + // We delete all under /Current/Collections/, it does not + // hurt if every DBserver does this, since it is an idempotent + // operation. + report.add(VPackValue(CURRENT_COLLECTIONS + dbName)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_DELETE); } + } + } + } + + // UpdateCurrentForCollections + auto curcolls = cur.get(COLLECTIONS); + for (auto const& database : VPackObjectIterator(curcolls)) { + auto const dbName = database.key.copyString(); + + // UpdateCurrentForCollections (Current/Collections/Collection) + for (auto const& collection : VPackObjectIterator(database.value)) { + auto const colName = collection.key.copyString(); + + for (auto const& shard : VPackObjectIterator(collection.value)) { + auto const shName = shard.key.copyString(); + + // Shard in current and has servers + if (shard.value.hasKey(SERVERS)) { + auto servers = shard.value.get(SERVERS); + + if (servers.isArray() && servers.length() > 0 // servers in current + && servers[0].copyString() == serverId // we are leading + && !local.hasKey( + std::vector {dbName, shName}) // no local collection + && !shardMap.slice().hasKey(shName)) { // no such shard in plan + report.add( + VPackValue( + CURRENT_COLLECTIONS + dbName + "/" + colName + "/" + shName)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_DELETE); } + } + + } + } + } + } + + // Let's find database errors for databases which do not occur in Local + // but in Plan: + VPackSlice planDatabases = plan.get("Databases"); + VPackSlice curDatabases = cur.get("Databases"); + if (planDatabases.isObject() && curDatabases.isObject()) { + for (auto const& p : allErrors.databases) { + VPackSlice planDbEntry = planDatabases.get(p.first); + VPackSlice curDbEntry = curDatabases.get(p.first); + if (planDbEntry.isObject() && curDbEntry.isNone()) { + // Need to create an error entry: + report.add(VPackValue(CURRENT_DATABASES + p.first + "/" + serverId)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_SET); + report.add(VPackSlice("payload")); + { VPackObjectBuilder pp(&report); + VPackSlice errs(static_cast(p.second->data())); + report.add(ERROR_STR, errs.get(ERROR_STR)); + report.add(ERROR_NUM, errs.get(ERROR_NUM)); + report.add(ERROR_MESSAGE, errs.get(ERROR_MESSAGE)); + } + } + } + } + } + + // Finally, let's find shard errors for shards which do not occur in + // Local but in Plan, we need to make sure that these errors are reported + // in Current: + for (auto const& p : allErrors.shards) { + // First split the key: + std::string const& key = p.first; + auto pos = key.find('/'); + TRI_ASSERT(pos != std::string::npos); + std::string d = key.substr(0, pos); // database + auto pos2 = key.find('/', pos + 1); // collection + TRI_ASSERT(pos2 != std::string::npos); + std::string c = key.substr(pos + 1, pos2); + std::string s = key.substr(pos2 + 1); // shard name + + // Now find out if the shard appears in the Plan but not in Local: + VPackSlice inPlan = pdbs.get(std::vector({d, c, "shards", s})); + VPackSlice inLoc = local.get(std::vector({d, s})); + if (inPlan.isObject() && inLoc.isNone()) { + VPackSlice inCur = curcolls.get(std::vector({d, c, s})); + VPackSlice theErr(static_cast(p.second->data())); + if (inCur.isNone() || !equivalent(theErr, inCur)) { + report.add( + VPackValue(CURRENT_COLLECTIONS + d + "/" + c + "/" + s)); + { VPackObjectBuilder o(&report); + report.add(OP, VP_SET); + report.add("payload", theErr); } + } + } + } + + return result; + +} + + +arangodb::Result arangodb::maintenance::syncReplicatedShardsWithLeaders( + VPackSlice const& plan, VPackSlice const& current, VPackSlice const& local, + std::string const& serverId, std::vector& actions) { + + auto pdbs = plan.get(COLLECTIONS); + auto cdbs = current.get(COLLECTIONS); + + for (auto const& pdb : VPackObjectIterator(pdbs)) { + auto const& dbname = pdb.key.copyString(); + if (local.hasKey(dbname) && cdbs.hasKey(dbname)) { + for (auto const& pcol : VPackObjectIterator(pdb.value)) { + auto const& colname = pcol.key.copyString(); + if (cdbs.get(dbname).hasKey(colname)) { + for (auto const& pshrd : VPackObjectIterator(pcol.value.get(SHARDS))) { + auto const& shname = pshrd.key.copyString(); + + // shard does not exist locally so nothing we can do at this point + if (!local.hasKey(std::vector{dbname, shname})) { + continue; + } + + // current stuff is created by the leader this one here will just + // bring followers in sync so just continue here + auto cpath = std::vector {dbname, colname, shname}; + if (!cdbs.hasKey(cpath)) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << + "Shard " << shname << " not in current yet. Rescheduling maintenance."; + continue; + } + + // Plan's servers + auto ppath = std::vector {dbname, colname, SHARDS, shname}; + if (!pdbs.hasKey(ppath)) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Shard " << shname + << " does not have servers substructure in 'Plan'"; + continue; + } + auto const& pservers = pdbs.get(ppath); + + // Current's servers + cpath.push_back(SERVERS); + if (!cdbs.hasKey(cpath)) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Shard " << shname + << " does not have servers substructure in 'Current'"; + continue; + } + auto const& cservers = cdbs.get(cpath); + + // we are not planned to be a follower + if (indexOf(pservers, serverId) <= 0) { + continue; + } + // if we are considered to be in sync there is nothing to do + if (indexOf(cservers, serverId) > 0) { + continue; + } + + auto const leader = pservers[0].copyString(); + actions.emplace_back( + ActionDescription( + {{NAME, "SynchronizeShard"}, {DATABASE, dbname}, + {COLLECTION, colname}, {SHARD, shname}, {LEADER, leader}})); + + } + } + } + } + } + + return Result(); + +} + + + +/// @brief Phase two: See, what we can report to the agency +arangodb::Result arangodb::maintenance::phaseTwo ( + VPackSlice const& plan, VPackSlice const& cur, VPackSlice const& local, + std::string const& serverId, MaintenanceFeature& feature, VPackBuilder& report) { + + MaintenanceFeature::errors_t allErrors; + feature.copyAllErrors(allErrors); + + arangodb::Result result; + + report.add(VPackValue("phaseTwo")); + { VPackObjectBuilder p2(&report); + + // agency transactions + report.add(VPackValue("agency")); + { VPackObjectBuilder agency(&report); + // Update Current + try { + result = reportInCurrent(plan, cur, local, allErrors, serverId, report); + } catch (std::exception const& e) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Error reporting in current: " << e.what() << ". " + << __FILE__ << ":" << __LINE__; + }} + + // maintenace actions + report.add(VPackValue("actions")); + { VPackObjectBuilder agency(&report); + try { + std::vector actions; + result = syncReplicatedShardsWithLeaders( + plan, cur, local, serverId, actions); + + for (auto const& action : actions) { + feature.addAction(std::make_shared(action), false); + } + } catch (std::exception const& e) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Error scheduling shards: " << e.what() << ". " + << __FILE__ << ":" << __LINE__; + }} + + } + + report.add(VPackValue("Current")); + { VPackObjectBuilder p(&report); + report.add("Version", cur.get("Version")); } + + return result; + +} + diff --git a/arangod/Cluster/Maintenance.h b/arangod/Cluster/Maintenance.h new file mode 100644 index 0000000000..f6b8ddec24 --- /dev/null +++ b/arangod/Cluster/Maintenance.h @@ -0,0 +1,161 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_MAINTENANCE_H +#define ARANGODB_MAINTENANCE_MAINTENANCE_H + +#include "Basics/Result.h" +#include "Basics/VelocyPackHelper.h" +#include "Agency/Node.h" +#include "Cluster/MaintenanceFeature.h" + +namespace arangodb { + +class LogicalCollection; + +namespace maintenance { + +using Transactions = std::vector>; + +arangodb::Result diffPlanLocalForDatabases( + VPackSlice const&, std::vector const&, + std::vector&, std::vector&); + + +/** + * @brief Difference Plan and local for phase 1 of Maintenance run + * + * @param plan Snapshot of agency's planned state + * @param local Snapshot of local state + * @param serverId This server's UUID + * @param errors Copy of last maintenance feature errors + * @param actions Resulting actions from difference are packed in here + * + * @return Result + */ +arangodb::Result diffPlanLocal( + VPackSlice const& plan, VPackSlice const& local, std::string const& serverId, + MaintenanceFeature::errors_t& errors, std::vector& actions); + +/** + * @brief Difference Plan and local for phase 1 of Maintenance run + * + * @param plan Snapshot of agency's planned state + * @param current Snapshot of agency's current state + * @param local Snapshot of local state + * @param serverId This server's UUID + * @param feature Maintenance feature + * @param report Report + * + * @return Result + */ +arangodb::Result executePlan ( + VPackSlice const& plan, VPackSlice const& local, std::string const& serverId, + arangodb::MaintenanceFeature& feature, VPackBuilder& report); + +/** + * @brief Difference local and current states for phase 2 of Maintenance + * + * @param local Snapshot of local state + * @param current Snapshot of agency's current state + * @param serverId This server's UUID + * @param report Resulting agency transaction, which is to be sent + * + * @return Result + */ +arangodb::Result diffLocalCurrent ( + VPackSlice const& local, VPackSlice const& current, + std::string const& serverId, Transactions& report); + + +/** + * @brief Phase one: Execute plan, shard replication startups + * + * @param plan Snapshot of agency's planned state + * @param current Snapshot of agency's current state + * @param local Snapshot of local state + * @param serverId This server's UUID + * @param feature Maintenance feature + * @param report Resulting agency transaction, which is to be sent + * + * @return Result + */ +arangodb::Result phaseOne ( + VPackSlice const& plan, VPackSlice const& local, std::string const& serverId, + MaintenanceFeature& feature, VPackBuilder& report); + +/** + * @brief Phase two: Report in agency + * + * @param plan Snapshot of agency's planned state + * @param current Snapshot of agency's current state + * @param local Snapshot of local state + * @param serverId This server's UUID + * @param feature Maintenance feature + * @param report Report on what we did + * + * @return Result + */ +arangodb::Result phaseTwo ( + VPackSlice const& plan, VPackSlice const& cur, VPackSlice const& local, + std::string const& serverId, MaintenanceFeature& feature, VPackBuilder& report); + + +/** + * @brief Report local changes to current + * + * @param plan Snapshot of agency's planned state + * @param current Snapshot of agency's current state + * @param local Snapshot of local state + * @param serverId This server's UUID + * @param report Report on what we did + * + * @return Result + */ +arangodb::Result reportInCurrent( + VPackSlice const& plan, VPackSlice const& cur, VPackSlice const& local, + MaintenanceFeature::errors_t const& allErrors, + std::string const& serverId, VPackBuilder& report); + + +/** + * @brief Schedule synchroneous replications + * + * @param plan Plan's snapshot + * @param current Current's scnapshot + * @param local Local snapshot + * @param serverId My server's uuid + * @param actions Resulting actions + * @param rescheduleForSync Report to DBServerAgencySync, that we need to rerun + * + * @return Success story + */ +arangodb::Result syncReplicatedShardsWithLeaders( + VPackSlice const& plan, VPackSlice const& current, VPackSlice const& local, + std::string const& serverId, std::vector& actions); + +}} + +#endif + diff --git a/arangod/Cluster/MaintenanceFeature.cpp b/arangod/Cluster/MaintenanceFeature.cpp new file mode 100644 index 0000000000..0fcfd7be56 --- /dev/null +++ b/arangod/Cluster/MaintenanceFeature.cpp @@ -0,0 +1,681 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "MaintenanceFeature.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/ConditionLocker.h" +#include "Basics/ReadLocker.h" +#include "Basics/WriteLocker.h" +#include "Basics/MutexLocker.h" +#include "Cluster/ActionDescription.h" +#include "Cluster/CreateDatabase.h" +#include "Cluster/Action.h" +#include "Cluster/MaintenanceWorker.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::options; +using namespace arangodb::maintenance; + +MaintenanceFeature::MaintenanceFeature(application_features::ApplicationServer& server) + : ApplicationFeature(server, "Maintenance") { + +// startsAfter("EngineSelector"); // ??? what should this be +// startsBefore("StorageEngine"); + + init(); + +} // MaintenanceFeature::MaintenanceFeature + + + + +void MaintenanceFeature::init() { + _isShuttingDown=false; + _nextActionId=1; + + setOptional(true); + requiresElevatedPrivileges(false); // ??? this mean admin priv? + + // these parameters might be updated by config and/or command line options + _maintenanceThreadsMax = static_cast(TRI_numberProcessors()/4 +1); + _secondsActionsBlock = 2; + _secondsActionsLinger = 3600; + + return; + +} // MaintenanceFeature::init + + +void MaintenanceFeature::collectOptions(std::shared_ptr options) { + options->addSection("server", "Server features"); + + options->addHiddenOption( + "--server.maintenance-threads", + "maximum number of threads available for maintenance actions", + new Int32Parameter(&_maintenanceThreadsMax)); + + options->addHiddenOption( + "--server.maintenance-actions-block", + "minimum number of seconds finished Actions block duplicates", + new Int32Parameter(&_secondsActionsBlock)); + + options->addHiddenOption( + "--server.maintenance-actions-linger", + "minimum number of seconds finished Actions remain in deque", + new Int32Parameter(&_secondsActionsLinger)); + +} // MaintenanceFeature::collectOptions + + +/// do not start threads in prepare +void MaintenanceFeature::prepare() { +} // MaintenanceFeature::prepare + + +void MaintenanceFeature::start() { + int loop; + bool flag; + + // start threads + for (loop=0; loop<_maintenanceThreadsMax; ++loop) { + maintenance::MaintenanceWorker * newWorker = new maintenance::MaintenanceWorker(*this); + _activeWorkers.push_back(newWorker); + flag = newWorker->start(&_workerCompletion); + + if (!flag) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "MaintenanceFeature::start: newWorker start failed"; + } // if + } // for +} // MaintenanceFeature::start + + +void MaintenanceFeature::beginShutdown() { + + _isShuttingDown=true; + CONDITION_LOCKER(cLock, _actionRegistryCond); + _actionRegistryCond.broadcast(); + + return; + +} // MaintenanceFeature + + +void MaintenanceFeature::stop() { + + for (auto itWorker : _activeWorkers ) { + CONDITION_LOCKER(cLock, _workerCompletion); + + // loop on each worker, retesting at 10ms just in case + if (itWorker->isRunning()) { + _workerCompletion.wait(10000); + } // if + } // for + + return; + +} // MaintenanceFeature::stop + + +/// @brief Move an incomplete action to failed state +Result MaintenanceFeature::deleteAction(uint64_t action_id) { + Result result; + + // pointer to action, or nullptr + auto action = findActionId(action_id); + + if (action) { + if (maintenance::COMPLETE != action->getState()) { + action->setState(maintenance::FAILED); + } else { + result.reset(TRI_ERROR_BAD_PARAMETER,"deleteAction called after action complete."); + } // else + } else { + result.reset(TRI_ERROR_BAD_PARAMETER,"deleteAction could not find action to delete."); + } // else + + return result; + +} // MaintenanceFeature::deleteAction + + +// FIXMEMAINTENANCE: None of the addAction() and createAction() routines +// explicitly check to see if construction of action set FAILED. +// Therefore it is possible for an "executeNow" action to start running +// with known invalid parameters. + +/// @brief This is the API for creating an Action and executing it. +/// Execution can be immediate by calling thread, or asynchronous via thread pool. +/// not yet: ActionDescription parameter will be MOVED to new object. +Result MaintenanceFeature::addAction( + std::shared_ptr newAction, bool executeNow) { + + Result result; + + // the underlying routines are believed to be safe and throw free, + // but just in case + try { + + size_t action_hash = newAction->hash(); + WRITE_LOCKER(wLock, _actionRegistryLock); + + std::shared_ptr curAction = findActionHashNoLock(action_hash); + + // similar action not in the queue (or at least no longer viable) + if (curAction == nullptr || curAction->done()) { + + createAction(newAction, executeNow); + + if (!newAction || !newAction->ok()) { + /// something failed in action creation ... go check logs + result.reset(TRI_ERROR_BAD_PARAMETER, "createAction rejected parameters."); + } // if + } else { + // action already exist, need write lock to prevent race + result.reset(TRI_ERROR_BAD_PARAMETER, "addAction called while similar action already processing."); + } //else + + // executeNow process on this thread, right now! + if (result.ok() && executeNow) { + maintenance::MaintenanceWorker worker(*this, newAction); + worker.run(); + result = worker.result(); + } // if + } catch (...) { + result.reset(TRI_ERROR_INTERNAL, "addAction experience an unexpected throw."); + } // catch + + return result; + +} // MaintenanceFeature::addAction + + + +/// @brief This is the API for creating an Action and executing it. +/// Execution can be immediate by calling thread, or asynchronous via thread pool. +/// not yet: ActionDescription parameter will be MOVED to new object. +Result MaintenanceFeature::addAction( + std::shared_ptr const & description, + bool executeNow) { + + Result result; + + // the underlying routines are believed to be safe and throw free, + // but just in case + try { + std::shared_ptr newAction; + + // is there a known name field + auto find_it = description->get("name"); + + size_t action_hash = description->hash(); + WRITE_LOCKER(wLock, _actionRegistryLock); + + std::shared_ptr curAction = findActionHashNoLock(action_hash); + + // similar action not in the queue (or at least no longer viable) + if (!curAction || curAction->done()) { + newAction = createAction(description, executeNow); + + if (!newAction || !newAction->ok()) { + /// something failed in action creation ... go check logs + result.reset(TRI_ERROR_BAD_PARAMETER, "createAction rejected parameters."); + } // if + } else { + // action already exist, need write lock to prevent race + result.reset(TRI_ERROR_BAD_PARAMETER, "addAction called while similar action already processing."); + } //else + + // executeNow process on this thread, right now! + if (result.ok() && executeNow) { + maintenance::MaintenanceWorker worker(*this, newAction); + worker.run(); + result = worker.result(); + } // if + } catch (...) { + result.reset(TRI_ERROR_INTERNAL, "addAction experience an unexpected throw."); + } // catch + + return result; + +} // MaintenanceFeature::addAction + + +std::shared_ptr MaintenanceFeature::preAction( + std::shared_ptr const & description) { + + return createAction(description, true); + +} // MaintenanceFeature::preAction + + +std::shared_ptr MaintenanceFeature::postAction( + std::shared_ptr const & description) { + + return createAction(description, false); + +} // MaintenanceFeature::postAction + + +void MaintenanceFeature::createAction( + std::shared_ptr action, bool executeNow) { + + // mark as executing so no other workers accidentally grab it + if (executeNow) { + action->setState(maintenance::EXECUTING); + } // if + + // WARNING: holding write lock to _actionRegistry and about to + // lock condition variable + { + _actionRegistry.push_back(action); + + if (!executeNow) { + CONDITION_LOCKER(cLock, _actionRegistryCond); + _actionRegistryCond.signal(); + } // if + } // lock + +} + + +std::shared_ptr MaintenanceFeature::createAction( + std::shared_ptr const & description, + bool executeNow) { + + // write lock via _actionRegistryLock is assumed held + std::shared_ptr newAction; + + // name should already be verified as existing ... but trust no one + std::string name = description->get(NAME); + + // call factory + newAction = std::make_shared(*this, *description); + + // if a new action constructed successfully + if (newAction->ok()) { + + createAction(newAction, executeNow); + + } else { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "createAction: unknown action name given, \"" << name.c_str() << "\", or other construction failure."; + } // else + + return newAction; + +} // if + + +std::shared_ptr MaintenanceFeature::findAction( + std::shared_ptr const description) { + return findActionHash(description->hash()); +} + + +std::shared_ptr MaintenanceFeature::findActionHash(size_t hash) { + READ_LOCKER(rLock, _actionRegistryLock); + + return(findActionHashNoLock(hash)); + +} // MaintenanceFeature::findActionHash + + +std::shared_ptr MaintenanceFeature::findActionHashNoLock(size_t hash) { + // assert to test lock held? + + std::shared_ptr ret_ptr; + + for (auto action_it=_actionRegistry.begin(); + _actionRegistry.end() != action_it && !ret_ptr; ++action_it) { + if ((*action_it)->hash() == hash) { + ret_ptr=*action_it; + } // if + } // for + + return ret_ptr; + +} // MaintenanceFeature::findActionHashNoLock + + +std::shared_ptr MaintenanceFeature::findActionId(uint64_t id) { + READ_LOCKER(rLock, _actionRegistryLock); + + return(findActionIdNoLock(id)); + +} // MaintenanceFeature::findActionId + + +std::shared_ptr MaintenanceFeature::findActionIdNoLock(uint64_t id) { + // assert to test lock held? + + std::shared_ptr ret_ptr; + + for (auto action_it=_actionRegistry.begin(); + _actionRegistry.end() != action_it && !ret_ptr; ++action_it) { + if ((*action_it)->id() == id) { + ret_ptr=*action_it; + } // if + } // for + + return ret_ptr; + +} // MaintenanceFeature::findActionIdNoLock + + +std::shared_ptr MaintenanceFeature::findReadyAction() { + std::shared_ptr ret_ptr; + + while(!_isShuttingDown && !ret_ptr) { + + // scan for ready action (and purge any that are done waiting) + { + WRITE_LOCKER(wLock, _actionRegistryLock); + + for (auto loop=_actionRegistry.begin(); _actionRegistry.end()!=loop && !ret_ptr; ) { + auto state = (*loop)->getState(); + if (state == maintenance::READY) { + ret_ptr=*loop; + ret_ptr->setState(maintenance::EXECUTING); + } else if ((*loop)->done()) { + loop = _actionRegistry.erase(loop); + } else { + ++loop; + } // else + } // for + } // WRITE + + // no pointer ... wait 5 second + if (!_isShuttingDown && !ret_ptr) { + CONDITION_LOCKER(cLock, _actionRegistryCond); + _actionRegistryCond.wait(100000); + } // if + + } // while + + return ret_ptr; + +} // MaintenanceFeature::findReadyAction + + +VPackBuilder MaintenanceFeature::toVelocyPack() const { + VPackBuilder vb; + READ_LOCKER(rLock, _actionRegistryLock); + + { VPackArrayBuilder ab(&vb); + for (auto const& action : _actionRegistry ) { + action->toVelocyPack(vb); + } // for + + } + return vb; + +} // MaintenanceFeature::toVelocyPack +#if 0 +std::string MaintenanceFeature::toJson(VPackBuilder & builder) { +} // MaintenanceFeature::toJson +#endif + +std::string const SLASH("/"); + +arangodb::Result MaintenanceFeature::storeDBError ( + std::string const& database, std::shared_ptr> error) { + + MUTEX_LOCKER(guard, _dbeLock); + auto const it = _dbErrors.find(database); + if (it != _dbErrors.end()) { + std::stringstream error; + error << "database " << database << " already has pending error"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + try { + _dbErrors.emplace(database,error); + } catch (std::exception const& e) { + return Result(TRI_ERROR_FAILED, e.what()); + } + + return Result(); + +} + +arangodb::Result MaintenanceFeature::dbError ( + std::string const& database, std::shared_ptr>& error) const { + + MUTEX_LOCKER(guard, _dbeLock); + auto const it = _dbErrors.find(database); + error = (it != _dbErrors.end()) ? it->second : nullptr; + return Result(); + +} + +arangodb::Result MaintenanceFeature::removeDBError ( + std::string const& database) { + + try { + MUTEX_LOCKER(guard, _seLock); + _shardErrors.erase(database); + } catch (std::exception const& e) { + std::stringstream error; + error << "erasing dataabse error for " << database << " failed"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + return Result(); + +} + +arangodb::Result MaintenanceFeature::storeShardError ( + std::string const& database, std::string const& collection, + std::string const& shard, std::shared_ptr> error) { + + std::string key = database + SLASH + collection + SLASH + shard; + + MUTEX_LOCKER(guard, _seLock); + auto const it = _shardErrors.find(key); + if (it != _shardErrors.end()) { + std::stringstream error; + error << "shard " << key << " already has pending error"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + try { + _shardErrors.emplace(key,error); + } catch (std::exception const& e) { + return Result(TRI_ERROR_FAILED, e.what()); + } + + return Result(); + +} + +arangodb::Result MaintenanceFeature::shardError ( + std::string const& database, std::string const& collection, + std::string const& shard, std::shared_ptr>& error) const { + + std::string key = database + SLASH + collection + SLASH + shard; + + MUTEX_LOCKER(guard, _seLock); + auto const it = _shardErrors.find(key); + error = (it != _shardErrors.end()) ? it->second : nullptr; + return Result(); + +} + +arangodb::Result MaintenanceFeature::removeShardError (std::string const& key) { + + try { + MUTEX_LOCKER(guard, _seLock); + _shardErrors.erase(key); + } catch (std::exception const& e) { + std::stringstream error; + error << "erasing shard error for " << key << " failed"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + return Result(); + +} + +arangodb::Result MaintenanceFeature::removeShardError ( + std::string const& database, std::string const& collection, + std::string const& shard) { + return removeShardError(database + SLASH + collection + SLASH + shard); +} + + +arangodb::Result MaintenanceFeature::storeIndexError ( + std::string const& database, std::string const& collection, + std::string const& shard, std::string const& indexId, + std::shared_ptr> error) { + + using buffer_t = std::shared_ptr>; + std::string key = database + SLASH + collection + SLASH + shard; + + MUTEX_LOCKER(guard, _ieLock); + + auto errorsIt = _indexErrors.find(key); + if (errorsIt == _indexErrors.end()) { + try { + _indexErrors.emplace(key,std::map()); + } catch (std::exception const& e) { + return Result(TRI_ERROR_FAILED, e.what()); + } + } + auto& errors = _indexErrors.find(key)->second; + auto const it = errors.find(indexId); + + if (it != errors.end()) { + std::stringstream error; + error << "index " << indexId << " for shard " + << key << " already has pending error"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + try { + errors.emplace(indexId,error); + } catch (std::exception const& e) { + return Result(TRI_ERROR_FAILED, e.what()); + } + + return Result(); + +} + +arangodb::Result MaintenanceFeature::indexErrors ( + std::string const& database, std::string const& collection, + std::string const& shard, + std::map>>& error) const { + + std::string key = database + SLASH + collection + SLASH + shard; + + MUTEX_LOCKER(guard, _ieLock); + auto const& it = _indexErrors.find(key); + if (it != _indexErrors.end()) { + error = it->second; + } + + return Result(); + +} + +template +std::ostream& operator<<(std::ostream& os, std::setconst& st) { + size_t j = 0; + os << "["; + for (auto const& i : st) { + os << i; + if (++j < st.size()) { + os << ", "; + } + } + os << "]"; + return os; +} + + +arangodb::Result MaintenanceFeature::removeIndexErrors ( + std::string const& key, std::unordered_set indexIds) { + + MUTEX_LOCKER(guard, _ieLock); + + // If no entry for this shard exists bail out + auto kit = _indexErrors.find(key); + if (kit == _indexErrors.end()) { + std::stringstream error; + error << "erasing index " << indexIds << " error for shard " << key + << " failed as no such key is found in index error bucket"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + auto& errors = kit->second; + + try { + for (auto const& indexId : indexIds) { + errors.erase(indexId); + } + } catch (std::exception const& e) { + std::stringstream error; + error << "erasing index errors " << indexIds << " for " << key << " failed"; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << error.str(); + return Result(TRI_ERROR_FAILED, error.str()); + } + + return Result(); + +} + +arangodb::Result MaintenanceFeature::removeIndexErrors ( + std::string const& database, std::string const& collection, + std::string const& shard, std::unordered_set indexIds) { + + return removeIndexErrors( + database + SLASH + collection + SLASH + shard, indexIds); + +} + +arangodb::Result MaintenanceFeature::copyAllErrors(errors_t& errors) const { + { + MUTEX_LOCKER(guard, _seLock); + errors.shards = _shardErrors; + } + { + MUTEX_LOCKER(guard, _ieLock); + errors.indexes = _indexErrors; + } + { + MUTEX_LOCKER(guard, _dbeLock); + errors.databases = _dbErrors; + } + return Result(); +} + + diff --git a/arangod/Cluster/MaintenanceFeature.h b/arangod/Cluster/MaintenanceFeature.h new file mode 100644 index 0000000000..ee1694d8f3 --- /dev/null +++ b/arangod/Cluster/MaintenanceFeature.h @@ -0,0 +1,369 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2017 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_CLUSTER_MAINTENANCE_FEATURE +#define ARANGOD_CLUSTER_MAINTENANCE_FEATURE 1 + + +#include "ApplicationFeatures/ApplicationFeature.h" +#include "Basics/Common.h" +#include "Basics/ReadWriteLock.h" +#include "Basics/Result.h" +#include "Cluster/Action.h" +#include "Cluster/MaintenanceWorker.h" +#include "ProgramOptions/ProgramOptions.h" + +namespace arangodb { + +class MaintenanceFeature : public application_features::ApplicationFeature { + public: + explicit MaintenanceFeature(application_features::ApplicationServer&); + + MaintenanceFeature(); + + virtual ~MaintenanceFeature() {}; + + struct errors_t { + std::map>>> indexes; + + // dbname/collection/shardid -> error + std::unordered_map>> shards; + + // dbname -> error + std::unordered_map>> databases; + }; + + public: + void collectOptions(std::shared_ptr) override; + + // preparation phase for feature in the preparation phase, the features must + // not start any threads. furthermore, they must not write any files under + // elevated privileges if they want other features to access them, or if they + // want to access these files with dropped privileges + virtual void prepare() override; + + // start the feature + virtual void start() override; + + // notify the feature about a shutdown request + virtual void beginShutdown() override; + + // stop the feature + virtual void stop() override; + + // shut down the feature + virtual void unprepare() override {}; + + + // + // api features + // + + /// @brief This is the API for creating an Action and executing it. + /// Execution can be immediate by calling thread, or asynchronous via thread pool. + /// not yet: ActionDescription parameter will be MOVED to new object. + virtual Result addAction( + std::shared_ptr const & description, + bool executeNow=false); + + /// @brief This is the API for creating an Action and executing it. + /// Execution can be immediate by calling thread, or asynchronous via thread pool. + /// not yet: ActionDescription parameter will be MOVED to new object. + virtual Result addAction( + std::shared_ptr action, bool executeNow=false); + + /// @brief Internal API that allows existing actions to create pre actions + /// FIXDOC: Please explain how this works in a lot more detail, for example, + /// say if this can be called in the code of an Action and if the already + /// running action is postponed in this case. Explain semantics such that + /// somebody not knowing the code can use it. + std::shared_ptr preAction( + std::shared_ptr const & description); + + /// @brief Internal API that allows existing actions to create post actions + /// FIXDOC: Please explain how this works in a lot more detail, such that + /// somebody not knowing the code can use it. + std::shared_ptr postAction( + std::shared_ptr const & description); + +protected: + std::shared_ptr createAction( + std::shared_ptr const & description, + bool executeNow); + + void createAction( + std::shared_ptr action, bool executeNow); + +public: + /// @brief This API will attempt to fail an existing Action that is waiting + /// or executing. Will not fail Actions that have already succeeded or failed. + Result deleteAction(uint64_t id); + + /// @brief Create a VPackBuilder object with snapshot of current action registry + VPackBuilder toVelocyPack() const; + + /// @brief Returns json array of all MaintenanceActions within the deque + Result toJson(VPackBuilder & builder); + + /// @brief Return pointer to next ready action, or nullptr + std::shared_ptr findReadyAction(); + + /// @brief Process specific ID for a new action + /// @returns uint64_t + uint64_t nextActionId() {return _nextActionId++;}; + + bool isShuttingDown() const {return(_isShuttingDown);}; + + /// @brief Return number of seconds to say "not done" to block retries too soon + uint32_t getSecondsActionsBlock() const {return _secondsActionsBlock;}; + + /** + * @brief Find and return found action or nullptr + * @param desc Description of sought action + */ + std::shared_ptr findAction( + std::shared_ptr const desc); + + /** + * @brief add index error to bucket + * Errors are added by EnsureIndex + * + * @param database database + * @param collection collection + * @param shard shard + * @param indexId index' id + * + * @return success + */ + arangodb::Result storeIndexError ( + std::string const& database, std::string const& collection, + std::string const& shard, std::string const& indexId, + std::shared_ptr> error); + + /** + * @brief get all pending index errors for a specific shard + * + * @param database database + * @param collection collection + * @param shard shard + * @param errors errrors map returned to caller + * + * @return success + */ + arangodb::Result indexErrors( + std::string const& database, std::string const& collection, + std::string const& shard, + std::map>>& errors) const ; + + /** + * @brief remove 1+ errors from index error bucket + * Errors are removed by phaseOne, as soon as indexes no longer in plan + * + * @param database database + * @param collection collection + * @param shard shard + * @param indexId index' id + * + * @return success + */ + arangodb::Result removeIndexErrors ( + std::string const& database, std::string const& collection, + std::string const& shard, std::unordered_set indexIds); + arangodb::Result removeIndexErrors ( + std::string const& path, std::unordered_set indexIds); + + /** + * @brief add shard error to bucket + * Errors are added by CreateCollection, UpdateCollection + * + * @param database database + * @param collection collection + * @param shard shard + * + * @return success + */ + arangodb::Result storeShardError ( + std::string const& database, std::string const& collection, + std::string const& shard, std::shared_ptr> error); + + /** + * @brief get all pending shard errors + * + * @param database database + * @param collection collection + * @param shard shard + * + * @return success + */ + arangodb::Result shardError( + std::string const& database, std::string const& collection, + std::string const& shard, std::shared_ptr>& error) const; + + /** + * @brief remove error from shard bucket + * Errors are removed by phaseOne, as soon as indexes no longer in plan + * + * @param database database + * @param collection collection + * @param shard shard + * + * @return success + */ + arangodb::Result removeShardError ( + std::string const& database, std::string const& collection, + std::string const& shard); + arangodb::Result removeShardError (std::string const& key); + + /** + * @brief add shard error to bucket + * Errors are added by CreateCollection, UpdateCollection + * + * @param database database + * + * @return success + */ + arangodb::Result storeDBError ( + std::string const& database, std::shared_ptr> error); + + /** + * @brief get all pending shard errors + * + * @param database database + * + * @return success + */ + arangodb::Result dbError( + std::string const& database, std::shared_ptr>& error) const; + + /** + * @brief remove an error from db error bucket + * Errors are removed by phaseOne, as soon as indexes no longer in plan + * + * @param database database + * + * @return success + */ + arangodb::Result removeDBError (std::string const& database); + + /** + * @brief copy all error maps (shards, indexes and databases) for Maintenance + * + * @param errors errors struct into which all maintenace feature error are copied + * @return success + */ + arangodb::Result copyAllErrors(errors_t& errors) const; + +protected: + /// @brief common code used by multiple constructors + void init(); + + /// @brief Search for action by hash + /// @return shared pointer to action object if exists, _actionRegistry.end() if not + std::shared_ptr findActionHash(size_t hash); + + /// @brief Search for action by hash (but lock already held by caller) + /// @return shared pointer to action object if exists, nullptr if not + std::shared_ptr findActionHashNoLock(size_t hash); + + /// @brief Search for action by Id + /// @return shared pointer to action object if exists, nullptr if not + std::shared_ptr findActionId(uint64_t id); + + /// @brief Search for action by Id (but lock already held by caller) + /// @return shared pointer to action object if exists, nullptr if not + std::shared_ptr findActionIdNoLock(uint64_t hash); + +protected: + /// @brief tunable option for thread pool size + int32_t _maintenanceThreadsMax; + + /// @brief tunable option for number of seconds COMPLETE or FAILED actions block + /// duplicates from adding to _actionRegistry + int32_t _secondsActionsBlock; + + /// @brief tunable option for number of seconds COMPLETE and FAILE actions remain + /// within _actionRegistry + int32_t _secondsActionsLinger; + + /// @brief flag to indicate when it is time to stop thread pool + std::atomic _isShuttingDown; + + /// @brief simple counter for creating MaintenanceAction id. Ok for it to roll over. + std::atomic _nextActionId; + + // + // Lock notes: + // Reading _actionRegistry requires Read or Write lock via _actionRegistryLock + // Writing _actionRegistry requires BOTH: + // - CONDITION_LOCKER on _actionRegistryCond + // - then write lock via _actionRegistryLock + // + /// @brief all actions executing, waiting, and done + std::deque> _actionRegistry; + + /// @brief lock to protect _actionRegistry and state changes to MaintenanceActions within + mutable arangodb::basics::ReadWriteLock _actionRegistryLock; + + /// @brief condition variable to motivate workers to find new action + arangodb::basics::ConditionVariable _actionRegistryCond; + + /// @brief list of background workers + std::vector _activeWorkers; + + /// @brief condition variable to indicate thread completion + arangodb::basics::ConditionVariable _workerCompletion; + + /// Errors are managed through raiseIndexError / removeIndexError and + /// raiseShardError / renoveShardError. According locks must be held in said + /// methods. + + /// @brief lock for index error bucket + mutable arangodb::Mutex _ieLock; + /// @brief pending errors raised by EnsureIndex + std::map>>> _indexErrors; + + /// @brief lock for shard error bucket + mutable arangodb::Mutex _seLock; + /// @brief pending errors raised by CreateCollection/UpdateCollection + std::unordered_map>> _shardErrors; + + /// @brief lock for database error bucket + mutable arangodb::Mutex _dbeLock; + /// @brief pending errors raised by CreateDatabase + std::unordered_map>> _dbErrors; + + + +}; + +} + +#endif diff --git a/arangod/Cluster/MaintenanceRestHandler.cpp b/arangod/Cluster/MaintenanceRestHandler.cpp new file mode 100644 index 0000000000..0b34190259 --- /dev/null +++ b/arangod/Cluster/MaintenanceRestHandler.cpp @@ -0,0 +1,193 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "MaintenanceRestHandler.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/StringUtils.h" +#include "Basics/conversions.h" +#include "Cluster/MaintenanceFeature.h" +#include "Rest/HttpRequest.h" +#include "Rest/HttpResponse.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::basics; +using namespace arangodb::rest; + +MaintenanceRestHandler::MaintenanceRestHandler(GeneralRequest* request, + GeneralResponse* response) + : RestBaseHandler(request, response) { + +} + +RestStatus MaintenanceRestHandler::execute() { + // extract the sub-request type + auto const type = _request->requestType(); + + switch(type) { + + // retrieve list of all actions + case rest::RequestType::GET: + getAction(); + break; + + // add an action to the list (or execute it directly) + case rest::RequestType::PUT: + putAction(); + break; + + // remove an action, stopping it if executing + case rest::RequestType::DELETE_REQ: + deleteAction(); + break; + + default: + generateError(rest::ResponseCode::METHOD_NOT_ALLOWED, + (int)rest::ResponseCode::METHOD_NOT_ALLOWED); + break; + } // switch + + return RestStatus::DONE; +} + + +void MaintenanceRestHandler::putAction() { + bool good(true); + VPackSlice parameters; + std::map param_map; + + try { + parameters = _request->payload(); + } catch (VPackException const& ex) { + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + std::string("expecting a valid JSON object in the request. got: ") + ex.what()); + good=false; + } // catch + + if (good && _request->payload().isEmptyObject()) { + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_CORRUPTED_JSON); + good=false; + } + + // convert vpack into key/value map + if (good) { + good = parsePutBody(parameters); + + // bad json + if (!good) { + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + std::string("unable to parse JSON object into key/value pairs.")); + } // if + } // if + + if (good) { + Result result; + + // build the action + auto maintenance = ApplicationServer::getFeature("Maintenance"); + result = maintenance->addAction(_actionDesc); + + if (!result.ok()) { + // possible errors? TRI_ERROR_BAD_PARAMETER TRI_ERROR_TASK_DUPLICATE_ID TRI_ERROR_SHUTTING_DOWN + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + result.errorMessage()); + } // if + } // if + +} // MaintenanceRestHandler::putAction + + +bool MaintenanceRestHandler::parsePutBody(VPackSlice const & parameters) { + bool good(true); + + std::map desc; + auto prop = std::make_shared(); + + VPackObjectIterator it(parameters, true); + for ( ; it.valid() && good; ++it) { + VPackSlice key, value; + + key = it.key(); + value = it.value(); + + // attempt insert into map ... but needs to be unique + if (key.isString() && value.isString()) { + good = desc.insert( + {key.copyString(), value.copyString()}).second; + } else if (key.isString() && (key.copyString() == "properties") + && value.isObject()) { + // code here + prop.reset(new VPackBuilder(value)); + } else { + good = false; + } // else + } // for + + _actionDesc = std::make_shared(desc, prop); + + return good; + +} // MaintenanceRestHandler::parsePutBody + + +void MaintenanceRestHandler::getAction() { + // build the action + auto maintenance = ApplicationServer::getFeature("Maintenance"); + + VPackBuilder registry = maintenance->toVelocyPack(); + generateResult(rest::ResponseCode::OK, registry.slice()); + +} // MaintenanceRestHandler::getAction + + +void MaintenanceRestHandler::deleteAction() { + auto maintenance = ApplicationServer::getFeature("Maintenance"); + + std::vector const& suffixes = _request->suffixes(); + + // must be one parameter: "all" or number + if (1 == suffixes.size()) { + std::string param = suffixes[0]; + Result result; + + if (param == "all") { + // Jobs supports all. Should Action too? Going with "no" for now. + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + result.errorMessage()); + } else { + uint64_t action_id = StringUtils::uint64(param); + result = maintenance->deleteAction(action_id); + + // can fail on bad id or if action already succeeded. + if (!result.ok()) { + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + result.errorMessage()); + } // if + } // else + + } else { + generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER); + } + +} // MaintenanceRestHandler::deleteAction diff --git a/arangod/Cluster/MaintenanceRestHandler.h b/arangod/Cluster/MaintenanceRestHandler.h new file mode 100644 index 0000000000..bb6002ecf1 --- /dev/null +++ b/arangod/Cluster/MaintenanceRestHandler.h @@ -0,0 +1,83 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_CLUSTER_MAINTENANCE_REST_HANDLER +#define ARANGOD_CLUSTER_MAINTENANCE_REST_HANDLER 1 + +#include +#include + +#include "Cluster/Action.h" +#include "RestHandler/RestBaseHandler.h" + +namespace arangodb { + +//////////////////////////////////////////////////////////////////////////////// +/// @brief Object that directs processing of one user maintenance requests +//////////////////////////////////////////////////////////////////////////////// + +class MaintenanceRestHandler : public RestBaseHandler { + public: + MaintenanceRestHandler(GeneralRequest*, GeneralResponse*); + + public: + char const* name() const override { return "MaintenanceRestHandler"; } + + RequestLane lane() const override final { return RequestLane::CLUSTER_INTERNAL; } + + /// @brief Performs routing of request to appropriate subroutines + RestStatus execute() override; + + // + // Accessors + // + /// @brief retrieve parsed action description + maintenance::ActionDescription const & getActionDesc() const {return *_actionDesc;}; + + /// @brief retrieve unparsed action properties + VPackBuilder const & getActionProp() const {return *(_actionDesc->properties());}; + +protected: + /// @brief PUT method adds an Action to the worklist (or executes action directly) + void putAction(); + + /// @brief GET method returns worklist + void getAction(); + + /// @brief DELETE method shifts non-finished action to Failed list. + /// (finished actions are untouched) + void deleteAction(); + + + /// @brief internal routine to convert PUT body into _actionDesc and _actionProp + bool parsePutBody(VPackSlice const & parameters); + + +protected: + std::shared_ptr _actionDesc; + + +}; +} + +#endif diff --git a/arangod/Cluster/MaintenanceWorker.cpp b/arangod/Cluster/MaintenanceWorker.cpp new file mode 100644 index 0000000000..8a30b291b5 --- /dev/null +++ b/arangod/Cluster/MaintenanceWorker.cpp @@ -0,0 +1,176 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "MaintenanceWorker.h" + +#include "lib/Logger/Logger.h" +#include "Cluster/MaintenanceFeature.h" + +namespace arangodb { + +namespace maintenance { + +MaintenanceWorker::MaintenanceWorker(arangodb::MaintenanceFeature & feature) + : Thread("MaintenanceWorker"), + _feature(feature), _curAction(nullptr), _loopState(eFIND_ACTION), + _directAction(false) { + + return; + +} // MaintenanceWorker::MaintenanceWorker + + +MaintenanceWorker::MaintenanceWorker(arangodb::MaintenanceFeature & feature, + std::shared_ptr & directAction) + : Thread("MaintenanceWorker"), + _feature(feature), _curAction(directAction), _loopState(eRUN_FIRST), + _directAction(true) { + + return; + +} // MaintenanceWorker::MaintenanceWorker + + +void MaintenanceWorker::run() { + bool more(false); + + while(eSTOP != _loopState && !_feature.isShuttingDown()){ + + switch(_loopState) { + case eFIND_ACTION: + _curAction = _feature.findReadyAction(); + more = (bool)_curAction; + break; + + case eRUN_FIRST: + _curAction->startStats(); + more = _curAction->first(); + break; + + case eRUN_NEXT: + more = _curAction->next(); + break; + + default: + _loopState = eSTOP; + LOG_TOPIC(ERR, Logger::CLUSTER) + << "MaintenanceWorkerRun: unexpected state (" << _loopState << ")"; + + } // switch + + // determine next loop state + nextState(more); + + } // while + +} // MaintenanceWorker::run + + +void MaintenanceWorker::nextState(bool actionMore) { + + // bad result code forces actionMore to false + if (_curAction && (!_curAction->result().ok() + || FAILED == _curAction->getState())) + { + actionMore = false; + } // if + + // actionMore means iterate again + if (actionMore) { + // There should be an valid _curAction + if (_curAction) { + if (eFIND_ACTION == _loopState) { + _loopState = eRUN_FIRST; + } else { + _curAction->incStats(); + _loopState = eRUN_NEXT; + } // if + + // move execution to PreAction if it exists + if (_curAction->getPreAction()) { + std::shared_ptr tempPtr; + + _curAction->setState(WAITING); + tempPtr=_curAction; + _curAction=_curAction->getPreAction(); + _curAction->setPostAction( + std::make_shared(tempPtr->describe())); + _loopState = eRUN_FIRST; + } // if + } else { + // this state should not exist, but deal with it + _loopState = (_directAction ? eSTOP : eFIND_ACTION); + } // else + } else { + // finish the current action + if (_curAction) { + _lastResult = _curAction->result(); + + // if action's state not set, assume it succeeded when result ok + if (_curAction->result().ok() + && FAILED != _curAction->getState()) { + _curAction->endStats(); + _curAction->setState(COMPLETE); + + // continue execution with "next" action tied to this one + if (_curAction->getPostAction()) { + _curAction = _curAction->getPostAction(); + _curAction->clearPreAction(); + _loopState = (WAITING == _curAction->getState() ? eRUN_NEXT : eRUN_FIRST); + _curAction->setState(EXECUTING); + } else { + _curAction.reset(); + _loopState = (_directAction ? eSTOP : eFIND_ACTION); + } // else + } else { + std::shared_ptr failAction(_curAction); + + // fail all actions that would follow + do { + failAction->setState(FAILED); + failAction->endStats(); + failAction=failAction->getPostAction(); + } while(failAction); + _loopState = (_directAction ? eSTOP : eFIND_ACTION); + } // else + } else { + // no current action, go back to hunting for one + _loopState = (_directAction ? eSTOP : eFIND_ACTION); + } // else + } // else + + +} // MaintenanceWorker::nextState + +#if 0 +std::shared_ptr MaintenanceWorker::findReadyAction() { + std::shared_ptr ret_ptr; + + + + return ret_ptr; + +} // Maintenance::findReadyAction +#endif +} // namespace maintenance +} // namespace arangodb diff --git a/arangod/Cluster/MaintenanceWorker.h b/arangod/Cluster/MaintenanceWorker.h new file mode 100644 index 0000000000..5da4111363 --- /dev/null +++ b/arangod/Cluster/MaintenanceWorker.h @@ -0,0 +1,89 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_CLUSTER_MAINTENANCE_WORKER +#define ARANGOD_CLUSTER_MAINTENANCE_WORKER 1 + +#include "Basics/Thread.h" +#include "Cluster/Action.h" + +namespace arangodb { + +class MaintenanceFeature; + +namespace maintenance { + +class MaintenanceWorker : public Thread { + public: + MaintenanceWorker(MaintenanceFeature& feature); + MaintenanceWorker( + MaintenanceFeature& feature, std::shared_ptr& directAction); + + virtual ~MaintenanceWorker() {}; + + // + // MaintenanceWorker entry points + // + + /// @brief Thread object entry point + virtual void run(); + + /// @brief Share internal result state, likely derived from most recent action + /// @returns Result object + Result result() const {return _lastResult;}; + +protected: + enum WorkerState { + eSTOP = 1, + eFIND_ACTION = 2, + eRUN_FIRST = 3, + eRUN_NEXT = 4, + + }; + + /// @brief Determine next loop state + /// @param true if action wants to continue, false if action done + void nextState(bool); + + /// @brief Find an action that is ready to process within feature's deque + /// @return action to process, or nullptr + + arangodb::MaintenanceFeature & _feature; + + std::shared_ptr _curAction; + + WorkerState _loopState; + + bool _directAction; + + Result _lastResult; + +private: + MaintenanceWorker(MaintenanceWorker const &) = delete; + +};// class MaintenanceWorker + +} // namespace maintenance +} // namespace arangodb + +#endif diff --git a/arangod/Cluster/NonAction.cpp b/arangod/Cluster/NonAction.cpp new file mode 100644 index 0000000000..4031da0c7b --- /dev/null +++ b/arangod/Cluster/NonAction.cpp @@ -0,0 +1,49 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "NonAction.h" +#include "MaintenanceFeature.h" + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; + +NonAction::NonAction( + MaintenanceFeature& feature, ActionDescription const& desc) + : ActionBase(feature, desc) { + std::string const error = + std::string("Unknown maintenance action '") + desc.name() + "'"; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error; + _result = arangodb::Result(TRI_ERROR_INTERNAL, error); +} + +bool NonAction::first() { + std::string const error = + std::string("Unknown maintenance action '") + _description.name() + "'"; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error; + _result = arangodb::Result(TRI_ERROR_INTERNAL, error); + return false; +} + +NonAction::~NonAction() {} diff --git a/arangod/Cluster/NonAction.h b/arangod/Cluster/NonAction.h new file mode 100644 index 0000000000..31eac4a4da --- /dev/null +++ b/arangod/Cluster/NonAction.h @@ -0,0 +1,53 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_NON_ACTION_H +#define ARANGODB_MAINTENANCE_NON_ACTION_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +/** + * @brief Dummy action for failure to find action + */ +class NonAction : public ActionBase { + +public: + + NonAction(MaintenanceFeature&, ActionDescription const& d); + + virtual ~NonAction(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/ReplicationTimeoutFeature.cpp b/arangod/Cluster/ReplicationTimeoutFeature.cpp index a0a4735be6..f9d5c06711 100644 --- a/arangod/Cluster/ReplicationTimeoutFeature.cpp +++ b/arangod/Cluster/ReplicationTimeoutFeature.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ReplicationTimeoutFeature.h b/arangod/Cluster/ReplicationTimeoutFeature.h index 57610cce64..05c3f844c7 100644 --- a/arangod/Cluster/ReplicationTimeoutFeature.h +++ b/arangod/Cluster/ReplicationTimeoutFeature.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ResignShardLeadership.cpp b/arangod/Cluster/ResignShardLeadership.cpp new file mode 100644 index 0000000000..3d6609dad0 --- /dev/null +++ b/arangod/Cluster/ResignShardLeadership.cpp @@ -0,0 +1,136 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "ResignShardLeadership.h" +#include "MaintenanceFeature.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/FollowerInfo.h" +#include "Transaction/StandaloneContext.h" +#include "Transaction/Methods.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Databases.h" + +#include +#include +#include +#include + + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; + +ResignShardLeadership::ResignShardLeadership( + MaintenanceFeature& feature, ActionDescription const& desc) + : ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(DATABASE)) { + error << "database must be specified"; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!desc.has(SHARD)) { + error << "shard must be specified"; + } + TRI_ASSERT(desc.has(SHARD)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "ResignLeadership: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +ResignShardLeadership::~ResignShardLeadership() {}; + +bool ResignShardLeadership::first() { + + auto const& database = _description.get(DATABASE); + auto const& collection = _description.get(SHARD); + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "trying to withdraw as leader of shard '" << database << "/" << collection; + + // This starts a write transaction, just to wait for any ongoing + // write transaction on this shard to terminate. We will then later + // report to Current about this resignation. If a new write operation + // starts in the meantime (which is unlikely, since no coordinator that + // has seen the _ will start a new one), it is doomed, and we ignore the + // problem, since similar problems can arise in failover scenarios anyway. + + try { + + // Guard database againts deletion for now + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto col = vocbase->lookupCollection(collection); + if (col == nullptr) { + std::stringstream error; + error << "Failed to lookup local collection " << collection + << " in database " + database; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "EnsureIndex: " << error.str(); + _result.reset(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, error.str()); + return false; + } + + col->followers()->setTheLeader("LEADER_NOT_YET_KNOWN"); // resign + // Note that it is likely that we will be a follower for this shard + // with another leader in due course. However, we do not know the + // name of the new leader yet. This setting will make us a follower + // for now but we will not accept any replication operation from any + // leader, until we have negotiated a deal with it. Then the actual + // name of the leader will be set. + + // Get write transaction on collection + auto ctx = std::make_shared(*vocbase); + transaction::Methods trx(ctx, {}, {collection}, {}, transaction::Options()); + + Result res = trx.begin(); + + if (!res.ok()) { + THROW_ARANGO_EXCEPTION(res); + } + + } catch (std::exception const& e) { + std::stringstream error; + error << "exception thrown when resigning:" << e.what(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "ResignLeadership: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/ResignShardLeadership.h b/arangod/Cluster/ResignShardLeadership.h new file mode 100644 index 0000000000..884901e2fb --- /dev/null +++ b/arangod/Cluster/ResignShardLeadership.h @@ -0,0 +1,50 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_RESIGN_SHARD_LEADERSHIP_H +#define ARANGODB_MAINTENANCE_RESIGN_SHARD_LEADERSHIP_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +class ResignShardLeadership : public ActionBase { + +public: + + ResignShardLeadership(MaintenanceFeature&, ActionDescription const& d); + + virtual ~ResignShardLeadership(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/RestAgencyCallbacksHandler.cpp b/arangod/Cluster/RestAgencyCallbacksHandler.cpp index 33a31cd79e..3432e202c2 100644 --- a/arangod/Cluster/RestAgencyCallbacksHandler.cpp +++ b/arangod/Cluster/RestAgencyCallbacksHandler.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/RestAgencyCallbacksHandler.h b/arangod/Cluster/RestAgencyCallbacksHandler.h index fcecc0b519..2fe47b210d 100644 --- a/arangod/Cluster/RestAgencyCallbacksHandler.h +++ b/arangod/Cluster/RestAgencyCallbacksHandler.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ServerState.cpp b/arangod/Cluster/ServerState.cpp index fdca331b86..02bf76b241 100644 --- a/arangod/Cluster/ServerState.cpp +++ b/arangod/Cluster/ServerState.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/ServerState.h b/arangod/Cluster/ServerState.h index 42f4063b18..989a416136 100644 --- a/arangod/Cluster/ServerState.h +++ b/arangod/Cluster/ServerState.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/SynchronizeShard.cpp b/arangod/Cluster/SynchronizeShard.cpp new file mode 100644 index 0000000000..016e7143a9 --- /dev/null +++ b/arangod/Cluster/SynchronizeShard.cpp @@ -0,0 +1,957 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "SynchronizeShard.h" + +#include "Agency/TimeString.h" +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ActionDescription.h" +#include "Cluster/ClusterComm.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/FollowerInfo.h" +#include "Cluster/MaintenanceFeature.h" +#include "Cluster/ServerState.h" +#include "Replication/DatabaseTailingSyncer.h" +#include "Replication/DatabaseInitialSyncer.h" +#include "Replication/DatabaseReplicationApplier.h" +#include "Replication/GlobalInitialSyncer.h" +#include "Replication/GlobalReplicationApplier.h" +#include "Replication/ReplicationApplierConfiguration.h" +#include "Replication/ReplicationFeature.h" +#include "RestServer/DatabaseFeature.h" +#include "Transaction/StandaloneContext.h" +#include "Utils/SingleCollectionTransaction.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Databases.h" + +#include +#include +#include +#include + + +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; +using namespace arangodb::transaction; +using namespace arangodb; + +constexpr auto REPL_HOLD_READ_LOCK = "/_api/replication/holdReadLockCollection"; +constexpr auto REPL_ADD_FOLLOWER = "/_api/replication/addFollower"; +constexpr auto REPL_REM_FOLLOWER = "/_api/replication/removeFollower"; + +std::string const READ_LOCK_TIMEOUT ("startReadLockOnLeader: giving up"); +std::string const DB ("/_db/"); +std::string const SYSTEM ("/_db/_system"); +std::string const TTL ("ttl"); +std::string const REPL_BARRIER_API ("/_api/replication/barrier/"); +std::string const ENDPOINT("endpoint"); +std::string const INCREMENTAL("incremental"); +std::string const KEEP_BARRIER("keepBarrier"); +std::string const LEADER_ID("leaderId"); +std::string const SKIP_CREATE_DROP("skipCreateDrop"); +std::string const COLLECTIONS("collections"); +std::string const LAST_LOG_TICK("lastLogTick"); +std::string const BARRIER_ID("barrierId"); +std::string const FOLLOWER_ID("followerId"); +std::string const RESTRICT_TYPE("restrictType"); +std::string const RESTRICT_COLLECTIONS("restrictCollections"); +std::string const INCLUDE("include"); +std::string const INCLUDE_SYSTEM("includeSystem"); +using namespace std::chrono; + +SynchronizeShard::SynchronizeShard( + MaintenanceFeature& feature, ActionDescription const& desc) : + ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(COLLECTION)) { + error << "collection must be specified"; + } + TRI_ASSERT(desc.has(COLLECTION)); + + if (!desc.has(DATABASE)) { + error << "database must be specified"; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!desc.has(SHARD)) { + error << "SHARD must be stecified"; + } + TRI_ASSERT(desc.has(SHARD)); + + if (!desc.has(LEADER)) { + error << "leader must be stecified"; + } + TRI_ASSERT(desc.has(LEADER)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "SynchronizeShard: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +class SynchronizeShardCallback : public arangodb::ClusterCommCallback { +public: + SynchronizeShardCallback(SynchronizeShard* callie) {}; + virtual bool operator()(arangodb::ClusterCommResult*) override final { + return true; + } +}; + + +arangodb::Result getReadLockId ( + std::string const& endpoint, std::string const& database, + std::string const& clientId, double timeout, uint64_t& id) { + + std::string error("startReadLockOnLeader: Failed to get read lock - "); + + auto cc = arangodb::ClusterComm::instance(); + if (cc == nullptr) { // nullptr only happens during controlled shutdown + return arangodb::Result( + TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); + } + + auto comres = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::GET, + DB + database + REPL_HOLD_READ_LOCK, std::string(), + std::unordered_map(), timeout); + + auto result = comres->result; + + if (result != nullptr && result->getHttpReturnCode() == 200) { + auto const idv = comres->result->getBodyVelocyPack(); + auto const& idSlice = idv->slice(); + TRI_ASSERT(idSlice.isObject()); + TRI_ASSERT(idSlice.hasKey(ID)); + try { + id = std::stoll(idSlice.get(ID).copyString()); + } catch (std::exception const& e) { + error += " expecting id to be int64_t "; + error += idSlice.toJson(); + return arangodb::Result(TRI_ERROR_INTERNAL, error); + } + } else { + error += result->getHttpReturnMessage(); + return arangodb::Result(TRI_ERROR_INTERNAL, error); + } + + return arangodb::Result(); + +} + + +arangodb::Result count( + std::shared_ptr const col, uint64_t& c) { + + std::string collectionName(col->name()); + auto ctx = std::make_shared(col->vocbase()); + SingleCollectionTransaction trx( + ctx, collectionName, AccessMode::Type::READ); + + Result res = trx.begin(); + if (!res.ok()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Failed to start count transaction: " << res; + return res; + } + + OperationResult opResult = trx.count(collectionName, + arangodb::transaction::CountType::Normal); + res = trx.finish(opResult.result); + + if (res.fail()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "Failed to finish count transaction: " << res; + return res; + } + + VPackSlice s = opResult.slice(); + TRI_ASSERT(s.isNumber()); + c = s.getNumber(); + + return opResult.result; + +} + +arangodb::Result addShardFollower ( + std::string const& endpoint, std::string const& database, + std::string const& shard, uint64_t lockJobId, + std::string const& clientId, double timeout = 120.0 ) { + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "addShardFollower: tell the leader to put us into the follower list..."; + + auto cc = arangodb::ClusterComm::instance(); + if (cc == nullptr) { // nullptr only happens during controlled shutdown + return arangodb::Result( + TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); + } + + try { + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto collection = vocbase->lookupCollection(shard); + if (collection == nullptr) { + std::string errorMsg( + "SynchronizeShard::addShardFollower: Failed to lookup collection "); + errorMsg += shard; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << errorMsg; + return arangodb::Result(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, errorMsg); + } + + size_t c; + count(collection, c); + VPackBuilder body; + { VPackObjectBuilder b(&body); + body.add(FOLLOWER_ID, VPackValue(arangodb::ServerState::instance()->getId())); + body.add(SHARD, VPackValue(shard)); + body.add("checksum", VPackValue(std::to_string(c))); + if (lockJobId != 0) { + body.add("readLockId", VPackValue(lockJobId)); + }} + + auto comres = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::PUT, + DB + database + REPL_ADD_FOLLOWER, body.toJson(), + std::unordered_map(), timeout); + + auto result = comres->result; + std::string errorMessage ( + "addShardFollower: could not add us to the leader's follower list. "); + if (result == nullptr || result->getHttpReturnCode() != 200) { + if (lockJobId != 0) { + errorMessage += comres->stringifyErrorMessage(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << errorMessage; + } else { + errorMessage += "with shortcut."; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << errorMessage; + } + return arangodb::Result(TRI_ERROR_INTERNAL, errorMessage); + } + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "cancelReadLockOnLeader: success"; + return arangodb::Result(); + } catch (std::exception const& e) { + std::string errorMsg( + "SynchronizeShard::addShardFollower: Failed to lookup database "); + errorMsg += database; + errorMsg += " exception: "; + errorMsg += e.what(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << errorMsg; + return arangodb::Result(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, errorMsg); + } +} + + +arangodb::Result removeShardFollower ( + std::string const& endpoint, std::string const& database, + std::string const& shard, std::string const& clientId, double timeout = 120.0) { + + + LOG_TOPIC(WARN, Logger::MAINTENANCE) << + "removeShardFollower: tell the leader to take us off the follower list..."; + + auto cc = arangodb::ClusterComm::instance(); + if (cc == nullptr) { // nullptr only happens during controlled shutdown + return arangodb::Result( + TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); + } + + VPackBuilder body; + { VPackObjectBuilder b(&body); + body.add(SHARD, VPackValue(shard)); + body.add(FOLLOWER_ID, + VPackValue(arangodb::ServerState::instance()->getId())); } + + // Note that we always use the _system database here because the actual + // database might be gone already on the leader and we need to cancel + // the read lock under all circumstances. + auto comres = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::PUT, + DB + database + REPL_REM_FOLLOWER, body.toJson(), + std::unordered_map(), timeout); + + auto result = comres->result; + if (result == nullptr || result->getHttpReturnCode() != 200) { + std::string errorMessage( + "removeShardFollower: could not remove us from the leader's follower list: "); + errorMessage += result->getHttpReturnCode(); + errorMessage += comres->stringifyErrorMessage(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << errorMessage; + return arangodb::Result(TRI_ERROR_INTERNAL, errorMessage); + } + + LOG_TOPIC(WARN, Logger::MAINTENANCE) << "removeShardFollower: success" ; + return arangodb::Result(); + +} + +arangodb::Result cancelReadLockOnLeader ( + std::string const& endpoint, std::string const& database, + uint64_t lockJobId, std::string const& clientId, + double timeout = 10.0) { + + auto cc = arangodb::ClusterComm::instance(); + if (cc == nullptr) { // nullptr only happens during controlled shutdown + return arangodb::Result( + TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); + } + + VPackBuilder body; + { VPackObjectBuilder b(&body); + body.add(ID, VPackValue(std::to_string(lockJobId))); } + + // Note that we always use the _system database here because the actual + // database might be gone already on the leader and we need to cancel + // the read lock under all circumstances. + auto comres = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::DELETE_REQ, + SYSTEM + REPL_HOLD_READ_LOCK, body.toJson(), + std::unordered_map(), timeout); + + auto result = comres->result; + + if (result == nullptr || result->getHttpReturnCode() != 200) { + auto errorMessage = comres->stringifyErrorMessage(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "cancelReadLockOnLeader: exception caught for " << body.toJson() + << ": " << errorMessage; + return arangodb::Result(TRI_ERROR_INTERNAL, errorMessage); + } + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "cancelReadLockOnLeader: success"; + return arangodb::Result(); + +} + + +arangodb::Result cancelBarrier( + std::string const& endpoint, std::string const& database, + int64_t barrierId, std::string const& clientId, + double timeout = 120.0) { + + if (barrierId <= 0) { + return Result();; + } + + auto cc = arangodb::ClusterComm::instance(); + if (cc == nullptr) { // nullptr only happens during controlled shutdown + return arangodb::Result( + TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); + } + + auto comres = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::DELETE_REQ, + DB + database + REPL_BARRIER_API + std::to_string(barrierId), std::string(), + std::unordered_map(), timeout); + + if (comres->status == CL_COMM_SENT) { + auto result = comres->result; + if (result != nullptr && result->getHttpReturnCode() != 200 && + result->getHttpReturnCode() != 204) { + auto errorMessage = comres->stringifyErrorMessage(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "CancelBarrier: error" << errorMessage; + return arangodb::Result(TRI_ERROR_INTERNAL, errorMessage); + } + } else { + std::string error ("CancelBarrier: failed to send message to leader : status "); + error += comres->status; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error; + return arangodb::Result(TRI_ERROR_INTERNAL, error); + } + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "cancelBarrier: success"; + return arangodb::Result(); + +} + + +arangodb::Result SynchronizeShard::getReadLock( + std::string const& endpoint, std::string const& database, + std::string const& collection, std::string const& clientId, + uint64_t rlid, double timeout) { + + auto cc = arangodb::ClusterComm::instance(); + if (cc == nullptr) { // nullptr only happens during controlled shutdown + return arangodb::Result( + TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); + } + + VPackBuilder body; + { VPackObjectBuilder o(&body); + body.add(ID, VPackValue(std::to_string(rlid))); + body.add(COLLECTION, VPackValue(collection)); + body.add(TTL, VPackValue(timeout)); } + + auto url = DB + database + REPL_HOLD_READ_LOCK; + + cc->asyncRequest( + clientId, 2, endpoint, rest::RequestType::POST, url, + std::make_shared(body.toJson()), + std::unordered_map(), + std::make_shared(this), timeout, true, timeout); + + // Intentionally do not look at the outcome, even in case of an error + // we must make sure that the read lock on the leader is not active! + // This is done automatically below. + + size_t count = 0; + while (++count < 20) { // wait for some time until read lock established: + + // Now check that we hold the read lock: + auto putres = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::PUT, url, body.toJson(), + std::unordered_map(), timeout); + + auto result = putres->result; + if (result != nullptr && result->getHttpReturnCode() == 200) { + auto const vp = putres->result->getBodyVelocyPack(); + auto const& slice = vp->slice(); + TRI_ASSERT(slice.isObject()); + if (slice.hasKey("lockHeld") && slice.get("lockHeld").isBoolean() && + slice.get("lockHeld").getBool()) { + return arangodb::Result(); + } + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "startReadLockOnLeader: Lock not yet acquired..."; + } else { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "startReadLockOnLeader: Do not see read lock yet:" + << putres->stringifyErrorMessage(); + } + + std::this_thread::sleep_for(duration(.5)); + } + + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "startReadLockOnLeader: giving up"; + + try { + auto r = cc->syncRequest( + clientId, 1, endpoint, rest::RequestType::DELETE_REQ, url, body.toJson(), + std::unordered_map(), timeout); + if (r->result == nullptr && r->result->getHttpReturnCode() != 200) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "startReadLockOnLeader: cancelation error for shard - " << collection + << " " << r->getErrorCode() << ": " << r->stringifyErrorMessage(); + } + } catch (std::exception const& e) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "startReadLockOnLeader: expection in cancel: " << e.what(); + } + + return arangodb::Result(TRI_ERROR_CLUSTER_TIMEOUT, READ_LOCK_TIMEOUT); + +} + +bool isStopping() { + return application_features::ApplicationServer::isStopping(); +} + +arangodb::Result SynchronizeShard::startReadLockOnLeader( + std::string const& endpoint, std::string const& database, + std::string const& collection, std::string const& clientId, + uint64_t& rlid, double timeout) { + + // Read lock id + rlid = 0; + arangodb::Result result = + getReadLockId(endpoint, database, clientId, timeout, rlid); + if (!result.ok()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << result.errorMessage(); + return result; + } else { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "Got read lock id: " << rlid; + } + + result = getReadLock(endpoint, database, collection, clientId, rlid, timeout); + + return result; + +} + +enum ApplierType { + APPLIER_DATABASE, + APPLIER_GLOBAL +}; + +arangodb::Result replicationSynchronize( + std::shared_ptr const col, VPackSlice const& config, + ApplierType applierType, std::shared_ptr sy) { + + auto& vocbase = col->vocbase(); + + auto database = vocbase.name(); + + auto shard = col->name(); + + bool keepBarrier = config.get(KEEP_BARRIER).getBool(); + std::string leaderId; + if (config.hasKey(LEADER_ID)) { + leaderId = config.get(LEADER_ID).copyString(); + } + + ReplicationApplierConfiguration configuration = + ReplicationApplierConfiguration::fromVelocyPack(config, database); + configuration.validate(); + + std::shared_ptr syncer; + + config.toJson(); + + if (applierType == APPLIER_DATABASE) { + // database-specific synchronization + syncer.reset(new DatabaseInitialSyncer(vocbase, configuration)); + + if (!leaderId.empty()) { + syncer->setLeaderId(leaderId); + } + } else if (applierType == APPLIER_GLOBAL) { + configuration._skipCreateDrop = false; + syncer.reset(new GlobalInitialSyncer(configuration)); + } else { + TRI_ASSERT(false); + } + + try { + Result r = syncer->run(configuration._incremental); + + if (r.fail()) { + LOG_TOPIC(ERR, Logger::REPLICATION) + << "initial sync failed for database '" << database << "': " + << r.errorMessage(); + THROW_ARANGO_EXCEPTION_MESSAGE( + r.errorNumber(), "cannot sync from remote endpoint: " + + r.errorMessage() + ". last progress message was '" + syncer->progress() + + "'"); + } + + { VPackObjectBuilder o(sy.get()); + if (keepBarrier) { + sy->add(BARRIER_ID, VPackValue(syncer->stealBarrier())); + } + sy->add(LAST_LOG_TICK, VPackValue(syncer->getLastLogTick())); + sy->add(VPackValue(COLLECTIONS)); + { VPackArrayBuilder a(sy.get()); + for (auto const& i : syncer->getProcessedCollections()) { + VPackObjectBuilder e(sy.get()); + sy->add(ID, VPackValue(i.first)); + sy->add(NAME, VPackValue(i.second)); + }}} + + } catch (arangodb::basics::Exception const& ex) { + std::string s("cannot sync from remote endpoint: "); + s += ex.what() + std::string(". last progress message was '") + syncer->progress() + "'"; + return Result(ex.code(), ex.what()); + } catch (std::exception const& ex) { + std::string s("cannot sync from remote endpoint: "); + s += ex.what() + std::string(". last progress message was '") + syncer->progress() + "'"; + return Result(TRI_ERROR_INTERNAL, ex.what()); + } catch (...) { + std::string s( + "cannot sync from remote endpoint: unknown exception. last progress message was '"); + s+= syncer->progress() + "'"; + return Result(TRI_ERROR_INTERNAL); + } + + return arangodb::Result(); +} + + +arangodb::Result replicationSynchronizeFinalize(VPackSlice const& conf) { + + auto const database = conf.get(DATABASE).copyString(); + auto const collection = conf.get(COLLECTION).copyString(); + auto const leaderId = conf.get(LEADER_ID).copyString(); + auto const fromTick = conf.get("from").getNumber(); + + ReplicationApplierConfiguration configuration = + ReplicationApplierConfiguration::fromVelocyPack(conf, database); + // will throw if invalid + configuration.validate(); + + DatabaseGuard guard(database); + DatabaseTailingSyncer syncer(guard.database(), configuration, fromTick, true, 0); + + if (!leaderId.empty()) { + syncer.setLeaderId(leaderId); + } + + Result r; + try { + r = syncer.syncCollectionFinalize(collection); + } catch (arangodb::basics::Exception const& ex) { + r = Result(ex.code(), ex.what()); + } catch (std::exception const& ex) { + r = Result(TRI_ERROR_INTERNAL, ex.what()); + } catch (...) { + r = Result(TRI_ERROR_INTERNAL, "unknown exception"); + } + + if (r.fail()) { + LOG_TOPIC(ERR, Logger::REPLICATION) + << "syncCollectionFinalize failed: " << r.errorMessage(); + } + + return r; +} + + +bool SynchronizeShard::first() { + + std::string database = _description.get(DATABASE); + std::string planId = _description.get(COLLECTION); + std::string shard = _description.get(SHARD); + std::string leader = _description.get(LEADER); + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "SynchronizeShard: synchronizing shard '" << database << "/" << shard + << "' for central '" << database << "/" << planId << "'"; + + auto* clusterInfo = ClusterInfo::instance(); + auto const ourselves = arangodb::ServerState::instance()->getId(); + auto startTime = system_clock::now(); + auto const startTimeStr = timepointToString(startTime); + auto const clientId(database + planId + shard + leader); + + // First wait until the leader has created the shard (visible in + // Current in the Agency) or we or the shard have vanished from + // the plan: + while(true) { + + if (isStopping()) { + _result.reset(TRI_ERROR_INTERNAL, "shutting down"); + return false; + } + + std::vector planned; + auto result = clusterInfo->getShardServers(shard, planned); + + if (!result.ok() || + std::find(planned.begin(), planned.end(), ourselves) == planned.end() || + planned.front() != leader) { + // Things have changed again, simply terminate: + auto const endTime = system_clock::now(); + std::stringstream error; + error << "cancelled, " << database << "/" << shard << ", " << database + << "/" << planId << ", started " << startTimeStr << ", ended " + << timepointToString(endTime); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "SynchronizeOneShard: " << error.str(); + _result.reset(TRI_ERROR_FAILED, error.str()); + return false; + } + + std::shared_ptr ci = + clusterInfo->getCollection(database, planId); + TRI_ASSERT(ci != nullptr); + + std::string const cid = std::to_string(ci->id()); + std::shared_ptr cic = + ClusterInfo::instance()->getCollectionCurrent(database, cid); + std::vector current = cic->servers(shard); + + if (current.empty()) { + Result(TRI_ERROR_FAILED, + "synchronizeOneShard: cancelled, no servers in 'Current'"); + } + if (current.front() == leader) { + if (std::find(current.begin(), current.end(), ourselves) == current.end()) { + break; // start synchronization work + } + // We are already there, this is rather strange, but never mind: + auto const endTime = system_clock::now(); + std::stringstream error; + error + << "already done, " << database << "/" << shard + << ", " << database << "/" << planId << ", started " + << startTimeStr << ", ended " << timepointToString(endTime); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "SynchronizeOneShard: " << error.str(); + _result.reset(TRI_ERROR_FAILED, error.str()); + return false; + } + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "synchronizeOneShard: waiting for leader, " << database + << "/" << shard << ", " << database << "/" << planId; + + std::this_thread::sleep_for(duration(0.2)); + + } + + // Once we get here, we know that the leader is ready for sync, so we give it a try: + + try { + + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + auto collection = vocbase->lookupCollection(shard); + if (collection == nullptr) { + std::stringstream error; + error << "failed to lookup local shard " << shard; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "SynchronizeOneShard: " << error.str(); + _result.reset(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, error.str()); + return false; + } + + auto ep = clusterInfo->getServerEndpoint(leader); + size_t c; + if (!count(collection, c).ok()) { + std::stringstream error; + error << "failed to get a count on leader " << shard; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "SynchronizeShard " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + if (c == 0) { + // We have a short cut: + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << + "synchronizeOneShard: trying short cut to synchronize local shard '" << + database << "/" << shard << "' for central '" << database << "/" << + planId << "'"; + try { + + auto asResult = addShardFollower(ep, database, shard, 0, clientId, 60.0); + + if (asResult.ok()) { + + auto const endTime = system_clock::now(); + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "synchronizeOneShard: shortcut worked, done, " << database << "/" + << shard << ", " << database << "/" << planId <<", started: " + << startTimeStr << " ended: " << timepointToString(endTime); + collection->followers()->setTheLeader(leader); + notify(); + return false; + } + } catch (...) {} + } + + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "synchronizeOneShard: trying to synchronize local shard '" << database + << "/" << shard << "' for central '" << database << "/" << planId << "'"; + + try { + // First once without a read transaction: + + if (isStopping()) { + _result.reset(TRI_ERROR_INTERNAL, "server is shutting down"); + } + + collection->followers()->setTheLeader(leader); + + if (leader.empty()) { + collection->followers()->clear(); + } + + // do not reset followers when we resign at this time...we are + // still the only source of truth to trust, in particular, in the + // planned leader resignation, we will shortly after the call to + // this function here report the controlled resignation to the + // agency. This report must still contain the correct follower list + // or else the supervision is super angry with us. + + startTime = system_clock::now(); + + VPackBuilder config; + { VPackObjectBuilder o(&config); + config.add(ENDPOINT, VPackValue(ep)); + config.add(INCREMENTAL, VPackValue(true)); + config.add(KEEP_BARRIER, VPackValue(true)); + config.add(LEADER_ID, VPackValue(leader)); + config.add(SKIP_CREATE_DROP, VPackValue(true)); + config.add(RESTRICT_TYPE, VPackValue(INCLUDE)); + config.add(VPackValue(RESTRICT_COLLECTIONS)); + { VPackArrayBuilder a(&config); + config.add(VPackValue(shard)); } + config.add(INCLUDE_SYSTEM, VPackValue(true)); + config.add("verbose", VPackValue(false)); } + + auto details = std::make_shared(); + + Result syncRes = replicationSynchronize( + collection, config.slice(), APPLIER_DATABASE, details); + + auto sy = details->slice(); + auto const endTime = system_clock::now(); + bool longSync = false; + + // Long shard sync initialisation + if (endTime-startTime > seconds(5)) { + LOG_TOPIC(WARN, Logger::MAINTENANCE) + << "synchronizeOneShard: long call to syncCollection for shard" + << shard << " " << syncRes.errorMessage() << " start time: " + << timepointToString(startTime) << "end time: " + << timepointToString(system_clock::now()); + longSync = true; + } + + // + if (!syncRes.ok()) { + + std::stringstream error; + error << "could not initially synchronize shard " << shard + << syncRes.errorMessage(); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "SynchronizeOneShard: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + + } else { + + VPackSlice collections = sy.get(COLLECTIONS); + + if (collections.length() == 0 || + collections[0].get("name").copyString() != shard) { + + if (longSync) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: long sync, before cancelBarrier" + << timepointToString(system_clock::now()); + } + cancelBarrier(ep, database, sy.get(BARRIER_ID).getNumber(), clientId); + if (longSync) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: long sync, after cancelBarrier" + << timepointToString(system_clock::now()); + } + + std::stringstream error; + error << "shard " << shard << " seems to be gone from leader!"; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "SynchronizeOneShard: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + + } else { + + // Now start a read transaction to stop writes: + uint64_t lockJobId = 0; + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "synchronizeOneShard: startReadLockOnLeader: " << ep << ":" + << database << ":" << collection->name(); + Result result = startReadLockOnLeader( + ep, database, collection->name(), clientId, lockJobId); + if (result.ok()) { + LOG_TOPIC(DEBUG, Logger::FIXME) << "lockJobId: " << lockJobId; + } else { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: error in startReadLockOnLeader:" + << result.errorMessage(); + } + + cancelBarrier(ep, database, sy.get("barrierId").getNumber(), clientId); + + if (lockJobId != 0) { + + VPackBuilder builder; + { VPackObjectBuilder o(&builder); + builder.add(ENDPOINT, VPackValue(ep)); + builder.add(DATABASE, VPackValue(database)); + builder.add(COLLECTION, VPackValue(shard)); + builder.add(LEADER_ID, VPackValue(leader)); + builder.add("from", sy.get(LAST_LOG_TICK)); + builder.add("requestTimeout", VPackValue(60.0)); + builder.add("connectTimeout", VPackValue(60.0)); + } + + Result fres = replicationSynchronizeFinalize (builder.slice()); + + if (fres.ok()) { + result = addShardFollower(ep, database, shard, lockJobId, clientId, 60.0); + + if (!result.ok()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: failed to add follower" + << result.errorMessage(); + } + } else { + std::string errorMessage( + "synchronizeOneshard: error in syncCollectionFinalize: ") ; + errorMessage += fres.errorMessage(); + result = Result(TRI_ERROR_INTERNAL, errorMessage); + } + + // This result is unused, only in logs + Result lockResult = cancelReadLockOnLeader(ep, database, lockJobId, clientId, 60.0); + if (!lockResult.ok()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: read lock has timed out for shard " << shard; + } + } else { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: lockJobId was false for shard" << shard; + } + + if (result.ok()) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "synchronizeOneShard: synchronization worked for shard " << shard; + _result.reset(TRI_ERROR_NO_ERROR); + } else { + LOG_TOPIC(ERR, Logger::MAINTENANCE) + << "synchronizeOneShard: synchronization failed for shard " << shard; + std::string errorMessage( + "synchronizeOneShard: synchronization failed for shard " + + shard + ":" + result.errorMessage()); + _result = Result(TRI_ERROR_INTERNAL, errorMessage);; + } + } + } + } catch (std::exception const& e) { + auto const endTime = system_clock::now(); + std::stringstream error; + error << "synchronization of local shard '" << database << "/" << shard + << "' for central '" << database << "/" << planId << "' failed: " + << e.what() << timepointToString(endTime); + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error.str(); + _result.reset(TRI_ERROR_INTERNAL, e.what()); + return false; + } + } catch (std::exception const& e) { + LOG_TOPIC(WARN, Logger::MAINTENANCE) + << "action " << _description << " failed with exception " << e.what(); + _result.reset(TRI_ERROR_INTERNAL, e.what()); + return false; + } + + // Tell others that we are done: + auto const endTime = system_clock::now(); + LOG_TOPIC(INFO, Logger::MAINTENANCE) + << "synchronizeOneShard: done, " << database << "/" << shard << ", " + << database << "/" << planId << ", started: " + << timepointToString(startTime) << ", ended: " << timepointToString(endTime); + + notify(); + return false;; + +} + +SynchronizeShard::~SynchronizeShard() {}; + diff --git a/arangod/Cluster/SynchronizeShard.h b/arangod/Cluster/SynchronizeShard.h new file mode 100644 index 0000000000..bb83f81c32 --- /dev/null +++ b/arangod/Cluster/SynchronizeShard.h @@ -0,0 +1,64 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_SYNCHRONIZE_SHARD_H +#define ARANGODB_MAINTENANCE_SYNCHRONIZE_SHARD_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { + +class MaintenanceAction; + +namespace maintenance { + +class SynchronizeShard : public ActionBase { + +public: + + SynchronizeShard(MaintenanceFeature&, ActionDescription const& d); + + virtual ~SynchronizeShard(); + + virtual bool first() override; + +private: + arangodb::Result getReadLock( + std::string const& endpoint, std::string const& database, + std::string const& collection, std::string const& clientId, uint64_t rlid, + double timeout = 300.0); + + arangodb::Result startReadLockOnLeader( + std::string const& endpoint, std::string const& database, + std::string const& collection, std::string const& clientId, uint64_t& rlid, + double timeout = 300.0); + +}; + +}} + +#endif diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index e4fe437fd9..0a0d6365e1 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index 2ccacb6a19..459a85c88f 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/TraverserEngineRegistry.cpp b/arangod/Cluster/TraverserEngineRegistry.cpp index 963e8c63df..b2b5b509ae 100644 --- a/arangod/Cluster/TraverserEngineRegistry.cpp +++ b/arangod/Cluster/TraverserEngineRegistry.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/TraverserEngineRegistry.h b/arangod/Cluster/TraverserEngineRegistry.h index 2e9603eb9a..9437bec4e2 100644 --- a/arangod/Cluster/TraverserEngineRegistry.h +++ b/arangod/Cluster/TraverserEngineRegistry.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/UpdateCollection.cpp b/arangod/Cluster/UpdateCollection.cpp new file mode 100644 index 0000000000..592b7a3d45 --- /dev/null +++ b/arangod/Cluster/UpdateCollection.cpp @@ -0,0 +1,163 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#include "UpdateCollection.h" + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/FollowerInfo.h" +#include "Utils/DatabaseGuard.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/Methods/Collections.h" +#include "VocBase/Methods/Databases.h" + + +using namespace arangodb; +using namespace arangodb::application_features; +using namespace arangodb::maintenance; +using namespace arangodb::methods; + +UpdateCollection::UpdateCollection( + MaintenanceFeature& feature, ActionDescription const& desc) : + ActionBase(feature, desc) { + + std::stringstream error; + + if (!desc.has(COLLECTION)) { + error << "collection must be specified. "; + } + TRI_ASSERT(desc.has(COLLECTION)); + + if (!desc.has(DATABASE)) { + error << "database must be specified. "; + } + TRI_ASSERT(desc.has(DATABASE)); + + if (!desc.has(LEADER)) { + error << "leader must be stecified. "; + } + TRI_ASSERT(desc.has(LEADER)); + + if (!desc.has(LOCAL_LEADER)) { + error << "local leader must be stecified. "; + } + TRI_ASSERT(desc.has(LOCAL_LEADER)); + + if (!error.str().empty()) { + LOG_TOPIC(ERR, Logger::MAINTENANCE) << "UpdateCollection: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + setState(FAILED); + } + +} + +void handleLeadership( + LogicalCollection& collection, std::string const& localLeader, + std::string const& plannedLeader) { + + auto& followers = collection.followers(); + + if (plannedLeader.empty()) { // Planned to lead + if (!localLeader.empty()) { // We were not leader, assume leadership + followers->setTheLeader(std::string()); + followers->clear(); + } else { + // If someone (the Supervision most likely) has thrown + // out a follower from the plan, then the leader + // will not notice until it fails to replicate an operation + // to the old follower. This here is to drop such a follower + // from the local list of followers. Will be reported + // to Current in due course. This is not needed for + // correctness but is a performance optimization. + } + } else { // Planned to follow + if (localLeader.empty()) { + // Note that the following does not delete the follower list + // and that this is crucial, because in the planned leader + // resign case, updateCurrentForCollections will report the + // resignation together with the old in-sync list to the + // agency. If this list would be empty, then the supervision + // would be very angry with us! + followers->setTheLeader(plannedLeader); + } + // Note that if we have been a follower to some leader + // we do not immediately adjust the leader here, even if + // the planned leader differs from what we have set locally. + // The setting must only be adjusted once we have + // synchronized with the new leader and negotiated + // a leader/follower relationship! + } + +} + + +UpdateCollection::~UpdateCollection() {}; + +bool UpdateCollection::first() { + + auto const& database = _description.get(DATABASE); + auto const& collection = _description.get(COLLECTION); + auto const& plannedLeader = _description.get(LEADER); + auto const& localLeader = _description.get(LOCAL_LEADER); + auto const& props = properties(); + + try { + + DatabaseGuard guard(database); + auto vocbase = &guard.database(); + + Result found = methods::Collections::lookup( + vocbase, collection, [&](LogicalCollection& coll) { + LOG_TOPIC(DEBUG, Logger::MAINTENANCE) + << "Updating local collection " + collection; + + // We adjust local leadership, note that the planned + // resignation case is not handled here, since then + // ourselves does not appear in shards[shard] but only + // "_" + ourselves. + handleLeadership(coll, localLeader, plannedLeader); + _result = Collections::updateProperties(&coll, props); + }); + + if (found.fail()) { + std::stringstream error; + error << "failed to lookup local collection " << collection + << "in database " + database; + LOG_TOPIC(ERR, Logger::MAINTENANCE) << error.str(); + _result = actionError(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, error.str()); + return false; + } + } catch (std::exception const& e) { + std::stringstream error; + error << "action " << _description << " failed with exception " << e.what(); + LOG_TOPIC(WARN, Logger::MAINTENANCE) << "UpdateCollection: " << error.str(); + _result.reset(TRI_ERROR_INTERNAL, error.str()); + return false; + } + + notify(); + return false; + +} diff --git a/arangod/Cluster/UpdateCollection.h b/arangod/Cluster/UpdateCollection.h new file mode 100644 index 0000000000..d4fbe313a3 --- /dev/null +++ b/arangod/Cluster/UpdateCollection.h @@ -0,0 +1,50 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_MAINTENANCE_UPDATE_COLLECTION_H +#define ARANGODB_MAINTENANCE_UPDATE_COLLECTION_H + +#include "ActionBase.h" +#include "ActionDescription.h" + +#include + +namespace arangodb { +namespace maintenance { + +class UpdateCollection : public ActionBase { + +public: + + UpdateCollection(MaintenanceFeature&, ActionDescription const&); + + virtual ~UpdateCollection(); + + virtual bool first() override final; + +}; + +}} + +#endif diff --git a/arangod/Cluster/v8-cluster.cpp b/arangod/Cluster/v8-cluster.cpp index be773142a5..16d429b72a 100644 --- a/arangod/Cluster/v8-cluster.cpp +++ b/arangod/Cluster/v8-cluster.cpp @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/Cluster/v8-cluster.h b/arangod/Cluster/v8-cluster.h index 250ba13e4e..5479073b1d 100644 --- a/arangod/Cluster/v8-cluster.h +++ b/arangod/Cluster/v8-cluster.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/arangod/GeneralServer/GeneralServerFeature.cpp b/arangod/GeneralServer/GeneralServerFeature.cpp index bcda2e80d2..01c5bbaae6 100644 --- a/arangod/GeneralServer/GeneralServerFeature.cpp +++ b/arangod/GeneralServer/GeneralServerFeature.cpp @@ -32,6 +32,7 @@ #include "Basics/StringUtils.h" #include "Cluster/AgencyCallbackRegistry.h" #include "Cluster/ClusterFeature.h" +#include "Cluster/MaintenanceRestHandler.h" #include "Cluster/RestAgencyCallbacksHandler.h" #include "Cluster/RestClusterHandler.h" #include "Cluster/TraverserEngineRegistry.h" @@ -457,6 +458,9 @@ void GeneralServerFeature::defineHandlers() { traverserEngineRegistry); // And now some handlers which are registered in both /_api and /_admin + _handlerFactory->addHandler( + "/_admin/actions", RestHandlerCreator::createNoData); + _handlerFactory->addPrefixHandler( "/_api/job", RestHandlerCreator::createData< AsyncJobManager*>, diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index 7f9bcecc96..74cbc741e7 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -613,6 +613,7 @@ int MMFilesCollection::close() { { // We also have to unload the indexes. + READ_LOCKER(guard, _indexesLock); /// TODO - DEADLOCK!?!? WRITE_LOCKER(writeLocker, _dataLock); for (auto& idx : _indexes) { idx->unload(); @@ -683,7 +684,7 @@ int MMFilesCollection::sealDatafile(MMFilesDatafile* datafile, std::string dname("datafile-" + std::to_string(datafile->fid()) + ".db"); std::string filename = arangodb::basics::FileUtils::buildFilename(path(), dname); - + LOG_TOPIC(TRACE, arangodb::Logger::DATAFILES) << "closing and renaming journal file '" << datafile->getName() << "'"; @@ -1552,6 +1553,7 @@ void MMFilesCollection::fillIndex( uint32_t MMFilesCollection::indexBuckets() const { return _indexBuckets; } int MMFilesCollection::fillAllIndexes(transaction::Methods* trx) { + READ_LOCKER(guard, _indexesLock); return fillIndexes(trx, _indexes); } @@ -1829,7 +1831,7 @@ void MMFilesCollection::open(bool ignoreErrors) { if (!engine->inRecovery()) { // build the index structures, and fill the indexes - fillIndexes(&trx, _indexes); + fillAllIndexes(&trx); } // successfully opened collection. now adjust version number @@ -1916,7 +1918,7 @@ Result MMFilesCollection::read(transaction::Methods* trx, VPackSlice const& key, } } TRI_DEFER(if (lock) { unlockRead(useDeadlockDetector, trx->state()); }); - + Result res = lookupDocument(trx, key, result); if (res.fail()) { return res; @@ -1985,10 +1987,10 @@ bool MMFilesCollection::readDocumentConditional( void MMFilesCollection::prepareIndexes(VPackSlice indexesSlice) { TRI_ASSERT(indexesSlice.isArray()); - + bool foundPrimary = false; bool foundEdge = false; - + for (auto const& it : VPackArrayIterator(indexesSlice)) { auto const& s = it.get(arangodb::StaticStrings::IndexType); @@ -2001,15 +2003,17 @@ void MMFilesCollection::prepareIndexes(VPackSlice indexesSlice) { } } - for (auto const& idx : _indexes) { - if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { - foundPrimary = true; - } else if (TRI_COL_TYPE_EDGE == _logicalCollection.type() && - idx->type() == Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX) { - foundEdge = true; + {READ_LOCKER(guard, _indexesLock); + for (auto const& idx : _indexes) { + if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { + foundPrimary = true; + } else if (TRI_COL_TYPE_EDGE == _logicalCollection.type() && + idx->type() == Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX) { + foundEdge = true; + } } } - + std::vector> indexes; StorageEngine* engine = EngineSelectorFeature::ENGINE; @@ -2031,24 +2035,27 @@ void MMFilesCollection::prepareIndexes(VPackSlice indexesSlice) { } } - TRI_ASSERT(!_indexes.empty()); - if (_indexes[0]->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX || - (TRI_COL_TYPE_EDGE == _logicalCollection.type() && - (_indexes.size() < 2 || _indexes[1]->type() != Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX))) { -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - for (auto const& it : _indexes) { - LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "- " << it.get(); - } -#endif - std::string errorMsg("got invalid indexes for collection '"); + {READ_LOCKER(guard, _indexesLock); + TRI_ASSERT(!_indexes.empty()); + if (_indexes[0]->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX || + (TRI_COL_TYPE_EDGE == _logicalCollection.type() && + (_indexes.size() < 2 || _indexes[1]->type() != Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX))) { + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE + for (auto const& it : _indexes) { + LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "- " << it.get(); + } + #endif + std::string errorMsg("got invalid indexes for collection '"); - errorMsg.append(_logicalCollection.name()); - errorMsg.push_back('\''); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, errorMsg); + errorMsg.append(_logicalCollection.name()); + errorMsg.push_back('\''); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, errorMsg); + } } #ifdef ARANGODB_ENABLE_MAINTAINER_MODE { + READ_LOCKER(guard, _indexesLock); bool foundPrimary = false; for (auto const& it : _indexes) { @@ -2085,12 +2092,14 @@ std::shared_ptr MMFilesCollection::lookupIndex( std::string tmp = value.copyString(); arangodb::Index::IndexType const type = arangodb::Index::type(tmp.c_str()); - for (auto const& idx : _indexes) { - if (idx->type() == type) { - // Only check relevant indices - if (idx->matchesDefinition(info)) { - // We found an index for this definition. - return idx; + {READ_LOCKER(guard, _indexesLock); + for (auto const& idx : _indexes) { + if (idx->type() == type) { + // Only check relevant indices + if (idx->matchesDefinition(info)) { + // We found an index for this definition. + return idx; + } } } } @@ -2100,7 +2109,7 @@ std::shared_ptr MMFilesCollection::lookupIndex( std::shared_ptr MMFilesCollection::createIndex(transaction::Methods* trx, VPackSlice const& info, bool& created) { - // TODO Get LOCK for the vocbase + auto idx = lookupIndex(info); if (idx != nullptr) { created = false; @@ -2225,6 +2234,9 @@ int MMFilesCollection::saveIndex(transaction::Methods* trx, } bool MMFilesCollection::addIndex(std::shared_ptr idx) { + + WRITE_LOCKER(guard, _indexesLock); + auto const id = idx->id(); for (auto const& it : _indexes) { if (it->id() == id) { @@ -2292,12 +2304,14 @@ int MMFilesCollection::restoreIndex(transaction::Methods* trx, TRI_UpdateTickServer(newIdx->id()); - auto const id = newIdx->id(); - for (auto& it : _indexes) { - if (it->id() == id) { - // index already exists - idx = it; - return TRI_ERROR_NO_ERROR; + {READ_LOCKER(gurad, _indexesLock); + auto const id = newIdx->id(); + for (auto& it : _indexes) { + if (it->id() == id) { + // index already exists + idx = it; + return TRI_ERROR_NO_ERROR; + } } } @@ -2373,6 +2387,8 @@ bool MMFilesCollection::dropIndex(TRI_idx_iid_t iid) { /// @brief removes an index by id bool MMFilesCollection::removeIndex(TRI_idx_iid_t iid) { + WRITE_LOCKER(guard, _indexesLock); + size_t const n = _indexes.size(); for (size_t i = 0; i < n; ++i) { @@ -2752,6 +2768,7 @@ void MMFilesCollection::truncate(transaction::Methods* trx, }; primaryIdx->invokeOnAllElementsForRemoval(callback); + READ_LOCKER(guard, _indexesLock); auto indexes = _indexes; size_t const n = indexes.size(); @@ -3063,6 +3080,8 @@ Result MMFilesCollection::insertSecondaryIndexes( Result result; + READ_LOCKER(guard, _indexesLock); + auto indexes = _indexes; size_t const n = indexes.size(); @@ -3108,8 +3127,8 @@ Result MMFilesCollection::deleteSecondaryIndexes( Result result; - // TODO FIXME - auto indexes = _logicalCollection.getIndexes(); + READ_LOCKER(guard, _indexesLock); + auto indexes = _indexes; size_t const n = indexes.size(); for (size_t i = 1; i < n; ++i) { @@ -3280,7 +3299,7 @@ Result MMFilesCollection::update( options.mergeObjects, options.keepNull, *builder.get(), options.isRestore, revisionId); - if (res.fail()) { + if (res.fail()) { return res; } @@ -3341,7 +3360,7 @@ Result MMFilesCollection::update( operation.revert(trx); } else { result.setManaged(newDoc.begin(), documentId); - + if (options.waitForSync) { // store the tick that was used for writing the new document resultMarkerTick = operation.tick(); @@ -3414,7 +3433,7 @@ Result MMFilesCollection::replace( transaction::BuilderLeaser builder(trx); res = newObjectForReplace(trx, oldDoc, newSlice, isEdgeCollection, *builder.get(), options.isRestore, revisionId); - + if (res.fail()) { return res; } diff --git a/arangod/RestServer/arangod.cpp b/arangod/RestServer/arangod.cpp index c791c86bdf..0dadc63c1b 100644 --- a/arangod/RestServer/arangod.cpp +++ b/arangod/RestServer/arangod.cpp @@ -60,6 +60,7 @@ #include "Cache/CacheManagerFeature.h" #include "Cluster/ClusterFeature.h" #include "Cluster/EngineEqualityCheckFeature.h" +#include "Cluster/MaintenanceFeature.h" #include "Cluster/ReplicationTimeoutFeature.h" #include "GeneralServer/AuthenticationFeature.h" #include "GeneralServer/GeneralServerFeature.h" @@ -188,6 +189,7 @@ static int runServer(int argc, char** argv, ArangoGlobalContext &context) { server.addFeature(new LockfileFeature(server)); server.addFeature(new LoggerBufferFeature(server)); server.addFeature(new LoggerFeature(server, true)); + server.addFeature(new MaintenanceFeature(server)); server.addFeature(new MaxMapCountFeature(server)); server.addFeature(new NonceFeature(server)); server.addFeature(new PageSizeFeature(server)); diff --git a/arangod/Transaction/Methods.cpp b/arangod/Transaction/Methods.cpp index 5b3c8cd18b..a0a3d1d186 100644 --- a/arangod/Transaction/Methods.cpp +++ b/arangod/Transaction/Methods.cpp @@ -1667,8 +1667,11 @@ OperationResult transaction::Methods::insertLocal( res = collection->replace( this, value, documentResult, options , resultMarkerTick, needsLock, previousRevisionId , previousDocumentResult); - if(res.ok()){ - revisionId = TRI_ExtractRevisionId(VPackSlice(documentResult.vpack())); + if(res.ok() && !options.silent){ + // If we are silent, then revisionId will not be looked at further + // down. In the silent case, documentResult is empty, so nobody + // must actually look at it! + revisionId = TRI_ExtractRevisionId(VPackSlice(documentResult.vpack())); } } diff --git a/js/actions/api-cluster.js b/js/actions/api-cluster.js index 8a9506634f..6cf4a28bc5 100644 --- a/js/actions/api-cluster.js +++ b/js/actions/api-cluster.js @@ -36,8 +36,6 @@ var wait = require("internal").wait; // var internal = require('internal'); var _ = require('lodash'); -var fetchKey = cluster.fetchKey; - actions.defineHttp({ url: '_admin/cluster/removeServer', allowUseDatabase: true, @@ -469,14 +467,12 @@ actions.defineHttp({ return; } - /* remove? timeout not used var timeout = 60.0; try { if (req.parameters.hasOwnProperty('timeout')) { timeout = Number(req.parameters.timeout); } } catch (e) {} - */ var clusterId; try { @@ -489,13 +485,26 @@ actions.defineHttp({ let agency = ArangoAgency.agency(); - var Health; - try { - Health = ArangoAgency.get('Supervision/Health', false, true).arango.Supervision.Health; - } catch (e1) { - actions.resultError(req, res, actions.HTTP_NOT_FOUND, 0, - 'Failed to retrieve supervision node from agency!'); - return; + var Health = {}; + var startTime = new Date(); + while (true) { + try { + Health = ArangoAgency.get('Supervision/Health', false, true).arango.Supervision.Health; + } catch (e1) { + actions.resultError(req, res, actions.HTTP_NOT_FOUND, 0, + 'Failed to retrieve supervision node from agency!'); + return; + } + if (Object.keys(Health).length !== 0) { + break; + } + if (new Date() - startTime > timeout * 1000) { // milliseconds + actions.resultError(req, res, actions.HTTP_NOT_FOUND, 0, + 'Failed to get health status from agency in ' + timeout + ' seconds.'); + return; + } + console.warn("/_api/cluster/health not ready yet, retrying..."); + require("internal").wait(0.5); } Health = Object.entries(Health).reduce((Health, [serverId, struct]) => { diff --git a/js/client/modules/@arangodb/testsuites/resilience.js b/js/client/modules/@arangodb/testsuites/resilience.js index 0a5fc64e06..2acd0e71f1 100644 --- a/js/client/modules/@arangodb/testsuites/resilience.js +++ b/js/client/modules/@arangodb/testsuites/resilience.js @@ -28,7 +28,6 @@ const functionsDocumentation = { 'resilience': 'resilience tests', 'client_resilience': 'client resilience tests', - 'cluster_sync': 'cluster sync tests', 'active_failover': 'active failover tests' }; const optionsDocumentation = [ @@ -39,7 +38,6 @@ const tu = require('@arangodb/test-utils'); const testPaths = { 'resilience': ['js/server/tests/resilience'], 'client_resilience': ['js/client/tests/resilience'], - 'cluster_sync': ['js/server/tests/cluster-sync'], 'active_failover': ['js/client/tests/active-failover'] }; @@ -71,28 +69,6 @@ function clientResilience (options) { return tu.performTests(options, testCases, 'client_resilience', tu.runInArangosh); } -// ////////////////////////////////////////////////////////////////////////////// -// / @brief TEST: cluster_sync -// ////////////////////////////////////////////////////////////////////////////// - -function clusterSync (options) { - if (options.cluster) { - // may sound strange but these are actually pure logic tests - // and should not be executed on the cluster - return { - 'cluster_sync': { - 'status': true, - 'message': 'skipped because of cluster', - 'skipped': true - } - }; - } - let testCases = tu.scanTestPaths(testPaths.cluster_sync); - options.propagateInstanceInfo = true; - - return tu.performTests(options, testCases, 'cluster_sync', tu.runThere); -} - // ////////////////////////////////////////////////////////////////////////////// // / @brief TEST: active failover // ////////////////////////////////////////////////////////////////////////////// @@ -121,7 +97,6 @@ exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTest Object.assign(allTestPaths, testPaths); testFns['resilience'] = resilience; testFns['client_resilience'] = clientResilience; - testFns['cluster_sync'] = clusterSync; testFns['active_failover'] = activeFailover; for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; } for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); } diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index f5bccd71b6..d1cb7de280 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -346,7 +346,11 @@ "ERROR_SUPERVISION_GENERAL_FAILURE" : { "code" : 20501, "message" : "general supervision failure" }, "ERROR_DISPATCHER_IS_STOPPING" : { "code" : 21001, "message" : "dispatcher stopped" }, "ERROR_QUEUE_UNKNOWN" : { "code" : 21002, "message" : "named queue does not exist" }, - "ERROR_QUEUE_FULL" : { "code" : 21003, "message" : "named queue is full" } + "ERROR_QUEUE_FULL" : { "code" : 21003, "message" : "named queue is full" }, + "ERROR_ACTION_ALREADY_REGISTERED" : { "code" : 6001, "message" : "maintenance action already registered" }, + "ERROR_ACTION_OPERATION_UNABORTABLE" : { "code" : 6002, "message" : "this maintenance action cannot be stopped" }, + "ERROR_ACTION_UNFINISHED" : { "code" : 6003, "message" : "maintenance action still processing" }, + "ERROR_NO_SUCH_ACTION" : { "code" : 6004, "message" : "no such maintenance action" } }; // For compatibility with <= 3.3 diff --git a/js/common/tests/http/document-babies-spec.js b/js/common/tests/http/document-babies-spec.js index f29b928599..b13cf4f113 100644 --- a/js/common/tests/http/document-babies-spec.js +++ b/js/common/tests/http/document-babies-spec.js @@ -39,15 +39,10 @@ const uniqueCode = ERRORS.ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED.code; const invalidCode = ERRORS.ERROR_ARANGO_DOCUMENT_TYPE_INVALID.code; const keyBadCode = ERRORS.ERROR_ARANGO_DOCUMENT_KEY_BAD.code; -let clreq = - request.get('/_api/cluster/endpoints', {auth:{username:"root",password:""}}); -var cluster = (clreq.statusCode === 200); -var createOptions = { waitForSync: false }; -if (cluster) { - createOptions['replicationFactor'] = 2; - createOptions['numberOfShards'] = 2; - createOptions['waitForReplication'] = true; -} +var createOptions = { waitForSync: false, + replicationFactor: 2, + numberOfShards: 2 }; +// Note that the cluster options are ignored in single server let endpoint = {}; @@ -57,7 +52,8 @@ describe('babies collection document', function () { beforeEach(function () { db._drop(cn); - collection = db._create(cn, createOptions); + collection = db._create(cn, createOptions, "document", + {waitForSyncReplication: true}); }); afterEach(function () { diff --git a/js/server/modules/@arangodb/cluster.js b/js/server/modules/@arangodb/cluster.js index d6ee5637a7..54f6840d91 100644 --- a/js/server/modules/@arangodb/cluster.js +++ b/js/server/modules/@arangodb/cluster.js @@ -30,60 +30,12 @@ var console = require('console'); var arangodb = require('@arangodb'); -var ArangoCollection = arangodb.ArangoCollection; var ArangoError = arangodb.ArangoError; var errors = require("internal").errors; -var request = require('@arangodb/request').clusterRequest; var wait = require('internal').wait; var isEnterprise = require('internal').isEnterprise(); var _ = require('lodash'); -const curDatabases = '/arango/Current/Databases/'; -const curCollections = '/arango/Current/Collections/'; -const curVersion = '/arango/Current/Version'; -const agencyOperations = { - 'delete' : {'op' : 'delete'}, - 'increment' : {'op' : 'increment'} -}; - -// good enough isEqual which does not depend on equally sorted objects as _.isEqual -var isEqual = function(object, other) { - if (typeof object !== typeof other) { - return false; - } - - if (typeof object !== 'object') { - // explicitly use non strict equal - // eslint-disable-next-line eqeqeq - return object == other; - } - - if (Array.isArray(object)) { - if (object.length !== other.length) { - return false; - } - return object.every((value, index) => { - return isEqual(value, other[index]); - }); - } else if (object === null) { - return other === null; - } - - let myKeys = Object.keys(object); - let otherKeys = Object.keys(other); - - if (myKeys.length !== otherKeys.length) { - return false; - } - - return myKeys.every(key => { - if (!isEqual(object[key], other[key])) { - return false; - } - return true; - }); -}; - var endpointToURL = function (endpoint) { if (endpoint.substr(0, 6) === 'ssl://') { return 'https://' + endpoint.substr(6); @@ -95,1587 +47,6 @@ var endpointToURL = function (endpoint) { return 'http' + endpoint.substr(pos); }; -// ///////////////////////////////////////////////////////////////////////////// -// / @brief fetch a READ lock on a collection and will time out after a -// / number of seconds -// ///////////////////////////////////////////////////////////////////////////// - -function startReadLockOnLeader (endpoint, database, collName, timeout) { - var url = endpointToURL(endpoint) + '/_db/' + database; - var r = request({ url: url + '/_api/replication/holdReadLockCollection', - method: 'GET' }); - if (r.status !== 200) { - console.topic("heartbeat=error", 'startReadLockOnLeader: Could not get ID for shard', - collName, r); - return false; - } - try { - r = JSON.parse(r.body); - } catch (err) { - console.topic("heartbeat=error", 'startReadLockOnLeader: Bad response body from', - '/_api/replication/holdReadLockCollection', r, - JSON.stringify(err)); - return false; - } - const id = r.id; - - var body = { 'id': id, 'collection': collName, 'ttl': timeout }; - request({ url: url + '/_api/replication/holdReadLockCollection', - body: JSON.stringify(body), - method: 'POST', headers: {'x-arango-async': true} }); - // Intentionally do not look at the outcome, even in case of an error - // we must make sure that the read lock on the leader is not active! - // This is done automatically below. - - var count = 0; - while (++count < 20) { // wait for some time until read lock established: - // Now check that we hold the read lock: - r = request({ url: url + '/_api/replication/holdReadLockCollection', - body: JSON.stringify(body), method: 'PUT' }); - if (r.status === 200) { - let ansBody = {}; - try { - ansBody = JSON.parse(r.body); - } catch (err) { - } - if (ansBody.lockHeld) { - return id; - } else { - console.topic('heartbeat=debug', 'startReadLockOnLeader: Lock not yet acquired...'); - } - } else { - console.topic('heartbeat=debug', 'startReadLockOnLeader: Do not see read lock yet...'); - } - wait(0.5, false); - } - console.topic('heartbeat=error', 'startReadLockOnLeader: giving up'); - try { - r = request({ url: url + '/_api/replication/holdReadLockCollection', - body: JSON.stringify({'id': id}), method: 'DELETE' }); - } catch (err2) { - console.topic('heartbeat=error', 'startReadLockOnLeader: expection in cancel:', - JSON.stringify(err2)); - } - if (r.status !== 200) { - console.topic('heartbeat=error', 'startReadLockOnLeader: cancelation error for shard', - collName, r); - } - return false; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief cancel read lock -// ///////////////////////////////////////////////////////////////////////////// - -function cancelReadLockOnLeader (endpoint, database, lockJobId) { - // Note that we always use the _system database here because the actual - // database might be gone already on the leader and we need to cancel - // the read lock under all circumstances. - var url = endpointToURL(endpoint) + '/_db/_system' + - '/_api/replication/holdReadLockCollection'; - var r; - var body = {'id': lockJobId}; - try { - r = request({url, body: JSON.stringify(body), method: 'DELETE' }); - } catch (e) { - console.topic('heartbeat=error', 'cancelReadLockOnLeader: exception caught:', e); - return false; - } - if (r.status !== 200) { - console.topic('heartbeat=error', 'cancelReadLockOnLeader: error', lockJobId, r.status, - r.message, r.body, r.json); - return false; - } - console.topic('heartbeat=debug', 'cancelReadLockOnLeader: success'); - return true; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief cancel barrier from sync -// ///////////////////////////////////////////////////////////////////////////// - -function cancelBarrier (endpoint, database, barrierId) { - if (barrierId <= 0) { - return true; - } - var url = endpointToURL(endpoint) + '/_db/' + database + - '/_api/replication/barrier/' + barrierId; - var r = request({url, method: 'DELETE' }); - if (r.status !== 200 && r.status !== 204) { - console.topic('heartbeat=error', 'CancelBarrier: error', r); - return false; - } - console.topic('heartbeat=debug', 'cancelBarrier: success'); - return true; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief tell leader that we are in sync -// ///////////////////////////////////////////////////////////////////////////// - -function addShardFollower (endpoint, database, shard, lockJobId) { - console.topic('heartbeat=debug', 'addShardFollower: tell the leader to put us into the follower list...'); - var url = endpointToURL(endpoint) + '/_db/' + database + - '/_api/replication/addFollower'; - let db = require('internal').db; - var body; - if (lockJobId === undefined) { - body = {followerId: ArangoServerState.id(), shard, - checksum: db._collection(shard).count() + ''}; - } else { - body = {followerId: ArangoServerState.id(), shard, - checksum: db._collection(shard).count() + '', - readLockId: lockJobId}; - } - var r = request({url, body: JSON.stringify(body), method: 'PUT'}); - if (r.status !== 200) { - if (lockJobId !== undefined) { - console.topic('heartbeat=error', "addShardFollower: could not add us to the leader's follower list.", r); - } else { - console.topic('heartbeat=debug', "addShardFollower: could not add us to the leader's follower list with shortcut.", r); - } - return false; - } - return true; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief tell leader that we are stop following -// ///////////////////////////////////////////////////////////////////////////// - -function removeShardFollower (endpoint, database, shard) { - console.topic('heartbeat=debug', 'removeShardFollower: tell the leader to take us off the follower list...'); - var url = endpointToURL(endpoint) + '/_db/' + database + - '/_api/replication/removeFollower'; - var body = {followerId: ArangoServerState.id(), shard}; - var r = request({url, body: JSON.stringify(body), method: 'PUT'}); - if (r.status !== 200) { - console.topic('heartbeat=error', "removeShardFollower: could not remove us from the leader's follower list: ", r.status, r.body); - return false; - } - return true; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief -// ///////////////////////////////////////////////////////////////////////////// - -function fetchKey(structure, ...path) { - let current = structure; - do { - let key = path.shift(); - if (typeof current !== 'object' || !current.hasOwnProperty(key)) { - return undefined; - } - current = current[key]; - } while (path.length > 0); - return current; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief return a shardId => server map -// ///////////////////////////////////////////////////////////////////////////// - -function getShardMap (plannedCollections) { - var shardMap = { }; - - var database; - - for (database in plannedCollections) { - if (plannedCollections.hasOwnProperty(database)) { - var collections = plannedCollections[database]; - var collection; - - for (collection in collections) { - if (collections.hasOwnProperty(collection)) { - var shards = collections[collection].shards; - var shard; - - for (shard in shards) { - if (shards.hasOwnProperty(shard)) { - shardMap[shard] = shards[shard]; - } - } - } - } - } - } - - return shardMap; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief return a hash with the local databases -// ///////////////////////////////////////////////////////////////////////////// - -function getLocalDatabases () { - let result = { }; - let db = require('internal').db; - let curDb = db._name(); - try { - db._databases().forEach(function (database) { - db._useDatabase(database); - result[database] = { name: database, id: db._id() }; - }); - } finally { - db._useDatabase(curDb); - } - return result; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief return a hash with the local collections -// ///////////////////////////////////////////////////////////////////////////// - -function getLocalCollections () { - var result = { }; - var db = require('internal').db; - - db._collections().forEach(function (collection) { - var name = collection.name(); - - if (name.substr(0, 1) !== '_') { - var data = { - id: collection._id, - name: name, - type: collection.type(), - status: collection.status(), - planId: collection.planId(), - theLeader: collection.getLeader() - }; - - // merge properties - var properties = collection.properties(); - var p; - for (p in properties) { - if (properties.hasOwnProperty(p)) { - data[p] = properties[p]; - } - } - - result[name] = data; - } - }); - - return result; -} - -function organizeLeaderResign (database, collId, shardName) { - console.topic('heartbeat=info', "trying to withdraw as leader of shard '%s/%s' of '%s/%s'", - database, shardName, database, collId); - // This starts a write transaction, just to wait for any ongoing - // write transaction on this shard to terminate. We will then later - // report to Current about this resignation. If a new write operation - // starts in the meantime (which is unlikely, since no coordinator that - // has seen the _ will start a new one), it is doomed, and we ignore the - // problem, since similar problems can arise in failover scenarios anyway. - try { - // we know the shard exists locally! - var db = require('internal').db; - db._collection(shardName).setTheLeader("LEADER_NOT_YET_KNOWN"); // resign - // Note that it is likely that we will be a follower for this shard - // with another leader in due course. However, we do not know the - // name of the new leader yet. This setting will make us a follower - // for now but we will not accept any replication operation from any - // leader, until we have negotiated a deal with it. Then the actual - // name of the leader will be set. - db._executeTransaction( - { 'collections': { 'write': [shardName] }, - 'action': function () { } - }); - } catch (x) { - console.topic('heartbeat=error', 'exception thrown when resigning:', x); - } -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief lock key space -// ///////////////////////////////////////////////////////////////////////////// - -function lockSyncKeyspace () { - while (!global.KEY_SET_CAS('shardSynchronization', 'lock', 1, null)) { - wait(0.001, false); - } -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief unlock key space -// ///////////////////////////////////////////////////////////////////////////// - -function unlockSyncKeyspace () { - global.KEY_SET('shardSynchronization', 'lock', null); -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief launch a scheduled job if needed -// ///////////////////////////////////////////////////////////////////////////// - -function tryLaunchJob () { - const registerTask = require('internal').registerTask; - var isStopping = require('internal').isStopping; - if (isStopping()) { - return; - } - var doCleanup = false; - lockSyncKeyspace(); - try { - var jobs = global.KEYSPACE_GET('shardSynchronization'); - if (jobs.running === null) { - var shards = Object.keys(jobs.scheduled).sort(); - if (shards.length > 0) { - var done = false; - while (!done) { - var jobInfo = jobs.scheduled[shards[0]]; - try { - registerTask({ - database: jobInfo.database, - params: {database: jobInfo.database, shard: jobInfo.shard, - planId: jobInfo.planId, leader: jobInfo.leader}, - command: function (params) { - require('@arangodb/cluster').synchronizeOneShard( - params.database, params.shard, params.planId, params.leader); - }}); - done = true; - global.KEY_SET('shardSynchronization', 'running', jobInfo); - console.topic('heartbeat=debug', 'tryLaunchJob: have launched job', jobInfo); - delete jobs.scheduled[shards[0]]; - global.KEY_SET('shardSynchronization', 'scheduled', jobs.scheduled); - } catch (err) { - if (err.errorNum === errors.ERROR_ARANGO_DATABASE_NOT_FOUND.code) { - doCleanup = true; - done = true; - } - if (!require('internal').isStopping()) { - console.topic('heartbeat=debug', 'Could not registerTask for shard synchronization.', - err); - wait(1.0, false); - } else { - doCleanup = true; - done = true; - } - } - } - } - } - } - finally { - unlockSyncKeyspace(); - } - if (doCleanup) { // database was deleted - global.KEYSPACE_REMOVE("shardSynchronization"); - } -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief synchronize one shard, this is run as a V8 task -// ///////////////////////////////////////////////////////////////////////////// - -function synchronizeOneShard (database, shard, planId, leader) { - // synchronize this shard from the leader - // this function will throw if anything goes wrong - - var startTime = new Date(); - var isStopping = require('internal').isStopping; - var ourselves = global.ArangoServerState.id(); - - function terminateAndStartOther () { - lockSyncKeyspace(); - try { - global.KEY_SET('shardSynchronization', 'running', null); - } - finally { - unlockSyncKeyspace(); - } - tryLaunchJob(); // start a new one if needed - } - - // First wait until the leader has created the shard (visible in - // Current in the Agency) or we or the shard have vanished from - // the plan: - while (true) { - if (isStopping()) { - terminateAndStartOther(); - return; - } - var planned = []; - try { - planned = global.ArangoClusterInfo.getCollectionInfo(database, planId) - .shards[shard]; - } catch (e) {} - if (!Array.isArray(planned) || - planned.indexOf(ourselves) <= 0 || - planned[0] !== leader) { - // Things have changed again, simply terminate: - terminateAndStartOther(); - let endTime = new Date(); - console.topic('heartbeat=debug', 'synchronizeOneShard: cancelled, %s/%s, %s/%s, started %s, ended %s', - database, shard, database, planId, startTime.toString(), endTime.toString()); - return; - } - var current = []; - try { - current = global.ArangoClusterInfo.getCollectionInfoCurrent( - database, planId, shard).servers; - } catch (e2) {} - if (current[0] === leader) { - if (current.indexOf(ourselves) === -1) { - break; // start synchronization work - } - // We are already there, this is rather strange, but never mind: - terminateAndStartOther(); - let endTime = new Date(); - console.topic('heartbeat=debug', 'synchronizeOneShard: already done, %s/%s, %s/%s, started %s, ended %s', - database, shard, database, planId, startTime.toString(), endTime.toString()); - return; - } - console.topic('heartbeat=debug', 'synchronizeOneShard: waiting for leader, %s/%s, %s/%s', - database, shard, database, planId); - wait(0.2, false); - } - - // Once we get here, we know that the leader is ready for sync, so - // we give it a try: - var ok = false; - const rep = require('@arangodb/replication'); - - var db = require("internal").db; - var ep = ArangoClusterInfo.getServerEndpoint(leader); - - if (db._collection(shard).count() === 0) { - // We try a short cut: - console.topic('heartbeat=debug', "synchronizeOneShard: trying short cut to synchronize local shard '%s/%s' for central '%s/%s'", database, shard, database, planId); - try { - let ok = addShardFollower(ep, database, shard); - if (ok) { - terminateAndStartOther(); - let endTime = new Date(); - console.topic('heartbeat=debug', 'synchronizeOneShard: shortcut worked, done, %s/%s, %s/%s, started: %s, ended: %s', - database, shard, database, planId, startTime.toString(), endTime.toString()); - db._collection(shard).setTheLeader(leader); - return; - } - } catch (dummy) { } - } - - console.topic('heartbeat=debug', "synchronizeOneShard: trying to synchronize local shard '%s/%s' for central '%s/%s'", database, shard, database, planId); - try { - // First once without a read transaction: - var sy; - if (isStopping()) { - throw 'server is shutting down'; - } - - // Mark us as follower for this leader such that we begin - // accepting replication operations, note that this is also - // used for the initial synchronization: - db._collection(shard).setTheLeader(leader); - - let startTime = new Date(); - sy = rep.syncCollection(shard, - { endpoint: ep, incremental: true, keepBarrier: true, - leaderId: leader, skipCreateDrop: true }); - let endTime = new Date(); - let longSync = false; - if (endTime - startTime > 5000) { - console.topic('heartbeat=warn', 'synchronizeOneShard: long call to syncCollection for shard', shard, JSON.stringify(sy), "start time: ", startTime.toString(), "end time: ", endTime.toString()); - longSync = true; - } - if (sy.error) { - console.topic('heartbeat=error', 'synchronizeOneShard: could not initially synchronize', - 'shard ', shard, sy); - throw 'Initial sync for shard ' + shard + ' failed'; - } else { - if (sy.collections.length === 0 || - sy.collections[0].name !== shard) { - if (longSync) { - console.topic('heartbeat=error', 'synchronizeOneShard: long sync, before cancelBarrier', - new Date().toString()); - } - cancelBarrier(ep, database, sy.barrierId); - if (longSync) { - console.topic('heartbeat=error', 'synchronizeOneShard: long sync, after cancelBarrier', - new Date().toString()); - } - throw 'Shard ' + shard + ' seems to be gone from leader!'; - } else { - // Now start a read transaction to stop writes: - var lockJobId = false; - try { - lockJobId = startReadLockOnLeader(ep, database, - shard, 300); - console.topic('heartbeat=debug', 'lockJobId:', lockJobId); - } catch (err1) { - console.topic('heartbeat=error', 'synchronizeOneShard: exception in startReadLockOnLeader:', err1, err1.stack); - } - finally { - cancelBarrier(ep, database, sy.barrierId); - } - if (lockJobId !== false) { - try { - var sy2 = REPLICATION_SYNCHRONIZE_FINALIZE({ - endpoint: ep, - database, - collection: shard, - leaderId: leader, - from: sy.lastLogTick, - requestTimeout: 60, - connectTimeout: 60 - }); - - try { - ok = addShardFollower(ep, database, shard, lockJobId); - } catch (err4) { - db._drop(shard, {isSystem: true }); - throw err4; - } - } catch (err3) { - ok = false; - console.topic('heartbeat=error', 'synchronizeOneshard: exception in', - 'syncCollectionFinalize:', err3); - } - finally { - if (!cancelReadLockOnLeader(ep, database, lockJobId)) { - console.topic('heartbeat=error', 'synchronizeOneShard: read lock has timed out', - 'for shard', shard); - ok = false; - } - } - } else { - console.topic('heartbeat=error', 'synchronizeOneShard: lockJobId was false for shard', - shard); - } - if (ok) { - console.topic('heartbeat=debug', 'synchronizeOneShard: synchronization worked for shard', - shard); - } else { - throw 'Did not work for shard ' + shard + '.'; - // just to log below in catch - } - } - } - } catch (err2) { - if (!isStopping()) { - let logLevel = 'error'; - // ignore failures of jobs where the database to sync has been removed on the leader - // {"errorNum":1400,"errorMessage":"cannot sync from remote endpoint: job not found on master at tcp://127.0.0.1:15179. last progress message was 'send batch finish command to url /_api/replication/batch/2103395': no response"} - if (err2 && (err2.errorNum === 1203 || err2.errorNum === 1400)) { - logLevel = 'debug'; - } else if (err2 && err2.errorNum === 1402 && err2.errorMessage.match(/HTTP 404/)) { - logLevel = 'debug'; - } - let endTime = new Date(); - console.topic('heartbeat='+logLevel, "synchronization of local shard '%s/%s' for central '%s/%s' failed: %s, started: %s, ended: %s", - database, shard, database, planId, JSON.stringify(err2), - startTime.toString(), endTime.toString()); - } - } - // Tell others that we are done: - terminateAndStartOther(); - let endTime = new Date(); - console.topic('heartbeat=debug', 'synchronizeOneShard: done, %s/%s, %s/%s, started: %s, ended: %s', - database, shard, database, planId, startTime.toString(), endTime.toString()); -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief schedule a shard synchronization -// ///////////////////////////////////////////////////////////////////////////// - -function scheduleOneShardSynchronization (database, shard, planId, leader) { - console.topic('heartbeat=debug', 'scheduleOneShardSynchronization:', database, shard, planId, - leader); - - lockSyncKeyspace(); - try { - var jobs = global.KEYSPACE_GET('shardSynchronization'); - if ((jobs.running !== null && jobs.running.shard === shard) || - jobs.scheduled.hasOwnProperty(shard)) { - console.topic('heartbeat=debug', 'task is already running or scheduled,', - 'ignoring scheduling request'); - return false; - } - - // If we reach this, we actually have to schedule a new task: - var jobInfo = { database, shard, planId, leader}; - jobs.scheduled[shard] = jobInfo; - global.KEY_SET('shardSynchronization', 'scheduled', jobs.scheduled); - console.topic('heartbeat=debug', 'scheduleOneShardSynchronization: have scheduled job', jobInfo); - } - finally { - unlockSyncKeyspace(); - } - return true; -} - -function createIndexes(collection, plannedIndexes) { - let existingIndexes = collection.getIndexes(); - - let localId = plannedIndex => { - return collection.name() + '/' + plannedIndex.id; - }; - - let clusterId = existingIndex => { - return existingIndex.split('/')[1]; - }; - - let findExisting = index => { - return existingIndexes.filter(existingIndex => { - return existingIndex.id === localId(index); - })[0]; - }; - let errors = plannedIndexes.reduce((errors, plannedIndex) => { - if (plannedIndex.type !== 'primary' && plannedIndex.type !== 'edge') { - if (findExisting(plannedIndex)) { - return errors; - } - try { - console.topic('heartbeat=debug', "creating index '%s/%s': %s", - collection._dbName, - collection.name(), - JSON.stringify(plannedIndex)); - collection.ensureIndex(plannedIndex); - } catch (e) { - errors[plannedIndex.id] = { - errorNum: e.errorNum, - errorMessage: e.errorMessage, - }; - }; - } - return errors; - }, {}); - - let indexesToDelete = existingIndexes.filter(index => { - if (index.type === 'primary' || index.type === 'edge') { - return false; - } - return plannedIndexes.filter(plannedIndex => { - return localId(plannedIndex) === index.id; - }).length === 0; - }); - - return indexesToDelete.reduce((errors, index) => { - console.topic('heartbeat=debug', "dropping index '%s/%s': %s", - collection._dbName, - collection.name(), - JSON.stringify(index)); - if (!collection.dropIndex(index)) { - errors[clusterId(index)] = { - errorNum: 4, - errorMessage: 'could not delete index locally', - }; - } - return errors; - }, errors); -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief executePlanForCollections -// ///////////////////////////////////////////////////////////////////////////// - -function executePlanForCollections(plannedCollections) { - let ourselves = global.ArangoServerState.id(); - let localErrors = {}; - - let db = require('internal').db; - db._useDatabase('_system'); - - let createShardError = function(errors, database, planId, shardName) { - if (Object.keys(errors).length > 0) { - let fullError = {}; - fullError[shardName] = { - info: {database, planId, shardName}, - }; - fullError[shardName] = Object.assign(fullError[shardName], errors); - return fullError; - } else { - return errors; - } - }; - - let localDatabases = getLocalDatabases(); - // Create shards in Plan that are not there locally: - Object.keys(plannedCollections).forEach(database => { - if (localDatabases.hasOwnProperty(database)) { - // switch into other database - db._useDatabase(database); - - try { - // iterate over collections of database - let localCollections = getLocalCollections(); - let collections = plannedCollections[database]; - - // diff the collections - Object.keys(collections).forEach(function (collectionName) { - let collectionInfo = collections[collectionName]; - let shards = collectionInfo.shards; - - collectionInfo.planId = collectionInfo.id; - localErrors = Object.keys(shards).reduce((errors, shardName) => { - let shardErrors = {}; - let plannedServers = shards[shardName]; - if (plannedServers.indexOf(ourselves) >= 0) { - let shouldBeLeader = plannedServers[0] === ourselves; - - let collectionStatus; - let collection; - if (!localCollections.hasOwnProperty(shardName)) { - // must create this shard - console.topic('heartbeat=debug', "creating local shard '%s/%s' for central '%s/%s'", - database, - shardName, - database, - collectionInfo.planId); - - let save = {id: collectionInfo.id, name: collectionInfo.name}; - delete collectionInfo.id; // must not - delete collectionInfo.name; - if (collectionInfo.keyOptions && - (collectionInfo.shardKeys.length !== 1 || collectionInfo.shardKeys[0] !== '_key')) { - // custom sharding... we must allow the coordinator to set a _key - collectionInfo.keyOptions.allowUserKeys = true; - } - if (collectionInfo.hasOwnProperty('globallyUniqueId')) { - console.warn('unexpected globallyUniqueId in %s', - JSON.stringify(collectionInfo)); - } - delete collectionInfo.globallyUniqueId; - if (collectionInfo.hasOwnProperty('objectId')) { - console.warn('unexpected objectId in %s', JSON.stringify(collectionInfo)); - } - delete collectionInfo.objectId; - try { - if (collectionInfo.type === ArangoCollection.TYPE_EDGE) { - db._createEdgeCollection(shardName, collectionInfo); - } else { - db._create(shardName, collectionInfo); - } - } catch (err2) { - shardErrors.collection = { - errorNum: err2.errorNum, - errorMessage: err2.errorMessage, - }; - console.topic('heartbeat=error', "creating local shard '%s/%s' for central '%s/%s' failed: %s", - database, - shardName, - database, - collectionInfo.planId, - JSON.stringify(err2)); - return Object.assign(errors, createShardError(shardErrors, database, collectionInfo.planId, shardName)); - } - collectionInfo.id = save.id; - collectionInfo.name = save.name; - collection = db._collection(shardName); - if (shouldBeLeader) { - collection.setTheLeader(""); // take power - } else { - collection.setTheLeader(plannedServers[0]); - } - collectionStatus = ArangoCollection.STATUS_LOADED; - } else { - collection = db._collection(shardName); - // We adjust local leadership, note that the planned - // resignation case is not handled here, since then - // ourselves does not appear in shards[shard] but only - // "_" + ourselves. We adjust local leadership, note - // that the planned resignation case is not handled - // here, since then ourselves does not appear in - // shards[shard] but only "_" + ourselves. See below - // under "Drop local shards" to see the proper handling - // of this case. Place is marked with *** in comments. - - if (shouldBeLeader) { - if (localCollections[shardName].theLeader !== "") { - collection.setTheLeader(""); // assume leadership - } else { - // If someone (the Supervision most likely) has thrown - // out a follower from the plan, then the leader - // will not notice until it fails to replicate an operation - // to the old follower. This here is to drop such a follower - // from the local list of followers. Will be reported - // to Current in due course. This is not needed for - // correctness but is a performance optimization. - let currentFollowers = collection.getFollowers(); - let plannedFollowers = plannedServers.slice(1); - let removedFollowers = currentFollowers.filter(follower => { - return plannedServers.indexOf(follower) === -1; - }); - removedFollowers.forEach(removedFollower => { - collection.removeFollower(removedFollower); - }); - } - } else { // !shouldBeLeader - if (localCollections[shardName].theLeader === "") { - // Note that the following does not delete the follower list - // and that this is crucial, because in the planned leader - // resign case, updateCurrentForCollections will report the - // resignation together with the old in-sync list to the - // agency. If this list would be empty, then the supervision - // would be very angry with us! - collection.setTheLeader(plannedServers[0]); - } - // Note that if we have been a follower to some leader - // we do not immediately adjust the leader here, even if - // the planned leader differs from what we have set locally. - // The setting must only be adjusted once we have - // synchronized with the new leader and negotiated - // a leader/follower relationship! - } - - collectionStatus = localCollections[shardName].status; - - // collection exists, now compare collection properties - let cmp = [ 'journalSize', 'waitForSync', 'doCompact', - 'indexBuckets', 'cacheEnabled' ]; - - let properties = cmp.reduce((obj, key) => { - if (localCollections[shardName].hasOwnProperty(key) && - localCollections[shardName][key] !== collectionInfo[key]) { - // property change - obj[key] = collectionInfo[key]; - } - return obj; - }, {}); - - if (Object.keys(properties).length > 0) { - console.topic('heartbeat=debug', "updating properties for local shard '%s/%s'", - database, - shardName); - - try { - collection.properties(properties); - } catch (err3) { - shardErrors.collection = { - errorNum: err3.errorNum, - errorMessage: err3.errorMessage, - }; - return Object.assign(errors, createShardError(shardErrors, database, collectionInfo.planId, shardName)); - } - } - } - - // Now check whether the status is OK: - if (collectionStatus !== collectionInfo.status) { - console.topic('heartbeat=debug', "detected status change for local shard '%s/%s'", - database, - shardName); - - if (collectionInfo.status === ArangoCollection.STATUS_UNLOADED) { - console.topic('heartbeat=debug', "unloading local shard '%s/%s'", - database, - shardName); - - collection.unload(); - } else if (collectionInfo.status === ArangoCollection.STATUS_LOADED) { - console.topic('heartbeat=debug', "loading local shard '%s/%s'", - database, - shardName); - collection.load(); - } - } - - let indexErrors = createIndexes(collection, collectionInfo.indexes || {}); - if (Object.keys(indexErrors).length > 0) { - shardErrors.indexErrors = indexErrors; - } - } - return Object.assign(errors, createShardError(shardErrors, database, collectionInfo.planId, shardName)); - }, localErrors); - }); - } catch(e) { - console.topic('heartbeat=error', "Got error executing plan", e, e.stack); - } finally { - // always return to previous database - db._useDatabase('_system'); - } - } - }); - // Drop local shards that do no longer exist in Plan: - let shardMap = getShardMap(plannedCollections); - - // iterate over all databases - Object.keys(localDatabases).forEach(database => { - let removeAll = !plannedCollections.hasOwnProperty(database); - - // switch into other database - db._useDatabase(database); - try { - // iterate over collections of database - let collections = getLocalCollections(); - - Object.keys(collections).forEach(collection => { - // found a local collection - // check if it is in the plan and we are responsible for it - if (removeAll || - !shardMap.hasOwnProperty(collection) || - shardMap[collection].indexOf(ourselves) === -1) { - - // May be we have been the leader and are asked to withdraw: *** - if (shardMap.hasOwnProperty(collection) && - shardMap[collection][0] === '_' + ourselves) { - if (collections[collection].theLeader === "") { - organizeLeaderResign(database, collections[collection].planId, - collection); - } - } else { - if (collections[collection].theLeader !== "") { - // Remove us from the follower list, this is a best - // effort: If an error occurs, this is no problem, since - // the leader will soon notice that the shard here is - // gone and will drop us automatically: - console.topic('heartbeat=debug', "removing local shard '%s/%s' of '%s/%s' from follower list", - database, collection, database, - collections[collection].planId); - let servers = shardMap[collection]; - if (servers !== undefined) { - let endpoint = ArangoClusterInfo.getServerEndpoint(servers[0]); - try { - removeShardFollower(endpoint, database, collection); - } catch (err) { - console.topic('heartbeat=debug', "caught exception during removal of local shard '%s/%s' of '%s/%s' from follower list", - database, collection, database, - collections[collection].planId, err); - } - } - } - console.topic('heartbeat=debug', "dropping local shard '%s/%s' of '%s/%s", - database, - collection, - database, - collections[collection].planId); - - try { - db._drop(collection, {timeout:1.0, isSystem: true}); - console.topic('heartbeat=debug', "dropping local shard '%s/%s' of '%s/%s => SUCCESS", - database, - collection, - database, - collections[collection].planId); - } - catch (err) { - console.topic('heartbeat=debug', "could not drop local shard '%s/%s' of '%s/%s within 1 second, trying again later", - database, - collection, - database, - collections[collection].planId); - } - } - } - }); - } finally { - db._useDatabase('_system'); - } - }); - - return localErrors; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief updateCurrentForCollections -// ///////////////////////////////////////////////////////////////////////////// - -function updateCurrentForCollections(localErrors, plannedCollections, - currentCollections) { - let ourselves = global.ArangoServerState.id(); - - let db = require('internal').db; - db._useDatabase('_system'); - - let localDatabases = getLocalDatabases(); - let database; - - let shardMap = getShardMap(plannedCollections); - - function assembleLocalCollectionInfo(info, error) { - if (error.collection) { - return { - error: true, - errorMessage: error.collection.errorMessage, - errorNum: error.collection.errorNum, - indexes: [], - servers: [ourselves], - }; - } - let coll = db._collection(info.name); - - let indexes = coll.getIndexes().map(index => { - let agencyIndex = {}; - Object.assign(agencyIndex, index); - // Fix up the IDs of the indexes: - let pos = index.id.indexOf("/"); - if (agencyIndex.hasOwnProperty("selectivityEstimate")) { - delete agencyIndex.selectivityEstimate; - } - if (pos >= 0) { - agencyIndex.id = index.id.slice(pos+1); - } else { - agencyIndex.id = index.id; - } - return agencyIndex; - }); - - if (error.indexErrors) { - indexes = indexes.concat(Object.keys(error.indexErrors).map(id => { - let indexError = error.indexErrors[id]; - return { - id, - error: true, - errorMessage: indexError.errorMessage, - errorNum: indexError.errorNum, - }; - })); - } - - return { - error: false, - errorMessage: '', - errorNum: 0, - indexes, - servers: [ourselves].concat(coll.getFollowers()), - }; - } - - function makeDropCurrentEntryCollection(dbname, col, shard) { - let trx = {}; - trx[curCollections + dbname + '/' + col + '/' + shard] = - {op: 'delete'}; - return trx; - } - - let trx = {}; - - // Go through local databases and collections and add stuff to Current - // as needed: - Object.keys(localDatabases).forEach(database => { - // All local databases should be in Current by now, if not, we ignore - // it, this will happen later. - try { - db._useDatabase(database); - - // iterate over collections (i.e. shards) of database - let localCollections = getLocalCollections(); - let shard; - for (shard in localCollections) { - if (localCollections.hasOwnProperty(shard)) { - let shardInfo = localCollections[shard]; - if (shardInfo.theLeader === "") { - let localCollectionInfo = assembleLocalCollectionInfo(shardInfo, localErrors[shard] || {}); - let currentCollectionInfo = fetchKey(currentCollections, database, shardInfo.planId, shard); - - if (!isEqual(localCollectionInfo, currentCollectionInfo)) { - trx[curCollections + database + '/' + shardInfo.planId + '/' + shardInfo.name] = { - op: 'set', - new: localCollectionInfo, - }; - } - } else { - let currentServers = fetchKey(currentCollections, database, shardInfo.planId, shard, 'servers'); - // we were previously leader and we are done resigning. update current and let supervision handle the rest - if (Array.isArray(currentServers) && currentServers[0] === ourselves) { - trx[curCollections + database + '/' + shardInfo.planId + '/' + shardInfo.name + '/servers'] = { - op: 'set', - new: ['_' + ourselves].concat(db._collection(shardInfo.name).getFollowers()), - }; - } - } - // mark error as handled in any case - delete localErrors[shard]; - } - } - } catch (e) { - console.topic('heartbeat=error', 'Got error while trying to sync current collections:', e, e.stack); - } finally { - // always return to previous database - db._useDatabase('_system'); - } - }); - - // Go through all current databases and collections and remove stuff - // if no longer present locally, provided : - for (database in currentCollections) { - if (currentCollections.hasOwnProperty(database)) { - if (localDatabases.hasOwnProperty(database)) { - // If a database has vanished locally, it is not our job to - // remove it in Current, that is what `updateCurrentForDatabases` - // does. - db._useDatabase(database); - - try { - // iterate over collections (i.e. shards) of database in Current - let localCollections = getLocalCollections(); - let collection; - for (collection in currentCollections[database]) { - if (currentCollections[database].hasOwnProperty(collection)) { - let shard; - for (shard in currentCollections[database][collection]) { - if (currentCollections[database][collection].hasOwnProperty(shard)) { - let cur = currentCollections[database][collection][shard]; - if (!localCollections.hasOwnProperty(shard) && - cur.servers && cur.servers[0] === ourselves && - !shardMap.hasOwnProperty(shard)) { - Object.assign(trx, makeDropCurrentEntryCollection(database, collection, shard)); - } - } - } - } - } - } finally { - // always return to previous database - db._useDatabase('_system'); - } - } - } - } - trx = Object.keys(localErrors).reduce((trx, shardName) => { - let error = localErrors[shardName]; - if (error.collection) { - trx[curCollections + error.info.database + '/' + error.info.planId + '/' + error.info.shardName] = { - op: 'set', - new: error.collection, - }; - } - return trx; - }, trx); - return trx; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief syncReplicatedShardsWithLeaders -// ///////////////////////////////////////////////////////////////////////////// - -function syncReplicatedShardsWithLeaders(plan, current, localErrors) { - let plannedDatabases = plan.Collections; - let currentDatabases = current.Collections; - let ourselves = global.ArangoServerState.id(); - - let db = require('internal').db; - db._useDatabase('_system'); - - let localDatabases = getLocalDatabases(); - - // Schedule sync tasks for shards which exist and we should be a follower: - Object.keys(plannedDatabases).forEach(databaseName => { - if (localDatabases.hasOwnProperty(databaseName) - && currentDatabases.hasOwnProperty(databaseName)) { - // switch into other database - db._useDatabase(databaseName); - // XXX shouldn't this be done during db init? - // create keyspace - try { - global.KEY_GET('shardSynchronization', 'lock'); - } catch (e) { - global.KEYSPACE_CREATE('shardSynchronization'); - global.KEY_SET('shardSynchronization', 'scheduled', {}); - global.KEY_SET('shardSynchronization', 'running', null); - global.KEY_SET('shardSynchronization', 'lock', null); - } - - try { - // iterate over collections of database - let localCollections = getLocalCollections(); - - let plannedCollections = plannedDatabases[databaseName]; - let currentCollections = currentDatabases[databaseName]; - - // find planned collections that need sync (not registered in current by the leader): - Object.keys(plannedCollections).forEach(collectionName => { - let plannedCollection = plannedCollections[collectionName]; - let currentShards = currentCollections[collectionName]; - // what should it bring - // collection.planId = collection.id; - if (currentShards !== undefined) { - let plannedShards = plannedCollection.shards; - Object.keys(plannedShards).forEach(shardName => { - // shard does not exist locally so nothing we can do at this point - if (!localCollections.hasOwnProperty(shardName)) { - return; - } - // current stuff is created by the leader - // this one here will just bring followers in sync - // so just continue here - if (!currentShards.hasOwnProperty(shardName)) { - return; - } - let currentServers = currentShards[shardName].servers; - let plannedServers = plannedShards[shardName]; - if (!plannedServers) { - console.topic('heartbeat=error', 'Shard ' + shardName + ' does not have servers substructure in plan'); - return; - } - if (!currentServers) { - console.topic('heartbeat=error', 'Shard ' + shardName + ' does not have servers substructure in current'); - return; - } - - // we are not planned to be a follower - if (plannedServers.indexOf(ourselves) <= 0) { - return; - } - // if we are considered to be in sync there is nothing to do - if (currentServers.indexOf(ourselves) > 0) { - return; - } - - let leader = plannedServers[0]; - scheduleOneShardSynchronization(databaseName, shardName, plannedCollection.id, leader); - }); - } - }); - } catch (e) { - console.topic('heartbeat=debug', 'Got an error synchronizing with leader', e, e.stack); - } finally { - // process any jobs - tryLaunchJob(); - // always return to previous database - db._useDatabase('_system'); - } - } - }); -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief take care of collections on primary DBservers according to Plan -// ///////////////////////////////////////////////////////////////////////////// - -function migratePrimary(plan, current) { - // will analyze local state and then issue db._create(), - // db._drop() etc. to sync plan and local state for shards - let localErrors = executePlanForCollections(plan.Collections); - - // diff current and local and prepare agency transactions or whatever - // to update current. Will report the errors created locally to the agency - let trx = updateCurrentForCollections(localErrors, plan.Collections, - current.Collections); - if (Object.keys(trx).length > 0) { - trx[curVersion] = {op: 'increment'}; - trx = [trx]; - // TODO: reduce timeout when we can: - try { - let res = global.ArangoAgency.write([trx]); - if (typeof res !== 'object' || !res.hasOwnProperty("results") || - typeof res.results !== 'object' || res.results.length !== 1 || - res.results[0] === 0) { - console.topic('heartbeat=error', 'migratePrimary: could not send transaction for Current to agency, result:', res); - } - } catch (err) { - console.topic('heartbeat=error', 'migratePrimary: caught exception when sending transaction for Current to agency:', err); - } - } - - // will do a diff between plan and current to find out - // the shards for which this db server is a planned - // follower. Background jobs for this activity are scheduled. - // This will then supervise any actions necessary - // to bring the shard into sync and ultimately register - // at the leader using addFollower - // this step should assume that the local state matches the - // plan...can NOT be sure that the plan was completely executed - // may react on the errors that have been created - syncReplicatedShardsWithLeaders(plan, current, localErrors); -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief executePlanForDatabases -// ///////////////////////////////////////////////////////////////////////////// - -function executePlanForDatabases(plannedDatabases) { - let localErrors = {}; - - let db = require('internal').db; - db._useDatabase('_system'); - - let localDatabases = getLocalDatabases(); - let name; - - // check which databases need to be created locally: - Object.keys(plannedDatabases).forEach(name => { - if (!localDatabases.hasOwnProperty(name)) { - // must create database - - // TODO: handle options and user information - - console.topic('heartbeat=debug', "creating local database '%s'", name); - - try { - db._createDatabase(name); - } catch (err) { - localErrors[name] = { error: true, errorNum: err.errorNum, - errorMessage: err.errorMessage, name: name }; - } - } - }); - - // check which databases need to be deleted locally - localDatabases = getLocalDatabases(); - - Object.keys(localDatabases).forEach(name => { - if (!plannedDatabases.hasOwnProperty(name) && name.substr(0, 1) !== '_') { - // must drop database - - console.topic('heartbeat=debug', "dropping local database '%s'", name); - - // Do we have to stop a replication applier first? - // Note that the secondary role no longer exists outside the schmutz -Simon - if (ArangoServerState.role() === 'SECONDARY') { - try { - db._useDatabase(name); - var rep = require('@arangodb/replication'); - var state = rep.applier.state(); - if (state.state.running === true) { - console.topic('heartbeat=debug', 'stopping replication applier first'); - rep.applier.stop(); - } - } - finally { - db._useDatabase('_system'); - } - } - db._dropDatabase(name); - } - }); - - return localErrors; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief updateCurrentForDatabases -// ///////////////////////////////////////////////////////////////////////////// - -function updateCurrentForDatabases(localErrors, currentDatabases) { - let ourselves = global.ArangoServerState.id(); - - function makeAddDatabaseAgencyOperation(payload) { - let create = {}; - create[curDatabases + payload.name + '/' + ourselves] = - {op: 'set', new: payload}; - return create; - }; - - function makeDropDatabaseAgencyOperation(name) { - let drop = {}; - drop[curDatabases + name + '/' + ourselves] = {'op':'delete'}; - return drop; - }; - - let db = require('internal').db; - db._useDatabase('_system'); - - let localDatabases = getLocalDatabases(); - let name; - let trx = {}; // Here we collect all write operations - - // Add entries that we have but that are not in Current: - for (name in localDatabases) { - if (localDatabases.hasOwnProperty(name)) { - if (!currentDatabases.hasOwnProperty(name) || - !currentDatabases[name].hasOwnProperty(ourselves) || - currentDatabases[name][ourselves].error) { - console.topic('heartbeat=debug', "adding entry in Current for database '%s'", name); - trx = Object.assign(trx, makeAddDatabaseAgencyOperation({error: false, errorNum: 0, name: name, - id: localDatabases[name].id, - errorMessage: ""})); - } - } - } - - // Remove entries from current that no longer exist locally: - for (name in currentDatabases) { - if (currentDatabases.hasOwnProperty(name) - && name.substr(0, 1) !== '_' - && localErrors[name] === undefined - ) { - if (!localDatabases.hasOwnProperty(name)) { - // we found a database we don't have locally - - if (currentDatabases[name].hasOwnProperty(ourselves)) { - // we are entered for a database that we don't have locally - console.topic('heartbeat=debug', "cleaning up entry for unknown database '%s'", name); - trx = Object.assign(trx, makeDropDatabaseAgencyOperation(name)); - } - } - } - } - - // Finally, report any errors that might have been produced earlier when - // we were trying to execute the Plan: - Object.keys(localErrors).forEach(name => { - console.topic('heartbeat=debug', "reporting error to Current about database '%s'", name); - trx = Object.assign(trx, makeAddDatabaseAgencyOperation(localErrors[name])); - }); - - return trx; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief take care of databases on any type of server according to Plan -// ///////////////////////////////////////////////////////////////////////////// - -function migrateAnyServer(plan, current) { - // will analyze local state and then issue db._createDatabase(), - // db._dropDatabase() etc. to sync plan and local state for databases - let localErrors = executePlanForDatabases(plan.Databases); - // diff current and local and prepare agency transactions or whatever - // to update current. will report the errors created locally to the agency - let trx = updateCurrentForDatabases(localErrors, current.Databases); - if (Object.keys(trx).length > 0) { - trx[curVersion] = {op: 'increment'}; - trx = [trx]; - // TODO: reduce timeout when we can: - try { - let res = global.ArangoAgency.write([trx]); - if (typeof res !== 'object' || !res.hasOwnProperty("results") || - typeof res.results !== 'object' || res.results.length !== 1 || - res.results[0] === 0) { - console.topic('heartbeat=error', 'migrateAnyServer: could not send transaction for Current to agency, result:', res); - } - } catch (err) { - console.topic('heartbeat=error', 'migrateAnyServer: caught exception when sending transaction for Current to agency:', err); - } - } -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief make sure that replication is set up for all databases -// ///////////////////////////////////////////////////////////////////////////// -function setupReplication () {// simon: is not used currently - console.topic('heartbeat=debug', 'Setting up replication...'); - - var db = require('internal').db; - var rep = require('@arangodb/replication'); - var dbs = db._databases(); - var i; - var ok = true; - for (i = 0; i < dbs.length; i++) { - var database = dbs[i]; - try { - console.topic('heartbeat=debug', 'Checking replication of database ' + database); - db._useDatabase(database); - - var state = rep.applier.state(); - if (state.state.running === false) { - var endpoint = ArangoClusterInfo.getServerEndpoint( - ArangoServerState.idOfPrimary()); - var config = { 'endpoint': endpoint, 'includeSystem': false, - 'incremental': false, 'autoStart': true, - 'requireFromPresent': true}; - console.topic('heartbeat=debug', 'Starting synchronization...'); - var res = rep.sync(config); - console.topic('heartbeat=debug', 'Last log tick: ' + res.lastLogTick + - ', starting replication...'); - rep.applier.properties(config); - var res2 = rep.applier.start(res.lastLogTick); - console.topic('heartbeat=debug', 'Result of replication start: ' + res2); - } - } catch (err) { - console.topic('heartbeat=error', 'Could not set up replication for database ', database, JSON.stringify(err)); - ok = false; - } - } - db._useDatabase('_system'); - return ok; -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief role change from secondary to primary -// / Note that the secondary role no longer exists outside the schmutz -Simon -// ///////////////////////////////////////////////////////////////////////////// - -function secondaryToPrimary () { - console.topic('heartbeat=info', 'Switching role from secondary to primary...'); - var db = require('internal').db; - var rep = require('@arangodb/replication'); - var dbs = db._databases(); - var i; - try { - for (i = 0; i < dbs.length; i++) { - var database = dbs[i]; - console.topic('heartbeat=info', 'Stopping asynchronous replication for db ' + - database + '...'); - db._useDatabase(database); - var state = rep.applier.state(); - if (state.state.running === true) { - try { - rep.applier.stop(); - } catch (err) { - console.topic('heartbeat=info', 'Exception caught whilst stopping replication!'); - } - } - rep.applier.forget(); - } - } - finally { - db._useDatabase('_system'); - } -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief role change from primary to secondary -// ///////////////////////////////////////////////////////////////////////////// - -function primaryToSecondary () { - // Note that the secondary role no longer exists outside the schmutz -Simon - console.topic('heartbeat=info', 'Switching role from primary to secondary...'); -} - -// ///////////////////////////////////////////////////////////////////////////// -// / @brief change handling trampoline function -// ///////////////////////////////////////////////////////////////////////////// - -function handleChanges (plan, current) { - // Note: This is never called with role === 'COORDINATOR' or on a single - // server. - var changed = false; - var role = ArangoServerState.role(); - // Need to check role change for automatic failover: - var myId = ArangoServerState.id(); - if (role === 'PRIMARY') { - if (!plan.DBServers[myId]) { - // Ooops! We do not seem to be a primary any more! - changed = ArangoServerState.redetermineRole(); - } - } /*else { // role === "SECONDARY" - // Note that the secondary role no longer exists outside the schmutz -Simon - if (plan.DBServers[myId]) { - changed = ArangoServerState.redetermineRole(); - if (!changed) { - // mop: oops...changing role has failed. retry next time. - return false; - } - } else { - var found = null; - var p; - for (p in plan) { - if (plan.hasOwnProperty(p) && plan[p] === myId) { - found = p; - break; - } - } - if (found !== ArangoServerState.idOfPrimary()) { // this is always "" now - // Note this includes the case that we are not found at all! - changed = ArangoServerState.redetermineRole(); - } - } - }*/ - var oldRole = role; - if (changed) { - role = ArangoServerState.role(); - console.topic('heartbeat=info', 'Our role has changed to ' + role); - // Note that the secondary role no longer exists outside the schmutz -Simon - if (oldRole === 'SECONDARY' && role === 'PRIMARY') { - secondaryToPrimary(); - } else if (oldRole === 'PRIMARY' && role === 'SECONDARY') { - primaryToSecondary(); - } - } - - migrateAnyServer(plan, current); - if (role === 'PRIMARY') { - migratePrimary(plan, current); - } else { // if (role == 'SECONDARY') { - setupReplication(); - } - - return true; -} - // ///////////////////////////////////////////////////////////////////////////// // / @brief throw an ArangoError // ///////////////////////////////////////////////////////////////////////////// @@ -1813,37 +184,6 @@ var status = function () { return global.ArangoServerState.status(); }; -// ///////////////////////////////////////////////////////////////////////////// -// / @brief handlePlanChange -// ///////////////////////////////////////////////////////////////////////////// - -var handlePlanChange = function (plan, current) { - // This is never called on a coordinator, we still make sure that it - // is not executed on a single server or coordinator, just to be sure: - if (!isCluster() || isCoordinator() || !global.ArangoServerState.initialized()) { - return true; - } - - let versions = { - plan: plan.Version, - current: current.Version - }; - - console.topic('heartbeat=debug', 'handlePlanChange:', plan.Version, current.Version); - try { - versions.success = handleChanges(plan, current); - - console.topic('heartbeat=debug', 'plan change handling successful'); - } catch (err) { - console.topic('heartbeat=error', 'error details: %s', JSON.stringify(err)); - console.topic('heartbeat=error', 'error stack: %s', err.stack); - console.topic('heartbeat=error', 'plan change handling failed'); - versions.success = false; - } - console.topic('heartbeat=debug', 'handlePlanChange: done'); - return versions; -}; - // ///////////////////////////////////////////////////////////////////////////// // / @brief coordinatorId // ///////////////////////////////////////////////////////////////////////////// @@ -2156,8 +496,23 @@ function queryAgencyJob(id) { return {error: true, errorMsg: "Did not find job.", id, job: null}; } +function getLocalInfo () { + var db = require('internal').db; + var ret = { result: {}}; + db._collections().forEach( + function(col) { + if (col.name().charAt(0) !== '_') { + ret.result[col.name()] = col.properties(); + ret.result[col.name()].indexes = []; + col.getIndexes().forEach(function(i) { + ret.result[col.name()].indexes.push(i); + }); + } + }); + return ret; +} + exports.coordinatorId = coordinatorId; -exports.handlePlanChange = handlePlanChange; exports.isCluster = isCluster; exports.isCoordinator = isCoordinator; exports.role = role; @@ -2165,7 +520,6 @@ exports.shardList = shardList; exports.status = status; exports.wait = waitForDistributedResponse; exports.endpointToURL = endpointToURL; -exports.synchronizeOneShard = synchronizeOneShard; exports.shardDistribution = shardDistribution; exports.collectionShardDistribution = collectionShardDistribution; exports.rebalanceShards = rebalanceShards; @@ -2173,10 +527,5 @@ exports.moveShard = moveShard; exports.supervisionState = supervisionState; exports.waitForSyncRepl = waitForSyncRepl; exports.endpoints = endpoints; -exports.fetchKey = fetchKey; exports.queryAgencyJob = queryAgencyJob; - -exports.executePlanForDatabases = executePlanForDatabases; -exports.executePlanForCollections = executePlanForCollections; -exports.updateCurrentForDatabases = updateCurrentForDatabases; -exports.updateCurrentForCollections = updateCurrentForCollections; +exports.getLocalInfo = getLocalInfo; diff --git a/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js b/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js deleted file mode 100644 index 27b65a6a6d..0000000000 --- a/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js +++ /dev/null @@ -1,1240 +0,0 @@ -/* global describe, it, before, beforeEach, afterEach */ - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief JavaScript cluster functionality -// / -// / @file -// / -// / DISCLAIMER -// / -// / Copyright 2017 ArangoDB GmbH, Cologne, Germany -// / -// / Licensed under the Apache License, Version 2.0 (the "License") -// / you may not use this file except in compliance with the License. -// / You may obtain a copy of the License at -// / -// / http://www.apache.org/licenses/LICENSE-2.0 -// / -// / Unless required by applicable law or agreed to in writing, software -// / distributed under the License is distributed on an "AS IS" BASIS, -// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// / See the License for the specific language governing permissions and -// / limitations under the License. -// / -// / Copyright holder is ArangoDB GmbH, Cologne, Germany -// / -// / @author Andreas Streichardt -// ////////////////////////////////////////////////////////////////////////////// - -const wait = require('internal').wait; -const db = require('internal').db; -const cluster = require('@arangodb/cluster'); -const expect = require('chai').expect; -const ArangoCollection = require('@arangodb/arango-collection').ArangoCollection; - -describe('Cluster sync', function() { - before(function() { - require('@arangodb/sync-replication-debug').setup(); - }); - - describe('Databaseplan to local', function() { - beforeEach(function() { - db._databases().forEach(database => { - if (database !== '_system') { - db._dropDatabase(database); - } - }); - }); - it('should create a planned database', function() { - let plan = { - "Databases": { - "test": { - "id": 1, - "name": "test" - } - } - }; - let errors = cluster.executePlanForDatabases(plan.Databases); - let databases = db._databases(); - expect(databases).to.have.lengthOf(2); - expect(databases).to.contain('test'); - expect(errors).to.be.empty; - }); - it('should leave everything in place if a planned database already exists', function() { - let plan = { - Databases: { - "test": { - "id": 1, - "name": "test" - } - } - }; - db._createDatabase('test'); - let errors = cluster.executePlanForDatabases(plan.Databases); - let databases = db._databases(); - expect(databases).to.have.lengthOf(2); - expect(databases).to.contain('test'); - expect(errors).to.be.empty; - }); - it('should delete a database if it is not used anymore', function() { - db._createDatabase('peng'); - let plan = { - Databases: { - } - }; - cluster.executePlanForDatabases(plan.Databases); - let databases = db._databases(); - expect(databases).to.have.lengthOf(1); - expect(databases).to.contain('_system'); - }); - }); - describe('Collection plan to local', function() { - let numSystemCollections; - before(function() { - require('@arangodb/sync-replication-debug').setup(); - }); - - beforeEach(function() { - db._databases().forEach(database => { - if (database !== '_system') { - db._dropDatabase(database); - } - }); - db._createDatabase('test'); - db._useDatabase('test'); - numSystemCollections = db._collections().length; - }); - afterEach(function() { - db._useDatabase('_system'); - }); - it('should create and load a collection if it does not exist', function() { - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 3, - "type": 2, - "waitForSync": false - } - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let collections = db._collections(); - expect(collections.map(collection => collection.name())).to.contain('s100001'); - expect(db._collection('s100001').status()).to.equal(ArangoCollection.STATUS_LOADED); - }); - it('should create a collection if it does not exist (unloaded case)', function() { - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let collections = db._collections(); - expect(collections.map(collection => collection.name())).to.contain('s100001'); - let count = 0; - while (db._collection('s100001').status() === ArangoCollection.STATUS_UNLOADING && count++ < 100) { - wait(0.1); - } - expect(db._collection('s100001').status()).to.equal(ArangoCollection.STATUS_UNLOADED); - }); - it('should unload an existing collection', function() { - db._create('s100001'); - expect(db._collection('s100001').status()).to.equal(ArangoCollection.STATUS_LOADED); - let plan = { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - }; - cluster.executePlanForCollections(plan); - db._useDatabase('test'); - let count = 0; - while (db._collection('s100001').status() === ArangoCollection.STATUS_UNLOADING && count++ < 100) { - wait(0.1); - } - expect(db._collection('s100001').status()).to.equal(ArangoCollection.STATUS_UNLOADED); - }); - it('should delete a stale collection', function() { - db._create('s100001'); - let plan = { - Collections: { - test: { - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let collections = db._collections(); - expect(collections).to.have.lengthOf(numSystemCollections); - }); - it('should ignore a collection for which it is not responsible', function() { - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "swag", - ] - }, - "status": 3, - "type": 2, - "waitForSync": false - } - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let collections = db._collections(); - expect(collections).to.have.lengthOf(numSystemCollections); - }); - it('should delete a collection for which it lost responsibility', function() { - db._create('s100001'); - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "debug-follower", // this is a different server than we are - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let collections = db._collections(); - expect(collections).to.have.lengthOf(numSystemCollections); - }); - it('should create an additional index if instructed to do so', function() { - db._create('s100001'); - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - }, - { - "error": false, - "errorMessage": "", - "errorNum": 0, - "fields": [ - "user" - ], - "id": "100005", - "sparse": true, - "type": "hash", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let indexes = db._collection('s100001').getIndexes(); - expect(indexes).to.have.lengthOf(2); - }); - it('should report failure when creating an index does not work', function() { - let c = db._create('s100001'); - c.insert({"peng": "peng"}); - c.insert({"peng": "peng"}); - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - }, - { - "error": false, - "errorMessage": "", - "errorNum": 0, - "fields": [ - "peng" - ], - "id": "100005", - "sparse": true, - "type": "hash", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - } - }; - - let errors = cluster.executePlanForCollections(plan.Collections); - expect(errors).to.have.property('s100001') - .with.property('indexErrors') - .with.property('100005'); - }); - it('should remove an additional index if instructed to do so', function() { - db._create('s100001'); - db._collection('s100001').ensureIndex({ type: "hash", fields: [ "name" ] }); - let plan = { - Databases: { - "_system": { - "id": 1, - "name": "_system" - }, - "test": { - "id": 2, - "name": "test" - } - }, - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - } - }; - cluster.executePlanForCollections(plan.Collections); - db._useDatabase('test'); - let indexes = db._collection('s100001').getIndexes(); - expect(indexes).to.have.lengthOf(1); - }); - it('should report an error when collection creation failed', function() { - let plan = { - Collections: { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "Möter": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - } - }; - let errors = cluster.executePlanForCollections(plan.Collections); - expect(errors).to.be.an('object'); - expect(errors).to.have.property('Möter'); - }); - it('should be leading a collection when ordered to be leader', function() { - let plan = { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 3, - "type": 2, - "waitForSync": false - } - } - }; - let errors = cluster.executePlanForCollections(plan); - db._useDatabase('test'); - expect(db._collection('s100001').getLeader()).to.equal(""); - }); - it('should be following a leader when ordered to be follower', function() { - let plan = { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "the leader-leader", - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - }; - let errors = cluster.executePlanForCollections(plan); - db._useDatabase('test'); - expect(db._collection('s100001').getLeader()).to.equal("the leader-leader"); - }); - it('should be able to switch from leader to follower', function() { - let plan = { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - }; - let errors = cluster.executePlanForCollections(plan); - plan.test['100001'].shards['s100001'].unshift('der-hund'); - cluster.executePlanForCollections(plan); - db._useDatabase('test'); - expect(db._collection('s100001').getLeader()).to.equal("der-hund"); - }); - it('should be able to switch from follower to leader', function() { - let plan = { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "test", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "old-leader", - "repltest", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - }; - let errors = cluster.executePlanForCollections(plan); - plan.test['100001'].shards['s100001'] = ["repltest"]; - cluster.executePlanForCollections(plan); - db._useDatabase('test'); - expect(db._collection('s100001').getLeader()).to.equal(""); - }); - it('should kill any unplanned server from current', function() { - let collection = db._create('s100001'); - collection.setTheLeader(""); - collection.addFollower('test'); - collection.addFollower('test2'); - let plan = { - test: { - "100001": { - "deleted": false, - "doCompact": true, - "id": "100001", - "indexBuckets": 8, - "indexes": [ - { - "fields": [ - "_key" - ], - "id": "0", - "sparse": false, - "type": "primary", - "unique": true - } - ], - "isSystem": false, - "isVolatile": false, - "journalSize": 1048576, - "keyOptions": { - "allowUserKeys": true, - "type": "traditional" - }, - "name": "testi", - "numberOfShards": 1, - "replicationFactor": 2, - "shardKeys": [ - "_key" - ], - "shards": { - "s100001": [ - "repltest", - "test2", - ] - }, - "status": 2, - "type": 2, - "waitForSync": false - } - } - }; - cluster.executePlanForCollections(plan); - db._useDatabase('test'); - expect(collection.getFollowers()).to.deep.equal(['test2']); - }); - }); - describe('Update current database', function() { - beforeEach(function() { - db._databases().forEach(database => { - if (database !== '_system') { - db._dropDatabase(database); - } - }); - }); - it('should report a new database', function() { - db._createDatabase('testi'); - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - } - }; - let result = cluster.updateCurrentForDatabases({}, current); - expect(result).to.have.property('/arango/Current/Databases/testi/repltest'); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.property('op', 'set'); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.deep.property('new.name', 'testi'); - }); - it('should not do anything if there is nothing to do', function() { - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - } - }; - let result = cluster.updateCurrentForDatabases({}, current); - expect(Object.keys(result)).to.have.lengthOf(0); - }); - it('should remove a database from the agency if it is not present locally', function() { - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - }, - testi: { - repltest: { - id: 2, - name: 'testi', - } - } - }; - let result = cluster.updateCurrentForDatabases({}, current); - expect(result).to.have.property('/arango/Current/Databases/testi/repltest'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.property('op', 'delete'); - }); - it('should not delete other server database entries when removing', function() { - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - }, - testi: { - repltest: { - id: 2, - name: 'testi', - }, - testreplicant: { - id: 2, - name: 'testi', - } - } - }; - let result = cluster.updateCurrentForDatabases({}, current); - expect(result).to.have.property('/arango/Current/Databases/testi/repltest'); - expect(Object.keys(result)).to.have.lengthOf(1); - }); - it('should report any errors during database creation', function() { - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - }, - }; - let result = cluster.updateCurrentForDatabases({testi: {"error": true, name: "testi"}}, current); - expect(result).to.have.property('/arango/Current/Databases/testi/repltest'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.property('op', 'set'); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.deep.property('new.error', true); - }); - it('should override any previous error if creation finally worked', function() { - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - }, - testi: { - repltest: { - id: 2, - name: 'testi', - error: true, - } - }, - }; - db._createDatabase('testi'); - let result = cluster.updateCurrentForDatabases({}, current); - expect(result).to.have.property('/arango/Current/Databases/testi/repltest'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.property('op', 'set'); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.deep.property('new.name', 'testi'); - expect(result['/arango/Current/Databases/testi/repltest']).to.have.deep.property('new.error', false); - }); - it('should not do anything if nothing happened', function() { - let current = { - _system: { - repltest: { - id: 1, - name: '_system', - }, - }, - }; - let result = cluster.updateCurrentForDatabases({}, current); - expect(Object.keys(result)).to.have.lengthOf(0); - }); - }); - describe('Update current collection', function() { - beforeEach(function() { - db._useDatabase('_system'); - db._databases().forEach(database => { - if (database !== '_system') { - db._dropDatabase(database); - } - }); - db._createDatabase('testung'); - db._useDatabase('testung'); - }); - it('should report a new collection in current', function() { - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader(""); - let current = { - }; - let result = cluster.updateCurrentForCollections({}, current); - expect(Object.keys(result)).to.have.length.of.at.least(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi'); - expect(result['/arango/Current/Collections/testung/888111/testi']).to.have.property('op', 'set'); - expect(result['/arango/Current/Collections/testung/888111/testi']).to.have.deep.property('new.servers') - .that.is.an('array') - .that.deep.equals(['repltest']); - }); - it('should not do anything if nothing changed', function() { - let current = { - }; - let result = cluster.updateCurrentForCollections({}, current); - expect(Object.keys(result)).to.have.lengthOf(0); - }); - it('should not report any collections for which we are not leader (will be handled in replication)', function() { - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader("NOBODY"); // mark collection as follower - let current = { - }; - let result = cluster.updateCurrentForCollections({}, {}, current); - expect(Object.keys(result)).to.have.lengthOf(0); - }); - it('should not delete any collections for which we are not a leader locally', function() { - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false } ], "servers" : [ "the-master", "repltest" ] } - }, - } - }; - let result = cluster.updateCurrentForCollections({}, {}, current); - expect(Object.keys(result)).to.have.lengthOf(0); - }); - it('should resign leadership for which we are no more leader locally', function() { - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader("NOBODY"); - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false } ], "servers" : [ "repltest" ] } - }, - } - }; - let result = cluster.updateCurrentForCollections({}, {}, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi/servers') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi/servers') - .that.has.property('new') - .with.deep.equal(["_repltest"]); - }); - it('should report newly assumed leadership for which we were a follower previously and remove any leaders and followers (these have to reregister themselves separately)', function() { - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader(""); - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false } ], "servers" : [ "bogus-old-leader", "repltest", "useless-follower" ] } - }, - } - }; - let result = cluster.updateCurrentForCollections({}, {}, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('new.servers') - .with.deep.equal(["repltest"]); - }); - it('should delete any collections for which we are not a leader locally', function() { - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false } ], "servers" : [ "repltest" ] } - }, - } - }; - let result = cluster.updateCurrentForCollections({}, {}, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('op', 'delete'); - }); - it('should report newly created indices', function() { - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader(""); - collection.ensureIndex({"type": "hash", "fields": ["hund"]}); - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false } ], "servers" : [ "repltest" ] } - }, - } - }; - let result = cluster.updateCurrentForCollections({}, {}, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('new.indexes') - .with.lengthOf(2); - }); - it('should report collection errors', function() { - let errors = { - 'testi': { - collection: { - 'name': 'testi', - 'error': true, - 'errorNum': 666, - 'errorMessage': 'the number of the beast :S', - }, - info: { - database: 'testung', - planId: '888111', - shardName: 'testi', - } - } - }; - let result = cluster.updateCurrentForCollections(errors, {}); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('new.error') - .equals(true); - }); - it('should report index errors', function() { - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false } ], "servers" : [ "repltest" ] } - }, - } - }; - - let errors = { - 'testi': { - 'indexErrors': { - 1: { - 'id': 1, - 'error': true, - 'errorNum': 666, - 'errorMessage': 'the number of the beast :S', - } - }, - } - }; - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader(""); - let result = cluster.updateCurrentForCollections(errors, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('new.indexes.1.error') - .equals(true); - }); - it('should report deleted indexes', function() { - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false }, - { - "fields": [ - "test" - ], - "id": "testi", - "selectivityEstimate": 1, - "sparse": false, - "type": "hash", - "unique": false - } - ], "servers" : [ "repltest" ] } - }, - } - }; - - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader(""); - let result = cluster.updateCurrentForCollections({}, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('new.indexes') - .that.has.lengthOf(1); - }); - it('should report added indexes', function() { - let current = { - testung: { - 888111: { - testi : { "error" : false, "errorMessage" : "", "errorNum" : 0, "indexes" : [ { "id" : "0", "type" : "primary", "fields" : [ "_key" ], "selectivityEstimate" : 1, "unique" : true, "sparse" : false }, - ], "servers" : [ "repltest" ] } - }, - } - }; - - let props = { planId: '888111' }; - let collection = db._create('testi', props); - collection.setTheLeader(""); - collection.ensureIndex({type: "hash", fields: ["test"]}); - let result = cluster.updateCurrentForCollections({}, current); - expect(result).to.be.an('object'); - expect(Object.keys(result)).to.have.lengthOf(1); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.have.property('op', 'set'); - expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') - .that.has.deep.property('new.indexes') - .that.has.lengthOf(2); - }); - }); -}); diff --git a/js/server/tests/shell/shell-create-options-cluster-spec.js b/js/server/tests/shell/shell-create-options-cluster-spec.js index 18b6a12b19..0029e9b59e 100644 --- a/js/server/tests/shell/shell-create-options-cluster-spec.js +++ b/js/server/tests/shell/shell-create-options-cluster-spec.js @@ -39,7 +39,7 @@ describe('Cluster collection creation options', function() { db._drop('testi'); }); it('should wait for all followers to get in sync when waiting for replication', function() { - db._create("testi", {replicationFactor: 2, numberOfShards: 16}, {waitForSyncReplication: true}); + db._create("testi", {replicationFactor: 2, numberOfShards: 32}, {waitForSyncReplication: true}); let current = ArangoAgency.get('Current/Collections/_system'); let plan = ArangoAgency.get('Plan/Collections/_system'); let collectionId = Object.values(plan.arango.Plan.Collections['_system']).reduce((result, collectionDef) => { @@ -56,24 +56,5 @@ describe('Cluster collection creation options', function() { expect(entry.servers).to.have.lengthOf(2); }); }); - it('should not wait for all followers to get in sync when waiting for replication', function() { - db._create("testi", {replicationFactor: 2, numberOfShards: 16}, {waitForSyncReplication: false}); - let current = ArangoAgency.get('Current/Collections/_system'); - let plan = ArangoAgency.get('Plan/Collections/_system'); - let collectionId = Object.values(plan.arango.Plan.Collections['_system']).reduce((result, collectionDef) => { - if (result) { - return result; - } - if (collectionDef.name === 'testi') { - return collectionDef.id; - } - }, undefined); - - let someNotInSync = Object.values(current.arango.Current.Collections['_system'][collectionId]).some(entry => { - return entry.servers.length < 2; - }); - expect(someNotInSync).to.be.true; - }); - -}); \ No newline at end of file +}); diff --git a/lib/Basics/Result.cpp b/lib/Basics/Result.cpp new file mode 100644 index 0000000000..90d39926dc --- /dev/null +++ b/lib/Basics/Result.cpp @@ -0,0 +1,120 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2017 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +//////////////////////////////////////////////////////////////////////////////// + +#include "Result.h" +#include "Basics/VelocyPackHelper.h" + +using namespace arangodb; + +Result::Result() : _errorNumber(TRI_ERROR_NO_ERROR) {} + +Result::Result(int errorNumber) : _errorNumber(errorNumber){ + if (errorNumber != TRI_ERROR_NO_ERROR) { + _errorMessage = TRI_errno_string(errorNumber); + } +} + +Result::Result(int errorNumber, std::string const& errorMessage) + : _errorNumber(errorNumber), _errorMessage(errorMessage) {} + +Result::Result(int errorNumber, std::string&& errorMessage) + : _errorNumber(errorNumber), _errorMessage(std::move(errorMessage)) {} + +Result::Result(Result const& other) + : _errorNumber(other._errorNumber), _errorMessage(other._errorMessage) {} + +Result::Result(Result&& other) noexcept + : _errorNumber(other._errorNumber), + _errorMessage(std::move(other._errorMessage)) {} + +Result& Result::operator=(Result const& other) { + _errorNumber = other._errorNumber; + _errorMessage = other._errorMessage; + return *this; +} + +Result& Result::operator=(Result&& other) noexcept { + _errorNumber = other._errorNumber; + _errorMessage = std::move(other._errorMessage); + return *this; +} + +Result::~Result() {} + +bool Result::ok() const noexcept { return _errorNumber == TRI_ERROR_NO_ERROR; } + +bool Result::fail() const noexcept { return !ok(); } + +int Result::errorNumber() const noexcept { return _errorNumber; } + +bool Result::is(int errorNumber) const noexcept { + return _errorNumber == errorNumber; } + +bool Result::isNot(int errorNumber) const { return !is(errorNumber); } + +Result& Result::reset(int errorNumber) { + _errorNumber = errorNumber; + + if (errorNumber != TRI_ERROR_NO_ERROR) { + _errorMessage = TRI_errno_string(errorNumber); + } else { + _errorMessage.clear(); + } + return *this; +} + +Result& Result::reset(int errorNumber, std::string const& errorMessage) { + _errorNumber = errorNumber; + _errorMessage = errorMessage; + return *this; +} + +Result& Result::reset(int errorNumber, std::string&& errorMessage) noexcept { + _errorNumber = errorNumber; + _errorMessage = std::move(errorMessage); + return *this; +} + +Result& Result::reset(Result const& other) { + _errorNumber = other._errorNumber; + _errorMessage = other._errorMessage; + return *this; +} + +Result& Result::reset(Result&& other) noexcept { + _errorNumber = other._errorNumber; + _errorMessage = std::move(other._errorMessage); + return *this; +} + +std::string Result::errorMessage() const& { return _errorMessage; } + +std::string Result::errorMessage() && { return std::move(_errorMessage); } + +std::ostream& operator<<(std::ostream& out, arangodb::Result const& result) { + VPackBuilder dump; + { VPackObjectBuilder b(&dump); + dump.add("errorNumber", VPackValue(result.errorNumber())); + dump.add("errorMessage", VPackValue(result.errorMessage())); } + out << dump.slice().toJson(); + return out; +} diff --git a/lib/Basics/Result.h b/lib/Basics/Result.h index 8020cb7f61..af6041fa48 100644 --- a/lib/Basics/Result.h +++ b/lib/Basics/Result.h @@ -28,100 +28,146 @@ namespace arangodb { class Result { public: - Result() : _errorNumber(TRI_ERROR_NO_ERROR) {} + Result(); Result(bool avoidCastingErrors) = delete; - Result(int errorNumber) - : _errorNumber(errorNumber){ - if (errorNumber != TRI_ERROR_NO_ERROR) { - _errorMessage = TRI_errno_string(errorNumber); - } - } + Result(int errorNumber); - Result(int errorNumber, std::string const& errorMessage) - : _errorNumber(errorNumber), _errorMessage(errorMessage) {} - - Result(int errorNumber, std::string&& errorMessage) - : _errorNumber(errorNumber), _errorMessage(std::move(errorMessage)) {} + Result(int errorNumber, std::string const& errorMessage); + + /** + * @brief Construct with error number and message + * @param errorNumber Said error number + * @param errorMessage Said error message + */ + Result(int errorNumber, std::string&& errorMessage); - // copy - Result(Result const& other) - : _errorNumber(other._errorNumber), - _errorMessage(other._errorMessage) {} + /** + * @brief Construct as copy + * @param other To copy from + */ + Result(Result const& other); - Result& operator=(Result const& other) { - _errorNumber = other._errorNumber; - _errorMessage = other._errorMessage; - return *this; - } - - // move - Result(Result&& other) noexcept - : _errorNumber(other._errorNumber), - _errorMessage(std::move(other._errorMessage)) {} + /** + * @brief Construct as clone + * @param other The prototype + */ + Result(Result&& other) noexcept; + + /** + * @brief Assignment operator + * @param other To assign from + * @return Refernce to ourselves + */ + Result& operator=(Result const& other); - Result& operator=(Result&& other) noexcept { - _errorNumber = other._errorNumber; - _errorMessage = std::move(other._errorMessage); - return *this; - } + /** + * @brief Assignment operator + * @param other To assign from + * @return Refernce to ourselves + */ + Result& operator=(Result&& other) noexcept; - virtual ~Result() {} + /** + * @brief Default dtor + */ + virtual ~Result(); public: - bool ok() const { return _errorNumber == TRI_ERROR_NO_ERROR; } - bool fail() const { return !ok(); } - int errorNumber() const { return _errorNumber; } - bool is(int errorNumber) const { return _errorNumber == errorNumber; } - bool isNot(int errorNumber) const { return !is(errorNumber); } + /** + * @brief Nomen est omen + * @return OK? + */ + bool ok() const noexcept; - Result& reset(int errorNumber = TRI_ERROR_NO_ERROR) { - _errorNumber = errorNumber; + /** + * @see ok() + */ + bool fail() const noexcept; - if (errorNumber != TRI_ERROR_NO_ERROR) { - _errorMessage = TRI_errno_string(errorNumber); - } else { - _errorMessage.clear(); - } - return *this; - } + /** + * @brief Get error number + * @return error number + */ + int errorNumber() const noexcept; - Result& reset(int errorNumber, std::string const& errorMessage) { - _errorNumber = errorNumber; - _errorMessage = errorMessage; - return *this; - } + /** + * @brief Is specific error + * @param errorNumber Said specific error + * @return Equality with specific error + */ + bool is(int errorNumber) const noexcept; - Result& reset(int errorNumber, std::string&& errorMessage) noexcept { - _errorNumber = errorNumber; - _errorMessage = std::move(errorMessage); - return *this; - } + /** + * @see is(int errorNumber) + */ + bool isNot(int errorNumber) const; - Result& reset(Result const& other) { - _errorNumber = other._errorNumber; - _errorMessage = other._errorMessage; - return *this; - } + /** + * @brief Reset to specific error number. + * If ok, error message is cleared. + * @param errorNumber Said specific error number + * @return Reference to ourselves + */ + Result& reset(int errorNumber = TRI_ERROR_NO_ERROR); + + /** + * @brief Reset to specific error number with message. + * If ok, error message is cleared. + * @param errorNumber Said specific error number + * @param errorMessage Said specific error message + * @return Reference to ourselves + */ + Result& reset(int errorNumber, std::string const& errorMessage); + + /** + * @brief Reset to specific error number with message. + * If ok, error message is cleared. + * @param errorNumber Said specific error number + * @param errorMessage Said specific error message + * @return Reference to ourselves + */ + Result& reset(int errorNumber, std::string&& errorMessage) noexcept; - Result& reset(Result&& other) noexcept { - _errorNumber = other._errorNumber; - _errorMessage = std::move(other._errorMessage); - return *this; - } + /** + * @brief Reset to other error. + * @param other Said specific error + * @return Reference to ourselves + */ + Result& reset(Result const& other); - // the default implementation is const, but sub-classes might - // really do more work to compute. + /** + * @brief Reset to other error. + * @param other Said specific error + * @return Reference to ourselves + */ + Result& reset(Result&& other) noexcept; - virtual std::string errorMessage() const& { return _errorMessage; } - virtual std::string errorMessage() && { return std::move(_errorMessage); } + /** + * @brief Get error message + * @return Our error message + */ + virtual std::string errorMessage() const&; + + /** + * @brief Get error message + * @return Our error message + */ + virtual std::string errorMessage() &&; protected: int _errorNumber; std::string _errorMessage; }; + } +/** + * @brief Print to output stream + * @return Said output steam + */ +std::ostream& operator<<(std::ostream& out, arangodb::Result const& result); + #endif diff --git a/lib/Basics/errors.dat b/lib/Basics/errors.dat index e90cb63e4b..c74333a12d 100755 --- a/lib/Basics/errors.dat +++ b/lib/Basics/errors.dat @@ -309,7 +309,7 @@ ERROR_USER_NOT_FOUND,1703,"user not found","Will be raised when a user name is u ERROR_USER_CHANGE_PASSWORD,1704,"user must change his password","Will be raised when the user must change his password." ERROR_USER_EXTERNAL,1705,"user is external","Will be raised when the user is authenicated by an external server." -################################################################################ +############################################################################### ## Service management errors (legacy) ## These have been superceded by the Foxx management errors in public APIs. ################################################################################ @@ -487,3 +487,12 @@ ERROR_SUPERVISION_GENERAL_FAILURE,20501,"general supervision failure","General s ERROR_DISPATCHER_IS_STOPPING,21001,"dispatcher stopped","Will be returned if a shutdown is in progress." ERROR_QUEUE_UNKNOWN,21002,"named queue does not exist","Will be returned if a queue with this name does not exist." ERROR_QUEUE_FULL,21003,"named queue is full","Will be returned if a queue with this name is full." + +################################################################################ +## Maintenance errors +################################################################################ + +ERROR_ACTION_ALREADY_REGISTERED,6001,"maintenance action already registered","Action with this description has been registered already" +ERROR_ACTION_OPERATION_UNABORTABLE,6002,"this maintenance action cannot be stopped","This maintenance action cannot be stopped once it is started" +ERROR_ACTION_UNFINISHED,6003,"maintenance action still processing","This maintenance action is still processing" +ERROR_NO_SUCH_ACTION,6004,"no such maintenance action","No such maintenance action exists" \ No newline at end of file diff --git a/lib/Basics/voc-errors.cpp b/lib/Basics/voc-errors.cpp index 882e7c3662..9dd4d20159 100644 --- a/lib/Basics/voc-errors.cpp +++ b/lib/Basics/voc-errors.cpp @@ -346,4 +346,8 @@ void TRI_InitializeErrorMessages() { REG_ERROR(ERROR_DISPATCHER_IS_STOPPING, "dispatcher stopped"); REG_ERROR(ERROR_QUEUE_UNKNOWN, "named queue does not exist"); REG_ERROR(ERROR_QUEUE_FULL, "named queue is full"); + REG_ERROR(ERROR_ACTION_ALREADY_REGISTERED, "maintenance action already registered"); + REG_ERROR(ERROR_ACTION_OPERATION_UNABORTABLE, "this maintenance action cannot be stopped"); + REG_ERROR(ERROR_ACTION_UNFINISHED, "maintenance action still processing"); + REG_ERROR(ERROR_NO_SUCH_ACTION, "no such maintenance action"); } diff --git a/lib/Basics/voc-errors.h b/lib/Basics/voc-errors.h index d4dbca9bf6..71df44154a 100644 --- a/lib/Basics/voc-errors.h +++ b/lib/Basics/voc-errors.h @@ -1836,6 +1836,26 @@ constexpr int TRI_ERROR_QUEUE_UNKNOWN /// Will be returned if a queue with this name is full. constexpr int TRI_ERROR_QUEUE_FULL = 21003; +/// 6001: ERROR_ACTION_ALREADY_REGISTERED +/// "maintenance action already registered" +/// Action with this description has been registered already +constexpr int TRI_ERROR_ACTION_ALREADY_REGISTERED = 6001; + +/// 6002: ERROR_ACTION_OPERATION_UNABORTABLE +/// "this maintenance action cannot be stopped" +/// This maintenance action cannot be stopped once it is started +constexpr int TRI_ERROR_ACTION_OPERATION_UNABORTABLE = 6002; + +/// 6003: ERROR_ACTION_UNFINISHED +/// "maintenance action still processing" +/// This maintenance action is still processing +constexpr int TRI_ERROR_ACTION_UNFINISHED = 6003; + +/// 6004: ERROR_NO_SUCH_ACTION +/// "no such maintenance action" +/// No such maintenance action exists +constexpr int TRI_ERROR_NO_SUCH_ACTION = 6004; + /// register all errors for ArangoDB void TRI_InitializeErrorMessages(); diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index bb58e829ea..12bc49f2a0 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -173,6 +173,7 @@ add_library(${LIB_ARANGO} STATIC Basics/Nonce.cpp Basics/OpenFilesTracker.cpp Basics/ReadWriteLock.cpp + Basics/Result.cpp Basics/RocksDBLogger.cpp Basics/RocksDBUtils.cpp Basics/SharedPRNG.cpp diff --git a/lib/Logger/LogTopic.cpp b/lib/Logger/LogTopic.cpp index c9764267f1..f48879f67c 100644 --- a/lib/Logger/LogTopic.cpp +++ b/lib/Logger/LogTopic.cpp @@ -120,6 +120,7 @@ LogTopic Logger::FIXME("general", LogLevel::INFO); LogTopic Logger::GRAPHS("graphs", LogLevel::INFO); LogTopic Logger::HEARTBEAT("heartbeat", LogLevel::INFO); LogTopic Logger::HTTPCLIENT("httpclient", LogLevel::WARN); +LogTopic Logger::MAINTENANCE("maintenance", LogLevel::WARN); LogTopic Logger::MEMORY("memory", LogLevel::WARN); LogTopic Logger::MMAP("mmap"); LogTopic Logger::PERFORMANCE("performance", LogLevel::WARN); diff --git a/lib/Logger/Logger.h b/lib/Logger/Logger.h index f5c410327e..284e58ef48 100644 --- a/lib/Logger/Logger.h +++ b/lib/Logger/Logger.h @@ -145,6 +145,7 @@ class Logger { static LogTopic GRAPHS; static LogTopic HEARTBEAT; static LogTopic HTTPCLIENT; + static LogTopic MAINTENANCE; static LogTopic MEMORY; static LogTopic MMAP; static LogTopic PERFORMANCE; diff --git a/scripts/cluster-run-common.sh b/scripts/cluster-run-common.sh index ff09795fe6..75c6392018 100644 --- a/scripts/cluster-run-common.sh +++ b/scripts/cluster-run-common.sh @@ -11,6 +11,7 @@ function help() { echo " -j/--jwt-secret JWT-Secret (string default: )" echo " --log-level-agency Log level (agency) (string default: )" echo " --log-level-cluster Log level (cluster) (string default: )" + echo " -l/--log-level Log level (string default: )" echo " -i/--interactive Interactive mode (C|D|R default: '')" echo " -x/--xterm XTerm command (default: xterm)" echo " -o/--xterm-options XTerm options (default: --geometry=80x43)" @@ -70,6 +71,10 @@ while [[ -n "$1" ]]; do TRANSPORT=${2} shift ;; + -l|--log-level) + LOG_LEVEL=${2} + shift + ;; --log-level-agency) LOG_LEVEL_AGENCY=${2} shift diff --git a/scripts/generateMaintenanceTestData.sh b/scripts/generateMaintenanceTestData.sh new file mode 100755 index 0000000000..e8a3f56e2b --- /dev/null +++ b/scripts/generateMaintenanceTestData.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# FIXMEMAINTENANCE: please add a couple of lines about why this script exists + +header="R\"=(" +footer=")=\"" + +outfile=Plan.json +echo $header > $outfile +curl -s localhost:4001/_api/agency/read -d'[["/arango/Plan"]]'|jq .[0].arango.Plan >> $outfile +echo $footer >> $outfile + +outfile=Current.json +echo $header > $outfile +curl -s localhost:4001/_api/agency/read -d'[["/arango/Current"]]'|jq .[0].arango.Current >> $outfile +echo $footer >> $outfile + +outfile=Supervision.json +echo $header > $outfile +supervision=$(curl -s localhost:4001/_api/agency/read -d'[["/arango/Supervision"]]'|jq .[0].arango.Supervision) +echo $supervision | jq .>> $outfile +echo $footer >> $outfile + +servers=$(echo $supervision | jq .Health| jq 'keys[]') +for i in $servers; do + shortname=$(echo $supervision | jq -r .Health[$i].ShortName) + endpoint=$(echo $supervision | jq -r .Health[$i].Endpoint) + endpoint=$(echo $endpoint |sed s/tcp/http/g) + + dbs=$(curl -s $endpoint/_api/database|jq -r .result|jq -r '.[]') + + tmpfile=$shortname.tmp + outfile=$shortname.json + echo "{" >> $tmpfile + j=0 + for i in $dbs; do + if [ $j -gt 0 ]; then + echo -n "," >> $tmpfile + fi + echo -n "\"$i\" :" >> $tmpfile + curl -s $endpoint/_db/$i/_admin/execute?returnAsJSON=true -d 'return require("@arangodb/cluster").getLocalInfo()'|jq .result >> $tmpfile + (( j++ )) + done + echo "}" >> $tmpfile + echo "R\"=(" > $outfile + cat $tmpfile | jq . >> $outfile + echo ")=\"" >> $outfile + rm $tmpfile +done diff --git a/scripts/startLocalCluster.sh b/scripts/startLocalCluster.sh index c1290a50a0..404e4d7552 100755 --- a/scripts/startLocalCluster.sh +++ b/scripts/startLocalCluster.sh @@ -153,6 +153,7 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do --log.file cluster/$port.log \ --log.force-direct true \ --log.level $LOG_LEVEL_AGENCY \ + --javascript.allow-admin-execute true \ $STORAGE_ENGINE \ $AUTHENTICATION \ $SSLKEYFILE \ @@ -193,6 +194,7 @@ start() { --javascript.app-path cluster/apps$PORT \ --log.force-direct true \ --log.level $LOG_LEVEL_CLUSTER \ + --javascript.allow-admin-execute true \ $STORAGE_ENGINE \ $AUTHENTICATION \ $SSLKEYFILE \ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 56f09f9310..f59ad00954 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -128,6 +128,9 @@ set(ARANGODB_TESTS_SOURCES Geo/NearUtilsTest.cpp Geo/ShapeContainerTest.cpp Graph/ClusterTraverserCacheTest.cpp + Maintenance/MaintenanceFeatureTest.cpp + Maintenance/MaintenanceRestHandlerTest.cpp + Maintenance/MaintenanceTest.cpp Pregel/typedbuffer.cpp RocksDBEngine/Endian.cpp RocksDBEngine/KeyTest.cpp diff --git a/tests/Maintenance/Current.json b/tests/Maintenance/Current.json new file mode 100644 index 0000000000..45a580879f --- /dev/null +++ b/tests/Maintenance/Current.json @@ -0,0 +1,2006 @@ +R"=( +{ + "Foxxmaster": "CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8", + "ServersRegistered": { + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89": { + "endpoint": "tcp://[::1]:8629", + "engine": "rocksdb", + "version": 30400, + "host": "ac8ddefc7d1f4364ba655b4debcd076f" + }, + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291": { + "endpoint": "tcp://[::1]:8630", + "engine": "rocksdb", + "version": 30400, + "host": "ac8ddefc7d1f4364ba655b4debcd076f" + }, + "CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8": { + "endpoint": "tcp://[::1]:8530", + "engine": "rocksdb", + "version": 30400, + "host": "ac8ddefc7d1f4364ba655b4debcd076f" + }, + "Version": 1, + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83": { + "endpoint": "tcp://[::1]:8631", + "engine": "rocksdb", + "version": 30400, + "host": "ac8ddefc7d1f4364ba655b4debcd076f" + } + }, + "Databases": { + "_system": { + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83": { + "errorMessage": "", + "error": false, + "id": "1", + "name": "_system", + "errorNum": 0 + }, + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291": { + "errorMessage": "", + "error": false, + "id": "1", + "name": "_system", + "errorNum": 0 + }, + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89": { + "errorMessage": "", + "error": false, + "id": "1", + "name": "_system", + "errorNum": 0 + } + }, + "foo": { + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83": { + "errorMessage": "", + "error": false, + "id": "10033126", + "name": "foo", + "errorNum": 0 + }, + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291": { + "errorMessage": "", + "error": false, + "id": "8035848", + "name": "foo", + "errorNum": 0 + }, + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89": { + "errorMessage": "", + "error": false, + "id": "9035030", + "name": "foo", + "errorNum": 0 + } + } + }, + "Collections": { + "_system": { + "2010049": { + "s2010055": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010051": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010053": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010057": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010056": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010052": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010054": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010058": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010050": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + } + }, + "2010019": { + "s2010020": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010034": { + "s2010035": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "2010036", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + } + }, + "2010001": { + "s2010002": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010006": { + "s2010007": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010008": { + "s2010009": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010010": { + "s2010011": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010015": { + "s2010016": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010021": { + "s2010022": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "2010023", + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "2010024", + "sparse": true, + "type": "skiplist", + "unique": true + } + ] + } + }, + "2010061": { + "s2010063": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010062": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010065": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010064": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010025": { + "s2010026": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "2010027", + "sparse": true, + "type": "hash", + "unique": true + } + ] + } + }, + "2010028": { + "s2010029": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010003": { + "s2010004": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "user" + ], + "id": "2010005", + "sparse": true, + "type": "hash", + "unique": true + } + ] + } + }, + "2010031": { + "s2010032": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "2010033", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + } + }, + "2010017": { + "s2010018": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010037": { + "s2010038": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "2010039", + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + } + }, + "2010045": { + "s2010046": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + } + }, + "foo": { + "2010082": { + "s2010083": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010069": { + "s2010070": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010080": { + "s2010081": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010078": { + "s2010079": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010071": { + "s2010072": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010091": { + "s2010092": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010088": { + "s2010089": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "2010090", + "sparse": true, + "type": "hash", + "unique": true + } + ] + } + }, + "2010097": { + "s2010099": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010100": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010101": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010098": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + } + ] + } + }, + "2010102": { + "s2010107": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010106": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010109": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010104": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010108": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010110": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010111": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010103": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010105": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "2010084": { + "s2010085": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "2010086", + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "2010087", + "sparse": true, + "type": "skiplist", + "unique": true + } + ] + } + }, + "2010073": { + "s2010074": { + "error": false, + "errorMessage": "", + "errorNum": 0, + "servers": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + } + } + }, + "DBServers": { + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83": "none", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291": "none", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89": "none" + }, + "Singles": {}, + "Coordinators": { + "CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8": "none" + }, + "AsyncReplication": {}, + "NewServers": {}, + "Version": 422, + "Lock": "UNLOCKED", + "FoxxmasterQueueupdate": false, + "ShardsCopied": {} +} +)=" diff --git a/tests/Maintenance/DBServer0001.json b/tests/Maintenance/DBServer0001.json new file mode 100644 index 0000000000..ab8127a786 --- /dev/null +++ b/tests/Maintenance/DBServer0001.json @@ -0,0 +1,2020 @@ +R"=( +{ + "_system": { + "s2010002": { + "id": "4", + "name": "s2010002", + "type": 2, + "status": 3, + "planId": "2010001", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010001/s2010002", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9030915", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010002/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010004": { + "id": "5", + "name": "s2010004", + "type": 2, + "status": 3, + "planId": "2010003", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010003/s2010004", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010030 + }, + "objectId": "9030942", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010004/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "user" + ], + "id": "s2010004/2010005", + "selectivityEstimate": 1, + "sparse": true, + "type": "hash", + "unique": true + } + ] + }, + "s2010007": { + "id": "6", + "name": "s2010007", + "type": 2, + "status": 3, + "planId": "2010006", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010006/s2010007", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9030999", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010007/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010009": { + "id": "7", + "name": "s2010009", + "type": 2, + "status": 3, + "planId": "2010008", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010008/s2010009", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9031050", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010009/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010011": { + "id": "8", + "name": "s2010011", + "type": 2, + "status": 3, + "planId": "2010010", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010010/s2010011", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010014 + }, + "objectId": "9031117", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010011/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010016": { + "id": "9", + "name": "s2010016", + "type": 2, + "status": 3, + "planId": "2010015", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010015/s2010016", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9031207", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010016/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010018": { + "id": "10", + "name": "s2010018", + "type": 2, + "status": 3, + "planId": "2010017", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010017/s2010018", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010137 + }, + "objectId": "9031303", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010018/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010020": { + "id": "11", + "name": "s2010020", + "type": 2, + "status": 3, + "planId": "2010019", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010019/s2010020", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9031415", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010020/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010022": { + "id": "12", + "name": "s2010022", + "type": 2, + "status": 3, + "planId": "2010021", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010021/s2010022", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9031541", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010022/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "s2010022/2010023", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "s2010022/2010024", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + } + ] + }, + "s2010026": { + "id": "13", + "name": "s2010026", + "type": 2, + "status": 3, + "planId": "2010025", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010025/s2010026", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 7010004 + }, + "objectId": "9031842", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010026/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "s2010026/2010027", + "selectivityEstimate": 1, + "sparse": true, + "type": "hash", + "unique": true + } + ] + }, + "s2010029": { + "id": "14", + "name": "s2010029", + "type": 2, + "status": 3, + "planId": "2010028", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010028/s2010029", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9032087", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010029/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010032": { + "id": "15", + "name": "s2010032", + "type": 2, + "status": 3, + "planId": "2010031", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010031/s2010032", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 17040012 + }, + "objectId": "9032277", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010032/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "s2010032/2010033", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010035": { + "id": "16", + "name": "s2010035", + "type": 2, + "status": 3, + "planId": "2010034", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010034/s2010035", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 17040013 + }, + "objectId": "9032486", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010035/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "s2010035/2010036", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010038": { + "id": "17", + "name": "s2010038", + "type": 2, + "status": 3, + "planId": "2010037", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010037/s2010038", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 17040011 + }, + "objectId": "9032716", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010038/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "s2010038/2010039", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010046": { + "id": "18", + "name": "s2010046", + "type": 2, + "status": 3, + "planId": "2010045", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010045/s2010046", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9033458", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010046/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010051": { + "id": "20", + "name": "s2010051", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010051", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034002", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010051/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010051/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010051/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010051/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010051/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010051/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010051/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010051/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010052": { + "id": "22", + "name": "s2010052", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010052", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034012", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010052/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010052/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010052/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010052/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010052/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010052/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010052/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010052/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010054": { + "id": "23", + "name": "s2010054", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010054", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034017", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010054/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010054/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010054/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010054/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010054/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010054/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010054/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010054/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010055": { + "id": "19", + "name": "s2010055", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010055", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9033997", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010055/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010055/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010055/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010055/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010055/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010055/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010055/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010055/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010057": { + "id": "21", + "name": "s2010057", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010057", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034007", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010057/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010057/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010057/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010057/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010057/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010057/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010057/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010057/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010058": { + "id": "24", + "name": "s2010058", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010058", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034022", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010058/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010058/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010058/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010058/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010058/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010058/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010058/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010058/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010062": { + "id": "26", + "name": "s2010062", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010062", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034512", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010062/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010063": { + "id": "25", + "name": "s2010063", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010063", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034509", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010063/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010064": { + "id": "28", + "name": "s2010064", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010064", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034518", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010064/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010065": { + "id": "27", + "name": "s2010065", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010065", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9034515", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010065/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "foo": { + "s2010070": { + "id": "29", + "name": "s2010070", + "type": 2, + "status": 3, + "planId": "2010069", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010069/s2010070", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9035277", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010070/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010072": { + "id": "30", + "name": "s2010072", + "type": 2, + "status": 3, + "planId": "2010071", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010071/s2010072", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9035661", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010072/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010074": { + "id": "31", + "name": "s2010074", + "type": 2, + "status": 3, + "planId": "2010073", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010073/s2010074", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010077 + }, + "objectId": "9036060", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010074/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010079": { + "id": "32", + "name": "s2010079", + "type": 2, + "status": 3, + "planId": "2010078", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010078/s2010079", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9036482", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010079/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010081": { + "id": "33", + "name": "s2010081", + "type": 2, + "status": 3, + "planId": "2010080", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010080/s2010081", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9036911", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010081/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010083": { + "id": "34", + "name": "s2010083", + "type": 2, + "status": 3, + "planId": "2010082", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010082/s2010083", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9037355", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010083/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010085": { + "id": "35", + "name": "s2010085", + "type": 2, + "status": 3, + "planId": "2010084", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010084/s2010085", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9037819", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010085/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "s2010085/2010086", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "s2010085/2010087", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + } + ] + }, + "s2010089": { + "id": "36", + "name": "s2010089", + "type": 2, + "status": 3, + "planId": "2010088", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010088/s2010089", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 7010002 + }, + "objectId": "9038804", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010089/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "s2010089/2010090", + "selectivityEstimate": 1, + "sparse": true, + "type": "hash", + "unique": true + } + ] + }, + "s2010092": { + "id": "37", + "name": "s2010092", + "type": 2, + "status": 3, + "planId": "2010091", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010091/s2010092", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9039560", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010092/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010098": { + "id": "41", + "name": "s2010098", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010098", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9040583", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010098/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010098/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010099": { + "id": "38", + "name": "s2010099", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010099", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9040568", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010099/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010099/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010100": { + "id": "39", + "name": "s2010100", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010100", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9040573", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010100/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010100/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010101": { + "id": "40", + "name": "s2010101", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010101", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9040578", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010101/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010101/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010103": { + "id": "46", + "name": "s2010103", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010103", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9041430", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010103/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010105": { + "id": "47", + "name": "s2010105", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010105", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9041433", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010105/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010106": { + "id": "42", + "name": "s2010106", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010106", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9041418", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010106/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010108": { + "id": "44", + "name": "s2010108", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010108", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9041424", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010108/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010109": { + "id": "43", + "name": "s2010109", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010109", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9041421", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010109/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010111": { + "id": "45", + "name": "s2010111", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010111", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "9041427", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010111/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + } +} +)=" diff --git a/tests/Maintenance/DBServer0002.json b/tests/Maintenance/DBServer0002.json new file mode 100644 index 0000000000..8cfbdf94d4 --- /dev/null +++ b/tests/Maintenance/DBServer0002.json @@ -0,0 +1,2020 @@ +R"=( +{ + "_system": { + "s2010002": { + "id": "1000006", + "name": "s2010002", + "type": 2, + "status": 3, + "planId": "2010001", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010001/s2010002", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8032200", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010002/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010004": { + "id": "1000007", + "name": "s2010004", + "type": 2, + "status": 3, + "planId": "2010003", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010003/s2010004", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010030 + }, + "objectId": "8032214", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010004/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "user" + ], + "id": "s2010004/2010005", + "selectivityEstimate": 1, + "sparse": true, + "type": "hash", + "unique": true + } + ] + }, + "s2010007": { + "id": "1000008", + "name": "s2010007", + "type": 2, + "status": 3, + "planId": "2010006", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010006/s2010007", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8032271", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010007/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010009": { + "id": "1000009", + "name": "s2010009", + "type": 2, + "status": 3, + "planId": "2010008", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010008/s2010009", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8032318", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010009/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010011": { + "id": "1000010", + "name": "s2010011", + "type": 2, + "status": 3, + "planId": "2010010", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010010/s2010011", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010014 + }, + "objectId": "8032381", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010011/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010016": { + "id": "1000011", + "name": "s2010016", + "type": 2, + "status": 3, + "planId": "2010015", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010015/s2010016", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8032460", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010016/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010018": { + "id": "1000012", + "name": "s2010018", + "type": 2, + "status": 3, + "planId": "2010017", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010017/s2010018", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010137 + }, + "objectId": "8032546", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010018/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010020": { + "id": "1000013", + "name": "s2010020", + "type": 2, + "status": 3, + "planId": "2010019", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010019/s2010020", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8032648", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010020/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010022": { + "id": "1000014", + "name": "s2010022", + "type": 2, + "status": 3, + "planId": "2010021", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010021/s2010022", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8032760", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010022/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "s2010022/2010023", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "s2010022/2010024", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + } + ] + }, + "s2010026": { + "id": "1000015", + "name": "s2010026", + "type": 2, + "status": 3, + "planId": "2010025", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010025/s2010026", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 7010004 + }, + "objectId": "8033022", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010026/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "s2010026/2010027", + "selectivityEstimate": 1, + "sparse": true, + "type": "hash", + "unique": true + } + ] + }, + "s2010029": { + "id": "1000016", + "name": "s2010029", + "type": 2, + "status": 3, + "planId": "2010028", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010028/s2010029", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8033244", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010029/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010032": { + "id": "1000017", + "name": "s2010032", + "type": 2, + "status": 3, + "planId": "2010031", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010031/s2010032", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 17040012 + }, + "objectId": "8033400", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010032/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "s2010032/2010033", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010035": { + "id": "1000018", + "name": "s2010035", + "type": 2, + "status": 3, + "planId": "2010034", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010034/s2010035", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 17040013 + }, + "objectId": "8033587", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010035/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "s2010035/2010036", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010038": { + "id": "1000019", + "name": "s2010038", + "type": 2, + "status": 3, + "planId": "2010037", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010037/s2010038", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 17040011 + }, + "objectId": "8033790", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010038/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "s2010038/2010039", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010046": { + "id": "1000020", + "name": "s2010046", + "type": 2, + "status": 3, + "planId": "2010045", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010045/s2010046", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034360", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010046/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010050": { + "id": "1000026", + "name": "s2010050", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010050", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034814", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010050/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010050/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010050/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010050/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010050/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010050/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010050/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010050/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010051": { + "id": "1000021", + "name": "s2010051", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010051", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034789", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010051/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010051/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010051/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010051/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010051/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010051/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010051/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010051/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010053": { + "id": "1000022", + "name": "s2010053", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010053", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034794", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010053/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010053/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010053/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010053/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010053/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010053/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010053/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010053/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010054": { + "id": "1000025", + "name": "s2010054", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010054", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034809", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010054/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010054/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010054/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010054/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010054/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010054/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010054/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010054/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010056": { + "id": "1000024", + "name": "s2010056", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010056", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034804", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010056/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010056/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010056/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010056/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010056/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010056/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010056/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010056/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010057": { + "id": "1000023", + "name": "s2010057", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010057", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8034799", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010057/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010057/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010057/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010057/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010057/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010057/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010057/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010057/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010062": { + "id": "1000028", + "name": "s2010062", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010062", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8035383", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010062/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010063": { + "id": "1000027", + "name": "s2010063", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010063", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8035380", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010063/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010064": { + "id": "1000030", + "name": "s2010064", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010064", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8035389", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010064/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010065": { + "id": "1000029", + "name": "s2010065", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010065", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8035386", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010065/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "foo": { + "s2010070": { + "id": "1000031", + "name": "s2010070", + "type": 2, + "status": 3, + "planId": "2010069", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010069/s2010070", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8036080", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010070/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010072": { + "id": "1000032", + "name": "s2010072", + "type": 2, + "status": 3, + "planId": "2010071", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010071/s2010072", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8036432", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010072/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010074": { + "id": "1000033", + "name": "s2010074", + "type": 2, + "status": 3, + "planId": "2010073", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010073/s2010074", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 2010077 + }, + "objectId": "8036799", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010074/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010079": { + "id": "1000034", + "name": "s2010079", + "type": 2, + "status": 3, + "planId": "2010078", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010078/s2010079", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8037182", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010079/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010081": { + "id": "1000035", + "name": "s2010081", + "type": 2, + "status": 3, + "planId": "2010080", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010080/s2010081", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8037575", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010081/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010083": { + "id": "1000036", + "name": "s2010083", + "type": 2, + "status": 3, + "planId": "2010082", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010082/s2010083", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8037981", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010083/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010085": { + "id": "1000037", + "name": "s2010085", + "type": 2, + "status": 3, + "planId": "2010084", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010084/s2010085", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8038405", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010085/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "s2010085/2010086", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "s2010085/2010087", + "selectivityEstimate": 1, + "sparse": true, + "type": "skiplist", + "unique": true + } + ] + }, + "s2010089": { + "id": "1000038", + "name": "s2010089", + "type": 2, + "status": 3, + "planId": "2010088", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010088/s2010089", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 7010002 + }, + "objectId": "8039304", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010089/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "s2010089/2010090", + "selectivityEstimate": 1, + "sparse": true, + "type": "hash", + "unique": true + } + ] + }, + "s2010092": { + "id": "1000039", + "name": "s2010092", + "type": 2, + "status": 3, + "planId": "2010091", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010091/s2010092", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8039993", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010092/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010098": { + "id": "1000043", + "name": "s2010098", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010098", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8040896", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010098/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010098/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010099": { + "id": "1000040", + "name": "s2010099", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010099", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8040881", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010099/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010099/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010100": { + "id": "1000041", + "name": "s2010100", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010100", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8040886", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010100/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010100/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010101": { + "id": "1000042", + "name": "s2010101", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010101", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8040891", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010101/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010101/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010103": { + "id": "1000049", + "name": "s2010103", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010103", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8041670", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010103/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010104": { + "id": "1000047", + "name": "s2010104", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010104", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8041664", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010104/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010106": { + "id": "1000045", + "name": "s2010106", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010106", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8041658", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010106/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010107": { + "id": "1000044", + "name": "s2010107", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010107", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8041655", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010107/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010109": { + "id": "1000046", + "name": "s2010109", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010109", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8041661", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010109/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010110": { + "id": "1000048", + "name": "s2010110", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010110", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "8041667", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010110/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + } +} +)=" diff --git a/tests/Maintenance/DBServer0003.json b/tests/Maintenance/DBServer0003.json new file mode 100644 index 0000000000..d704c925e3 --- /dev/null +++ b/tests/Maintenance/DBServer0003.json @@ -0,0 +1,1158 @@ +R"=( +{ + "_system": { + "s2010050": { + "id": "3010009", + "name": "s2010050", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010050", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032697", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010050/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010050/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010050/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010050/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010050/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010050/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010050/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010050/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010052": { + "id": "3010007", + "name": "s2010052", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010052", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032687", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010052/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010052/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010052/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010052/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010052/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010052/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010052/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010052/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010053": { + "id": "3010011", + "name": "s2010053", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010053", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032731", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010053/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010053/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010053/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010053/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010053/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010053/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010053/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010053/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010055": { + "id": "3010010", + "name": "s2010055", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010055", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032726", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010055/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010055/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010055/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010055/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010055/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010055/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010055/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010055/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010056": { + "id": "3010006", + "name": "s2010056", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010056", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032682", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010056/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010056/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010056/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010056/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010056/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010056/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010056/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010056/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010058": { + "id": "3010008", + "name": "s2010058", + "type": 3, + "status": 3, + "planId": "2010049", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010049/s2010058", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032692", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010058/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010058/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gi" + ], + "geoJson": false, + "id": "s2010058/2010116", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "bestIndexedLevel": 17, + "fields": [ + "gij" + ], + "geoJson": true, + "id": "s2010058/2010122", + "maxNumCoverCells": 8, + "sparse": true, + "type": "geo", + "unique": false, + "worstIndexedLevel": 4 + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "s2010058/2010126", + "selectivityEstimate": 1, + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "s2010058/2010130", + "selectivityEstimate": 1, + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "s2010058/2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "s2010058/2010136", + "selectivityEstimate": 1, + "sparse": false, + "type": "skiplist", + "unique": false + } + ] + }, + "s2010062": { + "id": "3010013", + "name": "s2010062", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010062", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032901", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010062/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010063": { + "id": "3010012", + "name": "s2010063", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010063", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032898", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010063/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010064": { + "id": "3010015", + "name": "s2010064", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010064", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032907", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010064/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010065": { + "id": "3010014", + "name": "s2010065", + "type": 2, + "status": 3, + "planId": "2010061", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010061/s2010065", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10032904", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010065/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + }, + "foo": { + "s2010098": { + "id": "3010019", + "name": "s2010098", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010098", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035163", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010098/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010098/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010099": { + "id": "3010016", + "name": "s2010099", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010099", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035148", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010099/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010099/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010100": { + "id": "3010017", + "name": "s2010100", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010100", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035153", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010100/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010100/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010101": { + "id": "3010018", + "name": "s2010101", + "type": 3, + "status": 3, + "planId": "2010097", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010097/s2010101", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035158", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010101/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from", + "_to" + ], + "id": "s2010101/2", + "selectivityEstimate": 1, + "sparse": false, + "type": "edge", + "unique": false + } + ] + }, + "s2010104": { + "id": "3010021", + "name": "s2010104", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010104", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035494", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010104/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010105": { + "id": "3010025", + "name": "s2010105", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010105", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035506", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010105/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010107": { + "id": "3010020", + "name": "s2010107", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010107", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035491", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010107/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010108": { + "id": "3010022", + "name": "s2010108", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010108", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035497", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010108/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010110": { + "id": "3010023", + "name": "s2010110", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010110", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035500", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010110/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + }, + "s2010111": { + "id": "3010024", + "name": "s2010111", + "type": 2, + "status": 3, + "planId": "2010102", + "theLeader": "", + "cacheEnabled": false, + "globallyUniqueId": "c2010102/s2010111", + "isSystem": false, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional", + "lastValue": 0 + }, + "objectId": "10035503", + "statusString": "loaded", + "waitForSync": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "s2010111/0", + "selectivityEstimate": 1, + "sparse": false, + "type": "primary", + "unique": true + } + ] + } + } +} +)=" diff --git a/tests/Maintenance/MaintenanceFeatureTest.cpp b/tests/Maintenance/MaintenanceFeatureTest.cpp new file mode 100644 index 0000000000..5cb4670b83 --- /dev/null +++ b/tests/Maintenance/MaintenanceFeatureTest.cpp @@ -0,0 +1,800 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite for ClusterComm +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +/// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "catch.hpp" + +#include + +#include +#include + +#include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/ConditionLocker.h" +#include "Basics/ConditionVariable.h" +#include "Basics/Result.h" +#include "Cluster/Action.h" +#include "Cluster/MaintenanceFeature.h" + +// +// structure used to store expected states of action properties +// +struct Expected { + int _id; + int _result; + int _state; + int _progress; +}; + +typedef std::vector ExpectedVec_t; + +// +// TestProgressHandler lets us know once ApplicationServer is ready +// +class TestProgressHandler : public arangodb::application_features::ProgressHandler { +public: + TestProgressHandler() { + _serverReady=false; + + using std::placeholders::_1; + _state = std::bind(& TestProgressHandler::StateChange, this, _1); + + using std::placeholders::_2; + _feature = std::bind(& TestProgressHandler::FeatureChange, this, _1, _2); + } + + + void StateChange(arangodb::application_features::ServerState newState) { + if (arangodb::application_features::ServerState::IN_WAIT == newState) { + CONDITION_LOCKER(clock, _serverReadyCond); + _serverReady = true; + _serverReadyCond.broadcast(); + } + } + + void FeatureChange(arangodb::application_features::ServerState newState, std::string const &) { + } + + arangodb::basics::ConditionVariable _serverReadyCond; + std::atomic_bool _serverReady; + +};// class TestProgressHandler + + +using namespace arangodb::maintenance; + +// +// TestFeature wraps MaintenanceFeature to all test specific action objects +// by overriding the actionFactory() virtual function. Two versions: +// 1. default constructor for non-threaded actions +// 2. constructor with ApplicationServer pointer for threaded actions +// +class TestMaintenanceFeature : public arangodb::MaintenanceFeature { +public: + TestMaintenanceFeature(arangodb::application_features::ApplicationServer& as) + : arangodb::MaintenanceFeature(as) { + + // begin with no threads to allow queue validation + _maintenanceThreadsMax = 0; + as.addReporter(_progressHandler); + }; + + virtual ~TestMaintenanceFeature() { + + }; + + void setSecondsActionsBlock(uint32_t seconds) {_secondsActionsBlock = seconds;}; + + + /// @brief set thread count, then activate the threads via start(). One time usage only. + /// Code waits until background ApplicationServer known to have fully started. + void setMaintenanceThreadsMax(uint32_t threads) { + CONDITION_LOCKER(clock, _progressHandler._serverReadyCond); + while(!_progressHandler._serverReady) { + _progressHandler._serverReadyCond.wait(); + } // while + + _maintenanceThreadsMax = threads; + start(); + } // setMaintenanceThreadsMax + + + virtual arangodb::Result addAction( + std::shared_ptr action, bool executeNow=false) { + _recentAction = action; + return MaintenanceFeature::addAction(action, executeNow); + } + + virtual arangodb::Result addAction( + std::shared_ptr const & description, + bool executeNow=false) { + return MaintenanceFeature::addAction(description, executeNow); + } + + + bool verifyRegistryState(ExpectedVec_t & expected) { + bool good(true); + + VPackArrayIterator registry(toVelocyPack().slice()); + + auto action = registry.begin(); + auto check = expected.begin(); + + for ( ; registry.end() != action && expected.end()!=check; ++action, ++check) { + VPackSlice id = (*action).get("id"); + if (!(id.isInteger() && id.getInt() == check->_id)) { + std::cerr << "Id mismatch: action has " << id.getInt() + << " expected " << check->_id << std::endl; + good = false; + } // if + + VPackSlice result = (*action).get("result"); + if (!(result.isInteger() && check->_result == result.getInt())) { + std::cerr << "Result mismatch: action has " << result.getInt() + << " expected " << check->_result << std::endl; + good = false; + } // if + + VPackSlice state = (*action).get("state"); + if (!(state.isInteger() && check->_state == state.getInt())) { + std::cerr << "State mismatch: action has " << state.getInt() + << " expected " << check->_state << std::endl; + good = false; + } // if + + VPackSlice progress = (*action).get("progress"); + if (!(progress.isInteger() && check->_progress == progress.getInt())) { + std::cerr << "Progress mismatch: action has " << progress.getInt() + << " expected " << check->_progress << std::endl; + good = false; + } // if + } // for + + return good; + + } // verifyRegistryState + + + /// @brief poll registry until all actions finish + void waitRegistryComplete() { + bool again; + + do { + again = false; + std::this_thread::sleep_for(std::chrono::seconds(1)); + VPackArrayIterator registry(toVelocyPack().slice()); + for (auto action : registry ) { + VPackSlice state = action.get("state"); + again = again || (COMPLETE != state.getInt() && FAILED != state.getInt()); + } // for + } while(again); + } // waitRegistryComplete + +public: + std::shared_ptr _recentAction; + TestProgressHandler _progressHandler; + +};// TestMaintenanceFeature + + +// +// TestActionBasic simulates a multistep action by counting down +// on each call to first() and next() until iteration counter is zero. +// Returns false upon reaching zero +// +class TestActionBasic : public ActionBase { +public: + TestActionBasic( + arangodb::MaintenanceFeature& feature, ActionDescription const& description) + : ActionBase(feature, description), _iteration(1), _resultCode(0) { + + std::string value, iterate_count; + auto gres = description.get("iterate_count", iterate_count); + + if (gres.ok()) { + _iteration = std::atol(iterate_count.c_str()); + // safety check + if (_iteration < 0) { + _iteration = 1; + } // if + } // if + + if (description.get("result_code", value).ok()) { + _resultCode = std::atol(value.c_str()); + } // if + + if (description.get("preaction_result_code", value).ok()) { + std::map pred { + {"name","TestActionBasic"}, {"result_code",value}}; + if (gres.ok()) { + pred.insert({"iterate_count",iterate_count}); + } + _preAction = std::make_shared(pred); + } // if + + + if (description.get("postaction_result_code", value).ok()) { + std::map postd { + {"name","TestActionBasic"}, {"result_code",value}}; + if (gres.ok()) { + postd.insert({"iterate_count",iterate_count}); + } + _postAction = std::make_shared(postd); + } // if + }; + + virtual ~TestActionBasic() {}; + + bool first() override { + + // a pre action needs to push before setting _result + if (_preDesc) { + createPreAction(_preDesc); + } else if (0==_iteration) { + // time to set result? + _result.reset(_resultCode); + } // if + + // verify first() called once + if (0!=getProgress()) { + _result.reset(2); + } // if + + return(iteratorEndTest()); + }; + + bool next() override { + // time to set result? + if (0==_iteration) { + _result.reset(_resultCode); + } // if + + // verify next() called properly + if (0==getProgress()) { + _result.reset(2); + } // if + + return(iteratorEndTest()); + }; + + +protected: + bool iteratorEndTest() { + bool more; + + // + if (_result.ok()) { + more = 0 < _iteration--; + + // if going to stop, see if a postAction is needed + if (!more && _postDesc) { + createPostAction(_postDesc); + } // if + } else { + // !ok() ... always stop iteration + more = false; + } + return more; + } // iteratorEndTest + +public: + int _iteration; + int _resultCode; + std::shared_ptr _preDesc; + std::shared_ptr _postDesc; + +};// TestActionBasic + +// +// +// Unit Tests start here +// +// +TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") { + + std::chrono::system_clock::time_point baseTime(std::chrono::system_clock::now()); + std::chrono::system_clock::time_point noTime; + + SECTION("Iterate Action 0 times - ok") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","0"}}))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(result.ok()); + REQUIRE(tf._recentAction->result().ok()); + REQUIRE(0==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == COMPLETE); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(noTime == tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); + + } + + SECTION("Iterate Action 0 times - fail") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","0"},{"result_code","1"} + }))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(!result.ok()); + REQUIRE(!tf._recentAction->result().ok()); + REQUIRE(0==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == FAILED); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(noTime == tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); +} + + SECTION("Iterate Action 1 time - ok") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","1"}}))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(result.ok()); + REQUIRE(tf._recentAction->result().ok()); + REQUIRE(1==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == COMPLETE); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(baseTime <= tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); +} + + SECTION("Iterate Action 1 time - fail") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","1"},{"result_code","1"} + }))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(!result.ok()); + REQUIRE(!tf._recentAction->result().ok()); + REQUIRE(1==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == FAILED); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(baseTime <= tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); + REQUIRE(tf._recentAction->getLastStatTime() <= tf._recentAction->getDoneTime()); + } + + SECTION("Iterate Action 2 times - ok") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","2"} + }))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(result.ok()); + REQUIRE(tf._recentAction->result().ok()); + REQUIRE(2==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == COMPLETE); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(baseTime <= tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); + REQUIRE(tf._recentAction->getLastStatTime() <= tf._recentAction->getDoneTime()); + } + + SECTION("Iterate Action 100 times - ok") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"}}))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(result.ok()); + REQUIRE(tf._recentAction->result().ok()); + REQUIRE(100==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == COMPLETE); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(baseTime <= tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); + REQUIRE(tf._recentAction->getLastStatTime() <= tf._recentAction->getDoneTime()); + } + + SECTION("Iterate Action 100 times - fail") { + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature tf(as); + tf.setSecondsActionsBlock(0); // disable retry wait for now + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"},{"result_code","1"} + }))); + arangodb::Result result = tf.addAction( + std::make_shared(std::move(action_base_ptr)), true); + + REQUIRE(!result.ok()); + REQUIRE(!tf._recentAction->result().ok()); + REQUIRE(100==tf._recentAction->getProgress()); + REQUIRE(tf._recentAction->getState() == FAILED); + REQUIRE(tf._recentAction->done()); + REQUIRE(1==tf._recentAction->id()); + + REQUIRE(baseTime <= tf._recentAction->getCreateTime()); + REQUIRE(baseTime <= tf._recentAction->getStartTime()); + REQUIRE(baseTime <= tf._recentAction->getDoneTime()); + REQUIRE(baseTime <= tf._recentAction->getLastStatTime()); + REQUIRE(tf._recentAction->getCreateTime() <= tf._recentAction->getStartTime()); + REQUIRE(tf._recentAction->getStartTime() <= tf._recentAction->getDoneTime()); + REQUIRE(tf._recentAction->getLastStatTime() <= tf._recentAction->getDoneTime()); + } +} // MaintenanceFeatureUnthreaded + +TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") { + + SECTION("Populate action queue and validate") { + std::vector pre_thread, post_thread; + + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature * tf = new TestMaintenanceFeature(as); + as.addFeature(tf); + std::thread th( + &arangodb::application_features::ApplicationServer::run, &as, 0, nullptr); + + // + // 1. load up the queue without threads running + // a. 100 iterations then fail + + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + *tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"},{"result_code","1"} + }))); + arangodb::Result result = tf->addAction( + std::make_shared(std::move(action_base_ptr)), false); + + REQUIRE(result.ok()); // has not executed, ok() is about parse and list add + REQUIRE(tf->_recentAction->result().ok()); + pre_thread.push_back({1,0,READY,0}); + post_thread.push_back({1,1,FAILED,100}); + + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + *tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","2"} + }))); + result = tf->addAction( + std::make_shared(std::move(action_base_ptr)), false); + + REQUIRE(result.ok()); // has not executed, ok() is about parse and list add + REQUIRE(tf->_recentAction->result().ok()); + pre_thread.push_back({2,0,READY,0}); + post_thread.push_back({2,0,COMPLETE,2}); + + // c. duplicate of 'a', should fail to add + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + *tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"},{"result_code","1"} + }))); + result = tf->addAction( + std::make_shared(std::move(action_base_ptr)), false); + + REQUIRE(!result.ok()); // has not executed, ok() is about parse and list add + // _recentAction will NOT contain the aborted object ... don't test it + + // + // 2. see if happy about queue prior to threads running + REQUIRE(tf->verifyRegistryState(pre_thread)); + + // + // 3. start threads AFTER ApplicationServer known to be running + tf->setMaintenanceThreadsMax(1); + + // + // 4. loop while waiting for threads to complete all actions + tf->waitRegistryComplete(); + + // + // 5. verify completed actions + REQUIRE(tf->verifyRegistryState(post_thread)); + +#if 0 // for debugging + std::cout << tf->toVelocyPack().toJson() << std::endl; +#endif + + // + // 6. bring down the ApplicationServer, i.e. clean up + as.beginShutdown(); + + th.join(); + + } + + + SECTION("Action that generates a pre-action") { + std::vector pre_thread, post_thread; + + std::shared_ptr po = + std::make_shared( + "test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature * tf = new TestMaintenanceFeature(as); + as.addFeature(tf); + std::thread th( + &arangodb::application_features::ApplicationServer::run, &as, 0, nullptr); + + // + // 1. load up the queue without threads running + // a. 100 iterations then fail + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + *tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"},{"preaction_result_code","0"} + }))); + arangodb::Result result = tf->addAction( + std::make_shared(std::move(action_base_ptr)), false); + + REQUIRE(result.ok()); // has not executed, ok() is about parse and list add + REQUIRE(tf->_recentAction->result().ok()); + pre_thread.push_back({1,0,READY,0}); + post_thread.push_back({1,0,COMPLETE,100}); + post_thread.push_back({2,0,COMPLETE,100}); // preaction results + + // + // 2. see if happy about queue prior to threads running + REQUIRE(tf->verifyRegistryState(pre_thread)); + + // + // 3. start threads AFTER ApplicationServer known to be running + tf->setMaintenanceThreadsMax(1); + + // + // 4. loop while waiting for threads to complete all actions + tf->waitRegistryComplete(); + + // + // 5. verify completed actions + REQUIRE(tf->verifyRegistryState(post_thread)); + +#if 0 // for debugging + std::cout << tf->toVelocyPack().toJson() << std::endl; +#endif + + // + // 6. bring down the ApplicationServer, i.e. clean up + as.beginShutdown(); + th.join(); + } + + + SECTION("Action that generates a post-action") { + std::vector pre_thread, post_thread; + + std::shared_ptr po = std::make_shared("test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature * tf = new TestMaintenanceFeature(as); + as.addFeature(tf); + std::thread th(&arangodb::application_features::ApplicationServer::run, &as, 0, nullptr); + + // + // 1. load up the queue without threads running + // a. 100 iterations then fail + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + *tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"},{"postaction_result_code","0"} + }))); + arangodb::Result result = tf->addAction( + std::make_shared(std::move(action_base_ptr)), false); + + REQUIRE(result.ok()); // has not executed, ok() is about parse and list add + REQUIRE(tf->_recentAction->result().ok()); + pre_thread.push_back({1,0,READY,0}); + post_thread.push_back({1,0,COMPLETE,100}); + post_thread.push_back({2,0,COMPLETE,100}); // postaction results + + // + // 2. see if happy about queue prior to threads running + REQUIRE(tf->verifyRegistryState(pre_thread)); + + // + // 3. start threads AFTER ApplicationServer known to be running + tf->setMaintenanceThreadsMax(1); + + // + // 4. loop while waiting for threads to complete all actions + tf->waitRegistryComplete(); + + // + // 5. verify completed actions + REQUIRE(tf->verifyRegistryState(post_thread)); + +#if 0 // for debugging + std::cout << tf->toVelocyPack().toJson() << std::endl; +#endif + + // + // 6. bring down the ApplicationServer, i.e. clean up + as.beginShutdown(); + th.join(); + } + + SECTION("Action delete") { + std::vector pre_thread, post_thread; + + std::shared_ptr po = std::make_shared("test", std::string(), std::string(), "path"); + arangodb::application_features::ApplicationServer as(po, nullptr); + TestMaintenanceFeature * tf = new TestMaintenanceFeature(as); + as.addFeature(tf); + std::thread th(&arangodb::application_features::ApplicationServer::run, &as, 0, nullptr); + + // + // 1. load up the queue without threads running + // a. 100 iterations then fail + std::unique_ptr action_base_ptr; + action_base_ptr.reset( + (ActionBase*) new TestActionBasic( + *tf, ActionDescription(std::map{ + {"name","TestActionBasic"},{"iterate_count","100"},{"postaction_result_code","0"} + }))); + arangodb::Result result = tf->addAction( + std::make_shared(std::move(action_base_ptr)), false); + + REQUIRE(result.ok()); // has not executed, ok() is about parse and list add + REQUIRE(tf->_recentAction->result().ok()); + pre_thread.push_back({1,0,READY,0}); + post_thread.push_back({1,0,FAILED,0}); + + // + // 2. see if happy about queue prior to threads running + REQUIRE(tf->verifyRegistryState(pre_thread)); + tf->deleteAction(1); + + // + // 3. start threads AFTER ApplicationServer known to be running + tf->setMaintenanceThreadsMax(1); + + // + // 4. loop while waiting for threads to complete all actions + tf->waitRegistryComplete(); + + // + // 5. verify completed actions + REQUIRE(tf->verifyRegistryState(post_thread)); + +#if 0 // for debugging + std::cout << tf->toVelocyPack().toJson() << std::endl; +#endif + + // + // 6. bring down the ApplicationServer, i.e. clean up + as.beginShutdown(); + th.join(); + } + + + } // MaintenanceFeatureThreaded diff --git a/tests/Maintenance/MaintenanceRestHandlerTest.cpp b/tests/Maintenance/MaintenanceRestHandlerTest.cpp new file mode 100644 index 0000000000..95186a7f4d --- /dev/null +++ b/tests/Maintenance/MaintenanceRestHandlerTest.cpp @@ -0,0 +1,105 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite for ClusterComm +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2017-2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +/// @author Copyright 2017-2018, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "catch.hpp" + +#include + +#include +#include + +#include "Rest/HttpRequest.h" +#include "Rest/HttpResponse.h" +#include "Cluster/MaintenanceRestHandler.h" + +// GeneralResponse only has a "protected" constructor. +class TestResponse : public arangodb::HttpResponse { +public: + TestResponse() : arangodb::HttpResponse(arangodb::rest::ResponseCode::OK) {}; + +}; // class TestResponse + + +// give access to some protected routines for more thorough unit tests +class TestHandler : public arangodb::MaintenanceRestHandler { +public: + TestHandler(arangodb::GeneralRequest* req, arangodb::GeneralResponse* res) + : arangodb::MaintenanceRestHandler(req,res) {}; + + bool test_parsePutBody(VPackSlice const & parameters) { + return parsePutBody(parameters); + } + +}; // class TestHandler + + +TEST_CASE("MaintenanceRestHandler", "[cluster][maintenance][devel]") { + + SECTION("Parse REST PUT") { + VPackBuilder body; + + // intentionally building this in non-alphabetic order, and name not first + // {"name":"CreateCollection","collection":"a","database":"test","properties":{"journalSize":1111}} + { VPackObjectBuilder b(&body); + body.add("database", VPackValue("test")); + body.add("name", VPackValue("CreateCollection")); + body.add(VPackValue("properties")); + { + VPackObjectBuilder bb(&body); + body.add("journalSize", VPackValue(1111)); + } + body.add("collection", VPackValue("a")); + } + + std::string json_str(body.toJson()); + + std::unordered_map x; + arangodb::HttpRequest * dummyRequest = arangodb::HttpRequest::createHttpRequest(arangodb::rest::ContentType::JSON, + json_str.c_str(), json_str.length(), x); + dummyRequest->setRequestType(arangodb::rest::RequestType::PUT); + TestResponse * dummyResponse = new TestResponse; + TestHandler dummyHandler(dummyRequest, dummyResponse); + + REQUIRE(true==dummyHandler.test_parsePutBody(body.slice())); + REQUIRE(dummyHandler.getActionDesc().has("name")); + REQUIRE(dummyHandler.getActionDesc().get("name") == "CreateCollection"); + REQUIRE(dummyHandler.getActionDesc().has("collection")); + REQUIRE(dummyHandler.getActionDesc().get("collection") == "a"); + REQUIRE(dummyHandler.getActionDesc().has("database")); + REQUIRE(dummyHandler.getActionDesc().get("database") == "test"); + + VPackObjectIterator it(dummyHandler.getActionProp().slice(), true); + REQUIRE(it.key().copyString() == "journalSize"); + REQUIRE(it.value().getInt() == 1111); + + } + + SECTION("Local databases one more empty database should be dropped") { + + } +} diff --git a/tests/Maintenance/MaintenanceTest.cpp b/tests/Maintenance/MaintenanceTest.cpp new file mode 100644 index 0000000000..029436bff1 --- /dev/null +++ b/tests/Maintenance/MaintenanceTest.cpp @@ -0,0 +1,840 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite for Cluster maintenance +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2017 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Kaveh Vahedipour +/// @author Matthew Von-Maszewski +/// @author Copyright 2017, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "catch.hpp" + +#include "Cluster/Maintenance.h" + +#include +#include + +#include +#include +#include +#include +#include + +using namespace arangodb; +using namespace arangodb::consensus; +using namespace arangodb::maintenance; + +#ifndef _WIN32 + +char const* planStr = +#include "Plan.json" +; +char const* currentStr = +#include "Current.json" +; +char const* supervisionStr = +#include "Supervision.json" +; +char const* dbs0Str = +#include "DBServer0001.json" +; +char const* dbs1Str = +#include "DBServer0002.json" +; +char const* dbs2Str = +#include "DBServer0003.json" +; + +std::map matchShortLongIds(Node const& supervision) { + std::map ret; + for (auto const& dbs : supervision("Health").children()) { + if (dbs.first.front() == 'P') { + ret.emplace((*dbs.second)("ShortName").getString(),dbs.first); + } + } + return ret; +} + +Node createNodeFromBuilder(Builder const& builder) { + + Builder opBuilder; + { VPackObjectBuilder a(&opBuilder); + opBuilder.add("new", builder.slice()); } + + Node node(""); + node.handle(opBuilder.slice()); + return node; + +} + +Builder createBuilder(char const* c) { + + VPackOptions options; + options.checkAttributeUniqueness = true; + VPackParser parser(&options); + parser.parse(c); + + Builder builder; + builder.add(parser.steal()->slice()); + return builder; + +} + +Node createNode(char const* c) { + return createNodeFromBuilder(createBuilder(c)); +} + +// Random stuff +std::random_device rd; +std::mt19937 g(rd()); + +// Relevant agency +auto plan = createNode(planStr); +auto originalPlan = plan; +auto supervision = createNode(supervisionStr); +auto current = createNode(currentStr); + +std::vector shortNames { + "DBServer0001","DBServer0002","DBServer0003"}; + +// map +auto dbsIds = matchShortLongIds(supervision); + +std::string PLAN_COL_PATH = "/Collections/"; +std::string PLAN_DB_PATH = "/Databases/"; + +size_t localId = 1016002; + +VPackBuilder createDatabase(std::string const& dbname) { + Builder builder; + { VPackObjectBuilder o(&builder); + builder.add("id", VPackValue(std::to_string(localId++))); + builder.add( + "coordinator", VPackValue("CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8")); + builder.add(VPackValue("options")); + { VPackObjectBuilder oo(&builder); } + builder.add("name", VPackValue(dbname)); } + return builder; +} + +void createPlanDatabase(std::string const& dbname, Node& plan) { + plan(PLAN_DB_PATH + dbname) = createDatabase(dbname).slice(); +} + +VPackBuilder createIndex( + std::string const& type, std::vector const& fields, + bool unique, bool sparse, bool deduplicate) { + + VPackBuilder index; + { VPackObjectBuilder o(&index); + { index.add("deduplicate", VPackValue(deduplicate)); + index.add(VPackValue("fields")); + { VPackArrayBuilder a(&index); + for (auto const& field : fields) { + index.add(VPackValue(field)); + }} + index.add("id", VPackValue(std::to_string(localId++))); + index.add("sparse", VPackValue(sparse)); + index.add("type", VPackValue(type)); + index.add("unique", VPackValue(unique)); + }} + + return index; + +} + +void createPlanIndex( + std::string const& dbname, std::string const& colname, + std::string const& type, std::vector const& fields, + bool unique, bool sparse, bool deduplicate, Node& plan) { + + VPackBuilder val; + { VPackObjectBuilder o(&val); + val.add("new", createIndex(type, fields, unique, sparse, deduplicate).slice()); } + plan(PLAN_COL_PATH + dbname + "/" + colname + "/indexes").handle(val.slice()); + +} + +void createCollection( + std::string const& colname, VPackBuilder& col) { + + VPackBuilder keyOptions; + { VPackObjectBuilder o(&keyOptions); + keyOptions.add("lastValue", VPackValue(0)); + keyOptions.add("type", VPackValue("traditional")); + keyOptions.add("allowUserKeys", VPackValue(true)); } + + VPackBuilder shardKeys; + { VPackArrayBuilder a(&shardKeys); + shardKeys.add(VPackValue("_key")); } + + VPackBuilder indexes; + { VPackArrayBuilder a(&indexes); + indexes.add(createIndex("primary", {"_key"}, true, false, false).slice()); } + + col.add("id", VPackValue(std::to_string(localId++))); + col.add("status", VPackValue(3)); + col.add("keyOptions", keyOptions.slice()); + col.add("cacheEnabled", VPackValue(false)); + col.add("waitForSync", VPackValue(false)); + col.add("type", VPackValue(2)); + col.add("isSystem", VPackValue(true)); + col.add("name", VPackValue(colname)); + col.add("shardingStrategy", VPackValue("hash")); + col.add("statusString", VPackValue("loaded")); + col.add("shardKeys", shardKeys.slice()); + +} + +std::string S("s"); +std::string C("c"); + +void createPlanShards ( + size_t numberOfShards, size_t replicationFactor, VPackBuilder& col) { + + auto servers = shortNames; + std::shuffle(servers.begin(), servers.end(), g); + + col.add("numberOfShards", VPackValue(1)); + col.add("replicationFactor", VPackValue(2)); + col.add(VPackValue("shards")); + { VPackObjectBuilder s(&col); + for (size_t i = 0; i < numberOfShards; ++i) { + col.add(VPackValue(S + std::to_string(localId++))); + { VPackArrayBuilder a(&col); + size_t j = 0; + for (auto const& server : servers) { + if (j++ < replicationFactor) { + col.add(VPackValue(dbsIds[server])); + } + }}}} +} + +void createPlanCollection( + std::string const& dbname, std::string const& colname, + size_t numberOfShards, size_t replicationFactor, Node& plan) { + + VPackBuilder tmp; + { VPackObjectBuilder o(&tmp); + createCollection(colname, tmp); + tmp.add("isSmart", VPackValue(false)); + tmp.add("deleted", VPackValue(false)); + createPlanShards(numberOfShards, replicationFactor, tmp);} + + Slice col = tmp.slice(); + auto id = col.get("id").copyString(); + plan(PLAN_COL_PATH + dbname + "/" + col.get("id").copyString()) = col; + +} + +void createLocalCollection( + std::string const& dbname, std::string const& colname, Node& node) { + + size_t planId = std::stoull(colname); + VPackBuilder tmp; + { VPackObjectBuilder o(&tmp); + createCollection(colname, tmp); + tmp.add("planId", VPackValue(colname)); + tmp.add("theLeader", VPackValue("")); + tmp.add("globallyUniqueId", + VPackValue(C + colname + "/" + S + std::to_string(planId+1))); + tmp.add("objectId", VPackValue("9031415")); } + node(dbname + "/" + S + std::to_string(planId+1)) = tmp.slice(); + +} + +std::map collectionMap(Node const& plan) { + std::map ret; + auto const pb = plan("Collections").toBuilder(); + auto const ps = pb.slice(); + for (auto const& db : VPackObjectIterator(ps)) { + for (auto const& col : VPackObjectIterator(db.value)) { + ret.emplace( + db.key.copyString()+"/"+col.value.get("name").copyString(), + col.key.copyString()); + } + } + return ret; +} + + + +namespace arangodb { +class LogicalCollection; +} + +TEST_CASE("ActionDescription", "[cluster][maintenance]") { + + SECTION("Construct minimal ActionDescription") { + ActionDescription desc(std::map{{"name", "SomeAction"}}); + REQUIRE(desc.get("name") == "SomeAction"); + } + + SECTION("Construct minimal ActionDescription with nullptr props") { + std::shared_ptr props; + ActionDescription desc({{"name", "SomeAction"}}, props); + } + + SECTION("Construct minimal ActionDescription with empty props") { + std::shared_ptr props; + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + } + + SECTION("Retrieve non-assigned key from ActionDescription") { + std::shared_ptr props; + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + try { + auto bogus = desc.get("bogus"); + REQUIRE(bogus == "bogus"); + } catch (std::out_of_range const& e) {} + std::string value; + auto res = desc.get("bogus", value); + REQUIRE(value.empty()); + REQUIRE(!res.ok()); + } + + SECTION("Retrieve non-assigned key from ActionDescription") { + std::shared_ptr props; + ActionDescription desc({{"name", "SomeAction"}, {"bogus", "bogus"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + try { + auto bogus = desc.get("bogus"); + REQUIRE(bogus == "bogus"); + } catch (std::out_of_range const& e) {} + std::string value; + auto res = desc.get("bogus", value); + REQUIRE(value == "bogus"); + REQUIRE(res.ok()); + } + + SECTION("Retrieve non-assigned properties from ActionDescription") { + std::shared_ptr props; + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties() == nullptr); + } + + SECTION("Retrieve empty properties from ActionDescription") { + auto props = std::make_shared(); + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties()->isEmpty()); + } + + SECTION("Retrieve empty object properties from ActionDescription") { + auto props = std::make_shared(); + { VPackObjectBuilder empty(props.get()); } + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties()->slice().isEmptyObject()); + } + + SECTION("Retrieve string value from ActionDescription's properties") { + auto props = std::make_shared(); + { VPackObjectBuilder obj(props.get()); + props->add("hello", VPackValue("world")); } + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties()->slice().hasKey("hello")); + REQUIRE(desc.properties()->slice().get("hello").copyString() == "world"); + } + + SECTION("Retrieve double value from ActionDescription's properties") { + double pi = 3.14159265359; + auto props = std::make_shared(); + { VPackObjectBuilder obj(props.get()); + props->add("pi", VPackValue(3.14159265359)); } + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties()->slice().hasKey("pi")); + REQUIRE( + desc.properties()->slice().get("pi").getNumber() == pi); + } + + SECTION("Retrieve integer value from ActionDescription's properties") { + size_t one = 1; + auto props = std::make_shared(); + { VPackObjectBuilder obj(props.get()); + props->add("one", VPackValue(one)); } + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties()->slice().hasKey("one")); + REQUIRE( + desc.properties()->slice().get("one").getNumber() == one); + } + + SECTION("Retrieve array value from ActionDescription's properties") { + double pi = 3.14159265359; + size_t one = 1; + std::string hello("hello world!"); + auto props = std::make_shared(); + { VPackObjectBuilder obj(props.get()); + props->add(VPackValue("array")); + { VPackArrayBuilder arr(props.get()); + props->add(VPackValue(pi)); + props->add(VPackValue(one)); + props->add(VPackValue(hello)); }} + ActionDescription desc({{"name", "SomeAction"}}, props); + REQUIRE(desc.get("name") == "SomeAction"); + REQUIRE(desc.properties()->slice().hasKey("array")); + REQUIRE(desc.properties()->slice().get("array").isArray()); + REQUIRE(desc.properties()->slice().get("array").length() == 3); + REQUIRE(desc.properties()->slice().get("array")[0].getNumber()==pi); + REQUIRE(desc.properties()->slice().get("array")[1].getNumber()==one); + REQUIRE(desc.properties()->slice().get("array")[2].copyString() == hello ); + } + +} + +TEST_CASE("ActionPhaseOne", "[cluster][maintenance]") { + + std::map localNodes { + {dbsIds[shortNames[0]], createNode(dbs0Str)}, + {dbsIds[shortNames[1]], createNode(dbs1Str)}, + {dbsIds[shortNames[2]], createNode(dbs2Str)}}; + + MaintenanceFeature::errors_t errors; + + SECTION("In sync should have 0 effects") { + + std::vector actions; + + for (auto const& node : localNodes) { + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + if (actions.size() != 0) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 0); + } + + } + + + SECTION("Local databases one more empty database should be dropped") { + + std::vector actions; + + localNodes.begin()->second("db3") = + arangodb::velocypack::Slice::emptyObjectSlice(); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), localNodes.begin()->second.toBuilder().slice(), + localNodes.begin()->first, errors, actions); + + if (actions.size() != 1) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 1); + REQUIRE(actions.front().name() == "DropDatabase"); + REQUIRE(actions.front().get("database") == "db3"); + + } + + + SECTION("Local databases one more non empty database should be dropped") { + + std::vector actions; + localNodes.begin()->second("db3/col") = + arangodb::velocypack::Slice::emptyObjectSlice(); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), localNodes.begin()->second.toBuilder().slice(), + localNodes.begin()->first, errors, actions); + + REQUIRE(actions.size() == 1); + REQUIRE(actions.front().name() == "DropDatabase"); + REQUIRE(actions.front().get("database") == "db3"); + + } + + + SECTION( + "Add one collection to db3 in plan with shards for all db servers") { + + std::string dbname("db3"), colname("x"); + + plan = originalPlan; + createPlanDatabase(dbname, plan); + createPlanCollection(dbname, colname, 1, 3, plan); + + for (auto node : localNodes) { + std::vector actions; + + node.second("db3") = + arangodb::velocypack::Slice::emptyObjectSlice(); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + if (actions.size() != 1) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 1); + for (auto const& action : actions) { + REQUIRE(action.name() == "CreateCollection"); + } + } + + } + + + SECTION("Add two more collection to db3 in plan with shards for all db servers") { + + std::string dbname("db3"), colname1("x"), colname2("y"); + + plan = originalPlan; + createPlanDatabase(dbname, plan); + createPlanCollection(dbname, colname1, 1, 3, plan); + createPlanCollection(dbname, colname2, 1, 3, plan); + + for (auto node : localNodes) { + std::vector actions; + + node.second("db3") = + arangodb::velocypack::Slice::emptyObjectSlice(); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + if (actions.size() != 2) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 2); + for (auto const& action : actions) { + REQUIRE(action.name() == "CreateCollection"); + } + } + + } + + + SECTION("Add an index to _queues") { + + plan = originalPlan; + auto cid = collectionMap(plan).at("_system/_queues"); + auto shards = plan({"Collections","_system",cid,"shards"}).children(); + + createPlanIndex( + "_system", cid, "hash", {"someField"}, false, false, false, plan); + + for (auto node : localNodes) { + std::vector actions; + + auto local = node.second; + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), local.toBuilder().slice(), node.first, errors, + actions); + + size_t n = 0; + for (auto const& shard : shards) { + if (local.has({"_system", shard.first})) { + ++n; + } + } + + if (actions.size() != n) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == n); + for (auto const& action : actions) { + REQUIRE(action.name() == "EnsureIndex"); + } + } + + } + + + SECTION("Remove an index from plan") { + + std::string dbname("_system"); + std::string indexes("indexes"); + + plan = originalPlan; + auto cid = collectionMap(plan).at("_system/bar"); + auto shards = plan({"Collections",dbname,cid,"shards"}).children(); + + plan({"Collections", dbname, cid, indexes}).handle( + arangodb::velocypack::Slice::emptyObjectSlice()); + + for (auto node : localNodes) { + std::vector actions; + + auto local = node.second; + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), local.toBuilder().slice(), node.first, errors, + actions); + + size_t n = 0; + for (auto const& shard : shards) { + if (local.has({"_system", shard.first})) { + ++n; + } + } + + if (actions.size() != n) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == n); + for (auto const& action : actions) { + REQUIRE(action.name() == "DropIndex"); + } + } + + } + + + SECTION("Add one collection to local") { + + plan = originalPlan; + + for (auto node : localNodes) { + + std::vector actions; + createLocalCollection("_system", "1111111", node.second); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + if (actions.size() != 1) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 1); + for (auto const& action : actions) { + REQUIRE(action.name() == "DropCollection"); + REQUIRE(action.get("database") == "_system"); + REQUIRE(action.get("collection") == "s1111112"); + } + } + + } + + + SECTION("Modify journalSize in plan should update the according collection") { + + VPackBuilder v; v.add(VPackValue(0)); + + for (auto node : localNodes) { + + std::vector actions; + std::string dbname = "_system"; + std::string prop = "journalSize"; + + auto cb = + node.second(dbname).children().begin()->second->toBuilder(); + auto collection = cb.slice(); + auto colname = collection.get(NAME).copyString(); + + (*node.second(dbname).children().begin()->second)(prop) = + v.slice(); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + if (actions.size() != 1) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 1); + for (auto const& action : actions) { + + REQUIRE(action.name() == "UpdateCollection"); + REQUIRE(action.get("collection") == colname); + REQUIRE(action.get("database") == dbname); + auto const props = action.properties(); + + } + + } + } + + + SECTION("Have theLeader set to empty") { + + VPackBuilder v; v.add(VPackValue(std::string())); + + for (auto node : localNodes) { + + std::vector actions; + + auto& collection = *node.second("foo").children().begin()->second; + auto& leader = collection("theLeader"); + + bool check = false; + if (!leader.getString().empty()) { + check = true; + leader = v.slice(); + } + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + if (check) { + if (actions.size() != 1) { + std::cout << __FILE__ << ":" << __LINE__ << " " << actions << std::endl; + } + REQUIRE(actions.size() == 1); + for (auto const& action : actions) { + REQUIRE(action.name() == "UpdateCollection"); + REQUIRE(action.has("collection")); + REQUIRE(action.get("collection") == collection("name").getString()); + REQUIRE(action.get("localLeader").empty()); + } + } + } + } + + + SECTION( + "Empty db3 in plan should drop all local db3 collections on all servers") { + + plan(PLAN_COL_PATH + "db3") = + arangodb::velocypack::Slice::emptyObjectSlice(); + + createPlanDatabase("db3", plan); + + for (auto& node : localNodes) { + + std::vector actions; + node.second("db3") = node.second("_system"); + + arangodb::maintenance::diffPlanLocal( + plan.toBuilder().slice(), node.second.toBuilder().slice(), + node.first, errors, actions); + + REQUIRE(actions.size() == node.second("db3").children().size()); + for (auto const& action : actions) { + REQUIRE(action.name() == "DropCollection"); + } + + } + + } + + + SECTION("Resign leadership") { + + plan = originalPlan; + std::string const dbname("_system"); + std::string const colname("bar"); + auto cid = collectionMap(plan).at(dbname + "/" + colname); + auto& shards = plan({"Collections",dbname,cid,"shards"}).children(); + + for (auto const& node : localNodes) { + std::vector actions; + + std::string shname; + + for (auto const& shard : shards) { + shname = shard.first; + Slice servers = shard.second->toBuilder().slice(); + + REQUIRE(servers.isArray()); + REQUIRE(servers.length() == 2); + auto const leader = servers[0].copyString(); + auto const follower = servers[1].copyString(); + + if (leader == node.first) { + + VPackBuilder newServers; + { VPackArrayBuilder a(&newServers); + newServers.add(VPackValue(std::string("_") + leader)); + newServers.add(VPackValue(follower)); } + plan({"Collections", dbname, cid, "shards", shname}) = newServers.slice(); + break; + } + } + + arangodb::maintenance::diffPlanLocal ( + plan.toBuilder().slice(), node.second.toBuilder().slice(), node.first, + errors, actions); + + if (actions.size() != 1) { + std::cout << actions << std::endl; + } + REQUIRE(actions.size() == 1); + REQUIRE(actions.front().name() == "ResignShardLeadership"); + REQUIRE(actions.front().get(DATABASE) == dbname); + REQUIRE(actions.front().get(SHARD) == shname); + } + + } + +} + +TEST_CASE("ActionPhaseTwo", "[cluster][maintenance]") { + + plan = originalPlan; + + std::map localNodes { + {dbsIds[shortNames[0]], createNode(dbs0Str)}, + {dbsIds[shortNames[1]], createNode(dbs1Str)}, + {dbsIds[shortNames[2]], createNode(dbs2Str)}}; + + +/* SECTION("Diffing local and current in equilibrium") { + + VPackSlice p = plan.toBuilder().slice(); + REQUIRE(p.hasKey("Collections")); + REQUIRE(p.get("Collections").isObject()); + VPackSlice c = current.toBuilder().slice(); + + for (auto const& node : localNodes) { + + VPackSlice l = node.second.toBuilder().slice(); + REQUIRE(c.isObject()); + REQUIRE(p.isObject()); + REQUIRE(l.isObject()); + + VPackBuilder rb; + { VPackObjectBuilder o(&rb); + rb.add(VPackValue("phaseTwo")); + { VPackObjectBuilder oo(&rb); + arangodb::maintenance::reportInCurrent(p, c, l, node.first, rb); }} + + VPackSlice report = rb.slice(); + + VPackSlice pt = report.get("PhaseTwo"); + REQUIRE(pt.isObject()); + + if (!pt.isEmptyObject()) { + std::cout << pt.toJson() << std::endl; + } + REQUIRE(pt.isEmptyObject()); + + } + } +*/ +} + + +#endif diff --git a/tests/Maintenance/Plan.json b/tests/Maintenance/Plan.json new file mode 100644 index 0000000000..4f9650381c --- /dev/null +++ b/tests/Maintenance/Plan.json @@ -0,0 +1,1480 @@ +R"=( +{ + "Databases": { + "_system": { + "id": "1", + "name": "_system" + }, + "foo": { + "id": "2010068", + "coordinator": "CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8", + "options": {}, + "name": "foo" + } + }, + "Collections": { + "_system": { + "2010049": { + "status": 3, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "isSmart": false, + "id": "2010049", + "isSystem": false, + "deleted": false, + "type": 3, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from" + ], + "id": "1", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "fields": [ + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "fields": [ + "gi" + ], + "geoJson": false, + "id": "2010116", + "sparse": true, + "type": "geo", + "unique": false + }, + { + "fields": [ + "gij" + ], + "geoJson": true, + "id": "2010122", + "sparse": true, + "type": "geo", + "unique": false + }, + { + "deduplicate": false, + "fields": [ + "hi", + "_key" + ], + "id": "2010126", + "sparse": false, + "type": "hash", + "unique": true + }, + { + "deduplicate": false, + "fields": [ + "pi" + ], + "id": "2010130", + "sparse": true, + "type": "persistent", + "unique": false + }, + { + "fields": [ + "fi" + ], + "id": "2010132", + "minLength": 3, + "sparse": true, + "type": "fulltext", + "unique": false + }, + { + "deduplicate": true, + "fields": [ + "sli" + ], + "id": "2010136", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "name": "bar", + "shardingStrategy": "hash", + "statusString": "loaded", + "numberOfShards": 9, + "replicationFactor": 2, + "shardKeys": [ + "_key" + ], + "shards": { + "s2010055": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010051": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010053": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010057": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010056": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010052": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010054": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010058": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010050": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + } + }, + "2010019": { + "status": 3, + "shards": { + "s2010020": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010019", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_queues", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010034": { + "status": 3, + "shards": { + "s2010035": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010034", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "2010036", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "name": "_statistics", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010001": { + "status": 3, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "isSmart": false, + "id": "2010001", + "isSystem": true, + "deleted": false, + "type": 2, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_graphs", + "shardingStrategy": "hash", + "statusString": "loaded", + "numberOfShards": 1, + "replicationFactor": 2, + "shardKeys": [ + "_key" + ], + "shards": { + "s2010002": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + } + }, + "2010006": { + "status": 3, + "shards": { + "s2010007": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010006", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_modules", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010008": { + "status": 3, + "shards": { + "s2010009": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010008", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_iresearch_analyzers", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010010": { + "status": 3, + "shards": { + "s2010011": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010010", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_routing", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010015": { + "status": 3, + "shards": { + "s2010016": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010015", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_aqlfunctions", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010021": { + "status": 3, + "shards": { + "s2010022": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010021", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "2010023", + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "2010024", + "sparse": true, + "type": "skiplist", + "unique": true + } + ], + "name": "_jobs", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010061": { + "status": 3, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "isSmart": false, + "id": "2010061", + "isSystem": false, + "deleted": false, + "type": 2, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "baz", + "shardingStrategy": "hash", + "statusString": "loaded", + "numberOfShards": 4, + "replicationFactor": 3, + "shardKeys": [ + "_key" + ], + "shards": { + "s2010063": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010062": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010065": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010064": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ] + } + }, + "2010025": { + "status": 3, + "shards": { + "s2010026": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010025", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "2010027", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "name": "_apps", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010028": { + "status": 3, + "shards": { + "s2010029": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010028", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_appbundles", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010003": { + "status": 3, + "shards": { + "s2010004": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010003", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "user" + ], + "id": "2010005", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "name": "_users", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010031": { + "status": 3, + "shards": { + "s2010032": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010031", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "2010033", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "name": "_statisticsRaw", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010017": { + "status": 3, + "shards": { + "s2010018": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010017", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_frontend", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010037": { + "status": 3, + "shards": { + "s2010038": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010037", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "time" + ], + "id": "2010039", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "name": "_statistics15", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010045": { + "status": 3, + "shards": { + "s2010046": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010045", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010001", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_fishbowl", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + } + }, + "foo": { + "2010082": { + "status": 3, + "shards": { + "s2010083": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010082", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_queues", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010069": { + "status": 3, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "isSmart": false, + "id": "2010069", + "isSystem": true, + "deleted": false, + "type": 2, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_graphs", + "shardingStrategy": "hash", + "statusString": "loaded", + "numberOfShards": 1, + "replicationFactor": 2, + "shardKeys": [ + "_key" + ], + "shards": { + "s2010070": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + } + }, + "2010080": { + "status": 3, + "shards": { + "s2010081": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010080", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_frontend", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010078": { + "status": 3, + "shards": { + "s2010079": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010078", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_aqlfunctions", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010071": { + "status": 3, + "shards": { + "s2010072": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010071", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_modules", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010091": { + "status": 3, + "shards": { + "s2010092": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010091", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_appbundles", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010088": { + "status": 3, + "shards": { + "s2010089": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010088", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "mount" + ], + "id": "2010090", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "name": "_apps", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010097": { + "status": 3, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "isSmart": false, + "id": "2010097", + "isSystem": false, + "deleted": false, + "type": 3, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "fields": [ + "_from" + ], + "id": "1", + "sparse": false, + "type": "edge", + "unique": false + }, + { + "fields": [ + "_to" + ], + "id": "2", + "sparse": false, + "type": "edge", + "unique": false + } + ], + "name": "foobar", + "shardingStrategy": "hash", + "statusString": "loaded", + "numberOfShards": 4, + "replicationFactor": 3, + "shardKeys": [ + "_key" + ], + "shards": { + "s2010099": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010100": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010101": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010098": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ] + } + }, + "2010102": { + "status": 3, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "isSmart": false, + "id": "2010102", + "isSystem": false, + "deleted": false, + "type": 2, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "foobaz", + "shardingStrategy": "hash", + "statusString": "loaded", + "numberOfShards": 9, + "replicationFactor": 2, + "shardKeys": [ + "_key" + ], + "shards": { + "s2010107": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010106": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010109": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010104": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010108": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010110": [ + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291", + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83" + ], + "s2010111": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ], + "s2010103": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ], + "s2010105": [ + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89" + ] + } + }, + "2010084": { + "status": 3, + "shards": { + "s2010085": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010084", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "2010086", + "sparse": true, + "type": "skiplist", + "unique": true + }, + { + "deduplicate": true, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "2010087", + "sparse": true, + "type": "skiplist", + "unique": true + } + ], + "name": "_jobs", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + }, + "2010073": { + "status": 3, + "shards": { + "s2010074": [ + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291" + ] + }, + "keyOptions": { + "lastValue": 0, + "type": "traditional", + "allowUserKeys": true + }, + "cacheEnabled": false, + "waitForSync": false, + "id": "2010073", + "isSmart": false, + "type": 2, + "numberOfShards": 1, + "deleted": false, + "distributeShardsLike": "2010069", + "isSystem": true, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "name": "_routing", + "shardingStrategy": "hash", + "statusString": "loaded", + "replicationFactor": 2, + "shardKeys": [ + "_key" + ] + } + } + }, + "AsyncReplication": {}, + "Views": { + "_system": {} + }, + "Coordinators": { + "CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8": "none" + }, + "Version": 50, + "Lock": "UNLOCKED", + "DBServers": { + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83": "none", + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291": "none", + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89": "none" + }, + "Singles": {} +} +)=" diff --git a/tests/Maintenance/Supervision.json b/tests/Maintenance/Supervision.json new file mode 100644 index 0000000000..708be555e1 --- /dev/null +++ b/tests/Maintenance/Supervision.json @@ -0,0 +1,44 @@ +R"=( +{ + "Health": { + "PRMR-1b81e8d4-7119-49ba-a423-8684d59dde89": { + "Host": "ac8ddefc7d1f4364ba655b4debcd076f", + "Status": "GOOD", + "Endpoint": "tcp://[::1]:8629", + "Timestamp": "2018-08-15T16:07:46Z", + "ShortName": "DBServer0001", + "SyncStatus": "SERVING" + }, + "PRMR-57aa3986-2e78-4810-8000-7fd2f0693291": { + "Endpoint": "tcp://[::1]:8630", + "Host": "ac8ddefc7d1f4364ba655b4debcd076f", + "Status": "GOOD", + "Timestamp": "2018-08-16T08:16:44Z", + "ShortName": "DBServer0002", + "SyncStatus": "SERVING" + }, + "PRMR-e786c6cb-92fc-45cc-85dd-71abafcdaa83": { + "Endpoint": "tcp://[::1]:8631", + "Host": "ac8ddefc7d1f4364ba655b4debcd076f", + "Status": "GOOD", + "Timestamp": "2018-08-16T08:16:44Z", + "ShortName": "DBServer0003", + "SyncStatus": "SERVING" + }, + "CRDN-42df19c3-73d5-48f4-b02e-09b29008eff8": { + "Endpoint": "tcp://[::1]:8530", + "Host": "ac8ddefc7d1f4364ba655b4debcd076f", + "Status": "GOOD", + "Timestamp": "2018-08-16T08:16:44Z", + "ShortName": "Coordinator0001", + "SyncStatus": "SERVING" + } + }, + "DBServers": {}, + "State": { + "Mode": "Normal", + "Timestamp": "2018-08-15T16:07:46Z" + }, + "Shards": {} +} +)="