Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fetch commands for server from agency

This commit is contained in:
Jan Steemann 2013-12-09 17:44:19 +01:00
parent 2b06a0778d
commit 996be652f1
8 changed files with 332 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
arangod/Sharding/AgencyComm.cpp
View File

@ -27,6 +27,7 @@
#include "Sharding/AgencyComm.h"
#include "Basics/ReadLocker.h"
#include "Basics/StringUtils.h"
#include "Basics/WriteLocker.h"
#include "BasicsC/json.h"
#include "BasicsC/logging.h"
@ -105,7 +106,8 @@ AgencyCommResult::~AgencyCommResult () {
bool AgencyCommResult::processJsonNode (TRI_json_t const* node,
std::map<std::string, std::string>& out,
std::string const& stripPrefix) {
std::string const& stripKeyPrefix,
bool returnIndex) const {
if (! TRI_IsArrayJson(node)) {
return true;
}
@ -118,7 +120,7 @@ bool AgencyCommResult::processJsonNode (TRI_json_t const* node,
}
// make sure we don't strip more bytes than the key is long
const size_t offset = AgencyComm::_globalPrefix.size() + stripPrefix.size();
const size_t offset = AgencyComm::_globalPrefix.size() + stripKeyPrefix.size();
const size_t length = key->_value._string.length - 1;
std::string prefix;
@ -146,7 +148,10 @@ bool AgencyCommResult::processJsonNode (TRI_json_t const* node,
const size_t n = TRI_LengthVector(&nodes->_value._objects);
for (size_t i = 0; i < n; ++i) {
if (! processJsonNode((TRI_json_t const*) TRI_AtVector(&nodes->_value._objects, i), out, stripPrefix)) {
if (! processJsonNode((TRI_json_t const*) TRI_AtVector(&nodes->_value._objects, i),
out,
stripKeyPrefix,
returnIndex)) {
return false;
}
}
@ -162,7 +167,21 @@ bool AgencyCommResult::processJsonNode (TRI_json_t const* node,
}
if (! prefix.empty()) {
out[prefix] = std::string(value->_value._string.data, value->_value._string.length - 1);
if (returnIndex) {
// return "modifiedIndex"
TRI_json_t const* modifiedIndex = TRI_LookupArrayJson(node, "modifiedIndex");
if (! TRI_IsNumberJson(modifiedIndex)) {
return false;
}
// convert the number to an integer
out[prefix] = triagens::basics::StringUtils::itoa((uint64_t) modifiedIndex->_value._number);
}
else {
// otherwise return value
out[prefix] = std::string(value->_value._string.data, value->_value._string.length - 1);
}
}
}
@ -174,7 +193,8 @@ bool AgencyCommResult::processJsonNode (TRI_json_t const* node,
////////////////////////////////////////////////////////////////////////////////
bool AgencyCommResult::flattenJson (std::map<std::string, std::string>& out,
std::string const& stripPrefix) {
std::string const& stripKeyPrefix,
bool returnIndex) const {
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, _body.c_str());
if (! TRI_IsArrayJson(json)) {
@ -187,7 +207,7 @@ bool AgencyCommResult::flattenJson (std::map<std::string, std::string>& out,
// get "node" attribute
TRI_json_t const* node = TRI_LookupArrayJson(json, "node");
const bool result = processJsonNode(node, out, stripPrefix);
const bool result = processJsonNode(node, out, stripKeyPrefix, returnIndex);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return result;
@ -547,6 +567,7 @@ bool AgencyComm::setValue (std::string const& key,
send(agencyEndpoint->_connection,
triagens::rest::HttpRequest::HTTP_REQUEST_PUT,
_globalConnectionOptions._requestTimeout,
result,
buildUrl(key),
"value=" + value);
@ -590,6 +611,7 @@ AgencyCommResult AgencyComm::getValues (std::string const& key,
send(agencyEndpoint->_connection,
triagens::rest::HttpRequest::HTTP_REQUEST_GET,
_globalConnectionOptions._requestTimeout * 1000.0 * 1000.0,
result,
url);
@ -632,6 +654,7 @@ bool AgencyComm::removeValues (std::string const& key,
send(agencyEndpoint->_connection,
triagens::rest::HttpRequest::HTTP_REQUEST_DELETE,
_globalConnectionOptions._requestTimeout,
result,
url);
@ -664,12 +687,45 @@ int AgencyComm::casValue (std::string const& key,
/// @brief blocks on a change of a single value in the backend
////////////////////////////////////////////////////////////////////////////////
AgencyCommResult AgencyComm::watchValues (std::string const& key,
double timeout) {
AgencyCommResult AgencyComm::watchValue (std::string const& key,
uint64_t waitIndex,
double timeout) {
std::string url(buildUrl(key));
url += "?wait=true";
if (waitIndex > 0) {
url += "&waitIndex=" + triagens::basics::StringUtils::itoa(waitIndex);
}
AgencyCommResult result;
size_t numEndpoints;
// TODO
{
READ_LOCKER(AgencyComm::_globalLock);
numEndpoints = AgencyComm::_globalEndpoints.size();
assert(numEndpoints > 0);
}
size_t tries = 0;
while (tries++ < numEndpoints) {
AgencyEndpoint* agencyEndpoint = popEndpoint();
send(agencyEndpoint->_connection,
triagens::rest::HttpRequest::HTTP_REQUEST_GET,
timeout == 0.0 ? _globalConnectionOptions._requestTimeout : timeout,
result,
url);
if (requeueEndpoint(agencyEndpoint, result.successful())) {
// we're done
break;
}
// otherwise, try next
}
// if we get here, we could not send data to any endpoint successfully
return result;
}
@ -765,6 +821,7 @@ std::string AgencyComm::buildUrl (std::string const& relativePart) const {
bool AgencyComm::send (triagens::httpclient::GeneralClientConnection* connection,
triagens::rest::HttpRequest::HttpRequestType method,
double timeout,
AgencyCommResult& result,
std::string const& url) {
// only these methods can be called without a body
@ -772,7 +829,7 @@ bool AgencyComm::send (triagens::httpclient::GeneralClientConnection* connection
method == triagens::rest::HttpRequest::HTTP_REQUEST_GET ||
method == triagens::rest::HttpRequest::HTTP_REQUEST_HEAD);
return send(connection, method, result, url, "");
return send(connection, method, timeout, result, url, "");
}
////////////////////////////////////////////////////////////////////////////////
@ -781,6 +838,7 @@ bool AgencyComm::send (triagens::httpclient::GeneralClientConnection* connection
bool AgencyComm::send (triagens::httpclient::GeneralClientConnection* connection,
triagens::rest::HttpRequest::HttpRequestType method,
double timeout,
AgencyCommResult& result,
std::string const& url,
std::string const& body) {
@ -795,8 +853,8 @@ bool AgencyComm::send (triagens::httpclient::GeneralClientConnection* connection
url.c_str(),
body.c_str());
triagens::httpclient::SimpleHttpClient client(connection,
_globalConnectionOptions._requestTimeout,
triagens::httpclient::SimpleHttpClient client(connection,
timeout,
false);
// set up headers

31
arangod/Sharding/AgencyComm.h
View File

@ -121,10 +121,12 @@ namespace triagens {
bool processJsonNode (struct TRI_json_s const*,
std::map<std::string, std::string>&,
std::string const&);
std::string const&,
bool) const;
bool flattenJson (std::map<std::string, std::string>&,
std::string const&);
std::string const&,
bool) const;
std::string _message;
std::string _body;
@ -226,37 +228,38 @@ namespace triagens {
/// @brief sets a value in the back end
////////////////////////////////////////////////////////////////////////////////
bool setValue (std::string const& key,
std::string const& value);
bool setValue (std::string const&,
std::string const&);
////////////////////////////////////////////////////////////////////////////////
/// @brief gets one or multiple values from the back end
////////////////////////////////////////////////////////////////////////////////
AgencyCommResult getValues (std::string const& key,
bool recursive);
AgencyCommResult getValues (std::string const&,
bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief removes one or multiple values from the back end
////////////////////////////////////////////////////////////////////////////////
bool removeValues (std::string const& key,
bool recursive);
bool removeValues (std::string const&,
bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief compares and swaps a single value in the back end
////////////////////////////////////////////////////////////////////////////////
int casValue (std::string const& key,
std::string const& oldValue,
std::string const& newValue);
int casValue (std::string const&,
std::string const&,
std::string const&);
////////////////////////////////////////////////////////////////////////////////
/// @brief blocks on a change of a single value in the back end
////////////////////////////////////////////////////////////////////////////////
AgencyCommResult watchValues (std::string const& key,
double timeout);
AgencyCommResult watchValue (std::string const&,
uint64_t,
double);
// -----------------------------------------------------------------------------
// --SECTION-- private methods
@ -289,6 +292,7 @@ namespace triagens {
bool send (triagens::httpclient::GeneralClientConnection*,
triagens::rest::HttpRequest::HttpRequestType,
double,
AgencyCommResult&,
std::string const&);
@ -298,6 +302,7 @@ namespace triagens {
bool send (triagens::httpclient::GeneralClientConnection*,
triagens::rest::HttpRequest::HttpRequestType,
double,
AgencyCommResult&,
std::string const&,
std::string const&);

67
arangod/Sharding/ApplicationSharding.cpp
View File

@ -89,7 +89,7 @@ void ApplicationSharding::setupOptions (map<string, basics::ProgramOptionsDescri
}
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
/// @brief prepare validate the startup options
////////////////////////////////////////////////////////////////////////////////
bool ApplicationSharding::prepare () {
@ -119,7 +119,8 @@ bool ApplicationSharding::prepare () {
const string unified = triagens::rest::Endpoint::getUnifiedForm(_agencyEndpoints[i]);
if (unified.empty()) {
LOG_FATAL_AND_EXIT("invalid endpoint '%s' specified for --cluster.agency-endpoint", _agencyEndpoints[i].c_str());
LOG_FATAL_AND_EXIT("invalid endpoint '%s' specified for --cluster.agency-endpoint",
_agencyEndpoints[i].c_str());
}
AgencyComm::addEndpoint(unified);
@ -172,16 +173,38 @@ bool ApplicationSharding::start () {
if (role == ServerState::ROLE_UNDEFINED) {
// no role found
LOG_FATAL_AND_EXIT("unable to determine unambiguous role for server '%s'. No role configured at endpoints (%s)",
LOG_FATAL_AND_EXIT("unable to determine unambiguous role for server '%s'. No role configured in agency (%s)",
_myId.c_str(),
endpoints.c_str());
}
// check if my-address is set
if (_myAddress.empty()) {
// no address given, now ask the agency for out address
_myAddress = getEndpointForId();
}
if (_myAddress.empty()) {
LOG_FATAL_AND_EXIT("unable to determine internal address for server '%s'. "
"Please specify --cluster.my-address or configure the address for this server in the agency.",
_myId.c_str());
}
// now we can validate --cluster.my-address
const string unified = triagens::rest::Endpoint::getUnifiedForm(_myAddress);
if (unified.empty()) {
LOG_FATAL_AND_EXIT("invalid endpoint '%s' specified for --cluster.my-address",
_myAddress.c_str());
}
ServerState::instance()->setRole(role);
ServerState::instance()->setState(ServerState::STATE_STARTUP);
LOG_INFO("Cluster feature is turned on. Server id: '%s', role: %s, agency endpoints: %s",
LOG_INFO("Cluster feature is turned on. Server id: '%s', internal address: %s, role: %s, agency endpoints: %s",
_myId.c_str(),
_myAddress.c_str(),
ServerState::roleToString(role).c_str(),
endpoints.c_str());
@ -230,6 +253,38 @@ void ApplicationSharding::stop () {
// --SECTION-- private methods
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup the server's endpoint by scanning Config/MapIDToEnpdoint for
/// our id
////////////////////////////////////////////////////////////////////////////////
std::string ApplicationSharding::getEndpointForId () const {
// fetch value at Config/MapIDToEndpoint
AgencyComm comm;
AgencyCommResult result = comm.getValues("Config/MapIDToEndpoint/" + _myId, false);
if (result.successful()) {
std::map<std::string, std::string> out;
if (! result.flattenJson(out, "Config/MapIDToEndpoint/", false)) {
LOG_FATAL_AND_EXIT("Got an invalid JSON response for Config/MapIDToEndpoint");
}
// check if we can find ourselves in the list returned by the agency
std::map<std::string, std::string>::const_iterator it = out.find(_myId);
if (it != out.end()) {
LOG_TRACE("using remote value '%s' for --cluster.my-address",
(*it).second.c_str());
return (*it).second;
}
}
// not found
return "";
}
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup the server role by scanning TmpConfig/Coordinators for our id
////////////////////////////////////////////////////////////////////////////////
@ -249,7 +304,7 @@ ServerState::RoleEnum ApplicationSharding::checkCoordinatorsList () const {
}
std::map<std::string, std::string> out;
if (! result.flattenJson(out, "TmpConfig/Coordinators/")) {
if (! result.flattenJson(out, "TmpConfig/Coordinators/", false)) {
LOG_FATAL_AND_EXIT("Got an invalid JSON response for TmpConfig/Coordinators");
}
@ -283,7 +338,7 @@ ServerState::RoleEnum ApplicationSharding::checkServersList () const {
}
std::map<std::string, std::string> out;
if (! result.flattenJson(out, "TmpConfig/DBServers/")) {
if (! result.flattenJson(out, "TmpConfig/DBServers/", false)) {
LOG_FATAL_AND_EXIT("Got an invalid JSON response for TmpConfig/DBServers");
}

7
arangod/Sharding/ApplicationSharding.h
View File

@ -114,6 +114,13 @@ namespace triagens {
// --SECTION-- private methods
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup the server's endpoint by scanning Config/MapIDToEnpdoint for
/// our id
////////////////////////////////////////////////////////////////////////////////
std::string getEndpointForId () const;
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup the server role by scanning TmpConfig/Coordinators for our id
////////////////////////////////////////////////////////////////////////////////

137
arangod/Sharding/HeartbeatThread.cpp
View File

@ -72,21 +72,66 @@ HeartbeatThread::~HeartbeatThread () {
////////////////////////////////////////////////////////////////////////////////
/// @brief heartbeat main loop
/// the heartbeat thread constantly reports the current server status to the
/// agency. it does so by sending the current state string to the key
/// "State/ServerStates/" + my-id.
/// after transferring the current state to the agency, the heartbeat thread
/// will wait for changes on the "Commands/" + my-id key. If no changes occur,
/// then the request it aborted and the heartbeat thread will go on with
/// reporting its state to the agency again. If it notices a change when
/// watching the command key, it will wake up and apply the change locally.
////////////////////////////////////////////////////////////////////////////////
void HeartbeatThread::run () {
LOG_TRACE("starting heartbeat thread");
while (! _stop) {
LOG_TRACE("sending heartbeat");
// convert timeout to seconds
const double interval = (double) _interval / 1000.0 / 1000.0;
// value of /Commands/my-id at startup
uint64_t lastCommandIndex = getLastCommandIndex();
while (! _stop) {
LOG_TRACE("sending heartbeat to agency");
// send our state to the agency.
// we don't care if this fails
sendState();
if (_stop) {
break;
}
// watch Commands/my-id for changes
// TODO: check if this is CPU-intensive and whether we need to sleep
AgencyCommResult result = _agency.watchValue("Commands/" + _myId,
lastCommandIndex + 1,
interval);
if (_stop) {
break;
}
if (result.successful()) {
// value has changed!
handleStateChange(result, lastCommandIndex);
// sleep a while
CONDITION_LOCKER(guard, _condition);
guard.wait(_interval);
CONDITION_LOCKER(guard, _condition);
guard.wait(_interval);
}
else {
// value did not change, but we already blocked waiting for a change...
// nothing to do here
}
}
// another thread is waiting for this value to shut down properly
_stop = 2;
LOG_TRACE("stopping heartbeat thread");
LOG_TRACE("stopped heartbeat thread");
}
// -----------------------------------------------------------------------------
@ -107,16 +152,94 @@ bool HeartbeatThread::init () {
return true;
}
// -----------------------------------------------------------------------------
// --SECTION-- private methods
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief fetch the index id of the value of /Commands/my-id from the agency
/// this index value is determined initially and it is passed to the watch
/// command (we're waiting for an entry with a higher id)
////////////////////////////////////////////////////////////////////////////////
uint64_t HeartbeatThread::getLastCommandIndex () {
// get the initial command state
AgencyCommResult result = _agency.getValues("Commands/" + _myId, false);
if (result.successful()) {
std::map<std::string, std::string> out;
if (result.flattenJson(out, "Commands/", true)) {
// check if we can find ourselves in the list returned by the agency
std::map<std::string, std::string>::const_iterator it = out.find(_myId);
if (it != out.end()) {
// found something
LOG_TRACE("last command index was: '%s'", (*it).second.c_str());
return triagens::basics::StringUtils::uint64((*it).second);
}
}
}
// nothing found. this is not an error
return 0;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief handles a state change
/// this is triggered if the watch command reports a change
/// when this is called, it will update the index value of the last command
/// (we'll pass the updated index value to the next watches so we don't get
/// notified about this particular change again).
////////////////////////////////////////////////////////////////////////////////
bool HeartbeatThread::handleStateChange (AgencyCommResult const& result,
uint64_t& lastCommandIndex) {
std::map<std::string, std::string> out;
if (result.flattenJson(out, "Commands/", true)) {
// get the new value of "modifiedIndex"
std::map<std::string, std::string>::const_iterator it = out.find(_myId);
if (it != out.end()) {
lastCommandIndex = triagens::basics::StringUtils::uint64((*it).second);
}
}
out.clear();
if (result.flattenJson(out, "Commands/", false)) {
// get the new value!
std::map<std::string, std::string>::const_iterator it = out.find(_myId);
if (it != out.end()) {
const std::string command = (*it).second;
ServerState::StateEnum newState = ServerState::stringToState(command);
if (newState != ServerState::STATE_UNDEFINED) {
// state change.
ServerState::instance()->setState(newState);
return true;
}
}
}
return false;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief sends the current server's state to the agency
////////////////////////////////////////////////////////////////////////////////
bool HeartbeatThread::sendState () {
const std::string value = ServerState::stateToString(ServerState::instance()->getState()) + ":" + AgencyComm::generateStamp();
const std::string value = ServerState::stateToString(ServerState::instance()->getState()) +
":" +
AgencyComm::generateStamp();
// return value is intentionally not handled
// if sending the current state fails, we'll just try again in the next iteration
bool result = _agency.setValue("state/servers/state/" + _myId, value);
bool result = _agency.setValue("State/ServerStates/" + _myId, value);
if (result) {
_numFails = 0;

24
arangod/Sharding/HeartbeatThread.h
View File

@ -31,6 +31,7 @@
#include "Basics/Common.h"
#include "Basics/ConditionVariable.h"
#include "Basics/Thread.h"
#include "BasicsC/logging.h"
#include "Sharding/AgencyComm.h"
#ifdef __cplusplus
@ -88,8 +89,18 @@ namespace triagens {
////////////////////////////////////////////////////////////////////////////////
void stop () {
if (_stop > 0) {
return;
}
LOG_TRACE("stopping heartbeat thread");
_stop = 1;
_condition.signal();
while (_stop != 2) {
usleep(1000);
}
}
// -----------------------------------------------------------------------------
@ -110,6 +121,19 @@ namespace triagens {
private:
////////////////////////////////////////////////////////////////////////////////
/// @brief handles a state change
////////////////////////////////////////////////////////////////////////////////
bool handleStateChange (AgencyCommResult const&,
uint64_t&);
////////////////////////////////////////////////////////////////////////////////
/// @brief fetch the last value of /Commands/my-id from the agency
////////////////////////////////////////////////////////////////////////////////
uint64_t getLastCommandIndex ();
////////////////////////////////////////////////////////////////////////////////
/// @brief sends the current server's state to the agency
////////////////////////////////////////////////////////////////////////////////

21
arangod/Sharding/ServerState.cpp
View File

@ -99,6 +99,19 @@ std::string ServerState::roleToString (RoleEnum role) {
return "";
}
////////////////////////////////////////////////////////////////////////////////
/// @brief convert a string representation to a state
////////////////////////////////////////////////////////////////////////////////
ServerState::StateEnum ServerState::stringToState (std::string const& value) {
if (value == "SHUTDOWN") {
return STATE_SHUTDOWN;
}
// TODO: do we need to understand other states, too?
return STATE_UNDEFINED;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the string representation of a state
////////////////////////////////////////////////////////////////////////////////
@ -166,10 +179,10 @@ void ServerState::setState (StateEnum state) {
}
if (result) {
LOG_TRACE("changing state of %s server from %s to %s",
ServerState::roleToString(_role).c_str(),
ServerState::stateToString(_state).c_str(),
ServerState::stateToString(state).c_str());
LOG_INFO("changing state of %s server from %s to %s",
ServerState::roleToString(_role).c_str(),
ServerState::stateToString(_state).c_str(),
ServerState::stateToString(state).c_str());
_state = state;
}

5
arangod/Sharding/ServerState.h
View File

@ -124,6 +124,11 @@ namespace triagens {
static std::string stateToString (StateEnum);
////////////////////////////////////////////////////////////////////////////////
/// @brief convert a string representation to a state
////////////////////////////////////////////////////////////////////////////////
static StateEnum stringToState (std::string const&);
// -----------------------------------------------------------------------------
// --SECTION-- public methods