Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Included support for multiple shards

This commit is contained in:
Simon Grätzer 2016-10-13 16:41:17 +02:00
parent f67acf39be
commit ba437cab71
12 changed files with 350 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arangod/CMakeLists.txt
View File

@ -377,6 +377,7 @@ add_executable(${BIN_ARANGOD}
Pregel/Vertex.cpp Pregel/Vertex.cpp
Pregel/Worker.cpp Pregel/Worker.cpp
Pregel/WorkerContext.cpp Pregel/WorkerContext.cpp
Pregel/WorkerJob.cpp
Pregel/GraphState.cpp Pregel/GraphState.cpp
Pregel/Utils.cpp Pregel/Utils.cpp
${ADDITIONAL_BIN_ARANGOD_SOURCES} ${ADDITIONAL_BIN_ARANGOD_SOURCES}

9
arangod/Pregel/Conductor.cpp
View File

@ -89,8 +89,11 @@ void Conductor::start() {
VPackBuilder b; VPackBuilder b;
b.openObject(); b.openObject();
b.add(Utils::executionNumberKey, VPackValue(_executionNumber)); b.add(Utils::executionNumberKey, VPackValue(_executionNumber));
b.add(Utils::globalSuperstepKey, VPackValue(0));
b.add(Utils::algorithmKey, VPackValue(_algorithm));
b.add(Utils::coordinatorIdKey, VPackValue(coordinatorId)); b.add(Utils::coordinatorIdKey, VPackValue(coordinatorId));
b.add(Utils::vertexCollectionKey, VPackValue(_vertexCollection->name())); b.add(Utils::vertexCollectionNameKey, VPackValue(_vertexCollection->name()));
b.add(Utils::vertexCollectionPlanIdKey, VPackValue(_vertexCollection->planId_as_string()));
b.add(Utils::vertexShardsListKey, VPackValue(VPackValueType::Array)); b.add(Utils::vertexShardsListKey, VPackValue(VPackValueType::Array));
for (ShardID const &vit : it.second) { for (ShardID const &vit : it.second) {
b.add(VPackValue(vit)); b.add(VPackValue(vit));
@ -101,10 +104,6 @@ void Conductor::start() {
b.add(VPackValue(eit)); b.add(VPackValue(eit));
} }
b.close(); b.close();
//b.add(Utils::vertexCollectionKey, VPackValue(_vertexCollection));
//b.add(Utils::edgeCollectionKey, VPackValue(_edgeCollection));
b.add(Utils::globalSuperstepKey, VPackValue(0));
b.add(Utils::algorithmKey, VPackValue(_algorithm));
b.close(); b.close();
auto body = std::make_shared<std::string const>(b.toJson()); auto body = std::make_shared<std::string const>(b.toJson());

9
arangod/Pregel/OutMessageCache.cpp
View File

@ -33,6 +33,8 @@
#include <velocypack/Iterator.h> #include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h> #include <velocypack/velocypack-aliases.h>
#include <algorithm>
using namespace arangodb; using namespace arangodb;
using namespace arangodb::pregel; using namespace arangodb::pregel;
@ -121,15 +123,16 @@ void OutMessageCache::getMessages(ShardID const& shardId, VPackBuilder &outBuild
} }
void OutMessageCache::sendMessages() { void OutMessageCache::sendMessages() {
LOG(INFO) << "Sending messages to shards"; LOG(INFO) << "Sending messages to other machines";
auto localShards = _ctx->localVertexShardIDs();
std::shared_ptr<std::vector<ShardID>> shards = _ci->getShardList(_ctx->vertexCollectionPlanId()); std::shared_ptr<std::vector<ShardID>> shards = _ci->getShardList(_ctx->vertexCollectionPlanId());
LOG(INFO) << "Seeing shards: " << shards->size();
std::vector<ClusterCommRequest> requests; std::vector<ClusterCommRequest> requests;
for (auto const &it : *shards) { for (auto const &it : *shards) {
if (_ctx->vertexShardId() == it) { if (std::find(localShards.begin(), localShards.end(), it) != localShards.end()) {
LOG(INFO) << "Worker: Getting messages for myself"; LOG(INFO) << "Worker: Getting messages for myself";
VPackBuilder messages; VPackBuilder messages;
messages.openArray(); messages.openArray();
getMessages(it, messages); getMessages(it, messages);

5
arangod/Pregel/Utils.cpp
View File

@ -35,7 +35,8 @@ std::string const Utils::messagesPath = "messages";
std::string const Utils::writeResultsPath = "writeResults"; std::string const Utils::writeResultsPath = "writeResults";
std::string const Utils::executionNumberKey = "extn"; std::string const Utils::executionNumberKey = "extn";
std::string const Utils::vertexCollectionKey = "vertexCollection"; std::string const Utils::vertexCollectionNameKey = "vertecCollName";
std::string const Utils::vertexCollectionPlanIdKey = "vertecCollPlanID";
std::string const Utils::vertexShardsListKey = "vertexShards"; std::string const Utils::vertexShardsListKey = "vertexShards";
std::string const Utils::edgeShardsListKey = "edgeShards"; std::string const Utils::edgeShardsListKey = "edgeShards";
std::string const Utils::resultShardKey = "resultShard"; std::string const Utils::resultShardKey = "resultShard";
@ -46,6 +47,8 @@ std::string const Utils::messagesKey = "msgs";
std::string const Utils::senderKey = "sender"; std::string const Utils::senderKey = "sender";
std::string const Utils::doneKey = "done"; std::string const Utils::doneKey = "done";
std::string const Utils::edgeShardingKey = "_vertex";
std::string Utils::baseUrl(std::string dbName) { std::string Utils::baseUrl(std::string dbName) {
return "/_db/" + basics::StringUtils::urlEncode(dbName) + Utils::apiPrefix; return "/_db/" + basics::StringUtils::urlEncode(dbName) + Utils::apiPrefix;
} }

4
arangod/Pregel/Utils.h
View File

@ -44,7 +44,8 @@ namespace pregel {
static std::string const writeResultsPath; static std::string const writeResultsPath;
static std::string const executionNumberKey; static std::string const executionNumberKey;
static std::string const vertexCollectionKey; static std::string const vertexCollectionNameKey;
static std::string const vertexCollectionPlanIdKey;
static std::string const vertexShardsListKey; static std::string const vertexShardsListKey;
static std::string const edgeShardsListKey; static std::string const edgeShardsListKey;
static std::string const resultShardKey; static std::string const resultShardKey;
@ -56,6 +57,7 @@ namespace pregel {
static std::string const senderKey; static std::string const senderKey;
static std::string const doneKey; static std::string const doneKey;
static std::string const edgeShardingKey;
static std::string baseUrl(std::string dbName); static std::string baseUrl(std::string dbName);
}; };

321
arangod/Pregel/Worker.cpp
View File

@ -23,6 +23,7 @@
#include "Worker.h" #include "Worker.h"
#include "Vertex.h" #include "Vertex.h"
#include "Utils.h" #include "Utils.h"
#include "WorkerJob.h"
#include "WorkerContext.h" #include "WorkerContext.h"
#include "InMessageCache.h" #include "InMessageCache.h"
#include "OutMessageCache.h" #include "OutMessageCache.h"
@ -30,19 +31,19 @@
#include "Basics/MutexLocker.h" #include "Basics/MutexLocker.h"
#include "Cluster/ClusterInfo.h" #include "Cluster/ClusterInfo.h"
#include "Cluster/ClusterComm.h" #include "Cluster/ClusterComm.h"
#include "VocBase/ticks.h"
#include "VocBase/vocbase.h" #include "VocBase/vocbase.h"
#include "VocBase/LogicalCollection.h" #include "VocBase/LogicalCollection.h"
#include "VocBase/EdgeCollectionInfo.h" #include "VocBase/EdgeCollectionInfo.h"
#include "Indexes/Index.h"
#include "Dispatcher/DispatcherQueue.h" #include "Dispatcher/DispatcherQueue.h"
#include "Dispatcher/DispatcherFeature.h" #include "Dispatcher/DispatcherFeature.h"
#include "Utils/Transaction.h" #include "Utils/Transaction.h"
#include "Utils/SingleCollectionTransaction.h" #include "Utils/SingleCollectionTransaction.h"
#include "Utils/StandaloneTransactionContext.h" #include "Utils/StandaloneTransactionContext.h"
#include "Utils/OperationCursor.h" #include "Utils/OperationCursor.h"
#include "Indexes/EdgeIndex.h" #include "Indexes/EdgeIndex.h"
#include "Indexes/Index.h"
#include <velocypack/Iterator.h> #include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h> #include <velocypack/velocypack-aliases.h>
@ -56,112 +57,38 @@ Worker::Worker(unsigned int executionNumber,
TRI_vocbase_t *vocbase, TRI_vocbase_t *vocbase,
VPackSlice s) : _vocbase(vocbase), _ctx(new WorkerContext(executionNumber)) { VPackSlice s) : _vocbase(vocbase), _ctx(new WorkerContext(executionNumber)) {
//VPackSlice algo = s.get("algo");
VPackSlice coordID = s.get(Utils::coordinatorIdKey); VPackSlice coordID = s.get(Utils::coordinatorIdKey);
VPackSlice vertexCollName = s.get(Utils::vertexCollectionNameKey);
VPackSlice vertexCollPlanId = s.get(Utils::vertexCollectionPlanIdKey);
VPackSlice vertexShardIDs = s.get(Utils::vertexShardsListKey); VPackSlice vertexShardIDs = s.get(Utils::vertexShardsListKey);
VPackSlice edgeShardIDs = s.get(Utils::edgeShardsListKey); VPackSlice edgeShardIDs = s.get(Utils::edgeShardsListKey);
// TODO support more shards //if (!(coordID.isString() && vertexShardIDs.length() == 1 && edgeShardIDs.length() == 1)) {
if (!(coordID.isString() && vertexShardIDs.length() == 1 && edgeShardIDs.length() == 1)) { // THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "Only one shard per collection supported");
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "Only one shard per collection supported"); //}
if (!coordID.isString()
|| !vertexCollName.isString()
|| !vertexCollPlanId.isString()
|| !vertexShardIDs.isArray()
|| !edgeShardIDs.isArray()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "Supplied bad parameters to worker");
} }
_ctx->_coordinatorId = coordID.copyString(); _ctx->_coordinatorId = coordID.copyString();
_ctx->_database = vocbase->name(); _ctx->_database = vocbase->name();
_ctx->_vertexCollectionName = s.get(Utils::vertexCollectionKey).copyString();// readable name of collection _ctx->_vertexCollectionName = vertexCollName.copyString();// readable name of collection
_ctx->_vertexShardID = vertexShardIDs.at(0).copyString(); _ctx->_vertexCollectionPlanId = vertexCollPlanId.copyString();
_ctx->_edgeShardID = edgeShardIDs.at(0).copyString();
LOG(INFO) << "Received collection " << _ctx->_vertexCollectionName; VPackArrayIterator vertices(vertexShardIDs);
LOG(INFO) << "starting worker with (" << _ctx->_vertexShardID << ", " << _ctx->_edgeShardID << ")"; for (VPackSlice shardSlice : vertices) {
ShardID name = shardSlice.copyString();
SingleCollectionTransaction *trx = new SingleCollectionTransaction(StandaloneTransactionContext::Create(_vocbase), _ctx->_localVertexShardIDs.push_back(name);
_ctx->_vertexShardID, TRI_TRANSACTION_READ); lookupVertices(name);
int res = trx->begin();
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up vertices '%s'",
_ctx->_vertexShardID.c_str());
return;
}
// resolve planId
_ctx->_vertexCollectionPlanId = trx->documentCollection()->planId_as_string();
OperationResult result = trx->all( _ctx->_vertexShardID, 0, UINT64_MAX, OperationOptions());
// Commit or abort.
res = trx->finish(result.code);
if (!result.successful()) {
THROW_ARANGO_EXCEPTION_FORMAT(result.code, "while looking up graph '%s'", _ctx->_vertexCollectionName.c_str());
}
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up graph '%s'", _ctx->_vertexCollectionName.c_str());
}
VPackSlice vertices = result.slice();
if (vertices.isExternal()) {
vertices = vertices.resolveExternal();
}
_transactions.push_back(trx);// store transactions, otherwise VPackSlices become invalid
// ======= Look up edges
trx = new SingleCollectionTransaction(StandaloneTransactionContext::Create(_vocbase),
_ctx->_edgeShardID, TRI_TRANSACTION_READ);
res = trx->begin();
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up edges '%s'", _ctx->_edgeShardID.c_str());
}
_transactions.push_back(trx);
auto info = std::make_unique<arangodb::traverser::EdgeCollectionInfo>(trx, _ctx->_edgeShardID, TRI_EDGE_OUT,
StaticStrings::FromString, 0);
size_t edgeCount = 0;
VPackArrayIterator arr = VPackArrayIterator(vertices);
LOG(INFO) << "Found vertices: " << arr.size();
for (auto it : arr) {
LOG(INFO) << it.toJson();
if (it.isExternal()) {
it = it.resolveExternal();
} }
VPackArrayIterator edges(edgeShardIDs);
std::string vertexId = it.get(StaticStrings::KeyString).copyString(); for (VPackSlice shardSlice : edges) {
std::unique_ptr<Vertex> v (new Vertex(it)); ShardID name = shardSlice.copyString();
_vertices[vertexId] = v.get(); lookupEdges(name);
std::string key = _ctx->_vertexCollectionName+"/"+vertexId;// TODO geht das schneller
LOG(INFO) << "Retrieving edge " << key;
auto cursor = info->getEdges(key);
if (cursor->failed()) {
THROW_ARANGO_EXCEPTION_FORMAT(cursor->code, "while looking up edges '%s' from %s",
key.c_str(), _ctx->_edgeShardID.c_str());
} }
std::vector<TRI_doc_mptr_t*> result;
result.reserve(1000);
while (cursor->hasMore()) {
cursor->getMoreMptr(result, 1000);
for (auto const& mptr : result) {
VPackSlice s(mptr->vpack());
if (s.isExternal()) {
s = s.resolveExternal();
}
LOG(INFO) << s.toJson();
v->_edges.emplace_back(s);
edgeCount++;
}
}
LOG(INFO) << "done retrieving edge";
v.release();
}
trx->finish(res);
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "after looking up edges '%s'", _ctx->_edgeShardID.c_str());
}
LOG(INFO) << "Resolved " << _vertices.size() << " vertices";
LOG(INFO) << "Resolved " << edgeCount << " edges";
} }
Worker::~Worker() { Worker::~Worker() {
@ -170,10 +97,7 @@ Worker::~Worker() {
delete(it.second); delete(it.second);
} }
_vertices.clear(); _vertices.clear();
for (auto const &it : _transactions) {// clean transactions cleanupReadTransactions();
delete(it);
}
_transactions.clear();
} }
/// @brief Setup next superstep /// @brief Setup next superstep
@ -264,106 +188,101 @@ void Worker::writeResults() {
}*/ }*/
} }
void Worker::cleanupReadTransactions() {
// ========== WorkerJob ========== for (auto const &it : _readTrxList) {// clean transactions
if (it->getStatus() == TRI_TRANSACTION_RUNNING) {
WorkerJob::WorkerJob(Worker *worker, if (it->commit() != TRI_ERROR_NO_ERROR) {
std::shared_ptr<WorkerContext> ctx) : Job("Pregel Job"), _canceled(false), _worker(worker), _ctx(ctx) { LOG(WARN) << "Pregel worker: Failed to commit on a read transaction";
} }
}
void WorkerJob::work() { delete(it);
LOG(INFO) << "Worker job started";
if (_canceled) {
LOG(INFO) << "Job was canceled before work started";
return;
}
// TODO cache this
OutMessageCache outCache(_ctx);
unsigned int gss = _ctx->globalSuperstep();
bool isDone = true;
if (gss == 0) {
isDone = false;
for (auto const &it : _worker->_vertices) {
Vertex *v = it.second;
//std::string key = v->_data.get(StaticStrings::KeyString).copyString();
//VPackSlice messages = _ctx->readableIncomingCache()->getMessages(key);
v->compute(gss, MessageIterator(), &outCache);
bool active = v->state() == VertexActivationState::ACTIVE;
if (!active) LOG(INFO) << "vertex has halted";
_worker->_activationMap[it.first] = active;
} }
} else { _readTrxList.clear();
for (auto &it : _worker->_activationMap) { }
std::string key = _ctx->vertexCollectionName() + "/" + it.first;
VPackSlice messages = _ctx->readableIncomingCache()->getMessages(key);
MessageIterator iterator(messages);
if (iterator.size() > 0 || it.second) {
isDone = false;
LOG(INFO) << "Processing messages: " << messages.toString();
Vertex *v = _worker->_vertices[it.first];
v->compute(gss, iterator, &outCache);
bool active = v->state() == VertexActivationState::ACTIVE;
it.second = active;
if (!active) LOG(INFO) << "vertex has halted";
}
}
}
LOG(INFO) << "Finished executing vertex programs.";
if (_canceled) { void Worker::lookupVertices(ShardID const &vertexShard) {
return;
}
// ==================== send messages to other shards ====================
if (!isDone) {
outCache.sendMessages();
} else {
LOG(INFO) << "Worker job has nothing more to process";
}
// notify the conductor that we are done.
VPackBuilder package;
package.openObject();
package.add(Utils::senderKey, VPackValue(ServerState::instance()->getId()));
package.add(Utils::executionNumberKey, VPackValue(_ctx->executionNumber()));
package.add(Utils::globalSuperstepKey, VPackValue(gss));
package.add(Utils::doneKey, VPackValue(isDone));
package.close();
LOG(INFO) << "Sending finishedGSS to coordinator: " << _ctx->coordinatorId();
// TODO handle communication failures?
ClusterComm *cc = ClusterComm::instance(); SingleCollectionTransaction *trx = new SingleCollectionTransaction(StandaloneTransactionContext::Create(_vocbase),
std::string baseUrl = Utils::baseUrl(_worker->_vocbase->name()); vertexShard, TRI_TRANSACTION_READ);
CoordTransactionID coordinatorTransactionID = TRI_NewTickServer(); int res = trx->begin();
auto headers = if (res != TRI_ERROR_NO_ERROR) {
std::make_unique<std::unordered_map<std::string, std::string>>(); THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up vertices '%s'", vertexShard.c_str());
auto body = std::make_shared<std::string const>(package.toJson()); }
cc->asyncRequest("", coordinatorTransactionID,
"server:" + _ctx->coordinatorId(), OperationResult result = trx->all(vertexShard, 0, UINT64_MAX, OperationOptions());
rest::RequestType::POST, // Commit or abort.
baseUrl + Utils::finishedGSSPath, res = trx->finish(result.code);
body, headers, nullptr, 90.0); if (!result.successful()) {
THROW_ARANGO_EXCEPTION_FORMAT(result.code, "while looking up shard '%s'", vertexShard.c_str());
LOG(INFO) << "Worker job finished sending stuff"; }
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up shard '%s'", vertexShard.c_str());
}
VPackSlice vertices = result.slice();
if (vertices.isExternal()) {
vertices = vertices.resolveExternal();
}
_readTrxList.push_back(trx);// store transactions, otherwise VPackSlices become invalid
VPackArrayIterator arr = VPackArrayIterator(vertices);
LOG(INFO) << "Found vertices: " << arr.size();
for (auto it : arr) {
LOG(INFO) << it.toJson();
if (it.isExternal()) {
it = it.resolveExternal();
}
std::string vertexId = it.get(StaticStrings::KeyString).copyString();
std::unique_ptr<Vertex> v (new Vertex(it));
_vertices[vertexId] = v.get();
v.release();
}
} }
bool WorkerJob::cancel() { void Worker::lookupEdges(ShardID const &edgeShardID) {
LOG(INFO) << "Canceling worker job"; std::unique_ptr<SingleCollectionTransaction> trx(new SingleCollectionTransaction(StandaloneTransactionContext::Create(_vocbase),
_canceled = true; edgeShardID, TRI_TRANSACTION_READ));
return true; int res = trx->begin();
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up edges '%s'", edgeShardID.c_str());
}
auto info = std::make_unique<arangodb::traverser::EdgeCollectionInfo>(trx.get(), edgeShardID, TRI_EDGE_OUT,
StaticStrings::FromString, 0);
size_t edgeCount = 0;
for (auto const &it : _vertices) {
Vertex *v = it.second;
std::string _from = _ctx->_vertexCollectionName+"/"+it.first;// TODO geht das schneller
LOG(INFO) << "Retrieving edge _from: " << _from;
auto cursor = info->getEdges(_from);
if (cursor->failed()) {
THROW_ARANGO_EXCEPTION_FORMAT(cursor->code, "while looking up edges '%s' from %s",
_from.c_str(), edgeShardID.c_str());
}
std::vector<TRI_doc_mptr_t*> result;
result.reserve(1000);
while (cursor->hasMore()) {
cursor->getMoreMptr(result, 1000);
for (auto const& mptr : result) {
VPackSlice s(mptr->vpack());
if (s.isExternal()) {
s = s.resolveExternal();
}
LOG(INFO) << s.toJson();
v->_edges.emplace_back(s);
edgeCount++;
}
}
}
_readTrxList.push_back(trx.get());
trx.release();
} }
void WorkerJob::cleanup(rest::DispatcherQueue* queue) {
queue->removeJob(this);
delete this;
}
void WorkerJob::handleError(basics::Exception const& ex) {}

24
arangod/Pregel/Worker.h
View File

@ -46,6 +46,7 @@ namespace pregel {
void nextGlobalStep(VPackSlice data);// called by coordinator void nextGlobalStep(VPackSlice data);// called by coordinator
void receivedMessages(VPackSlice data); void receivedMessages(VPackSlice data);
void writeResults(); void writeResults();
void cleanupReadTransactions();
private: private:
/// @brief guard to make sure the database is not dropped while used by us /// @brief guard to make sure the database is not dropped while used by us
@ -55,25 +56,10 @@ namespace pregel {
std::unordered_map<std::string, Vertex*> _vertices; std::unordered_map<std::string, Vertex*> _vertices;
std::map<std::string, bool> _activationMap; std::map<std::string, bool> _activationMap;
std::vector<SingleCollectionTransaction*> _transactions; std::vector<SingleCollectionTransaction*> _readTrxList;
};
void lookupVertices(ShardID const& vertexShard);
class WorkerJob : public rest::Job { void lookupEdges(ShardID const& edgeShardID);
WorkerJob(WorkerJob const&) = delete;
WorkerJob& operator=(WorkerJob const&) = delete;
public:
WorkerJob(Worker *worker, std::shared_ptr<WorkerContext> ctx);
void work() override;
bool cancel() override;
void cleanup(rest::DispatcherQueue*) override;
void handleError(basics::Exception const& ex) override;
private:
std::atomic<bool> _canceled;
Worker *_worker;
std::shared_ptr<WorkerContext> _ctx;
}; };
} }
} }

13
arangod/Pregel/WorkerContext.h
View File

@ -65,13 +65,13 @@ namespace pregel {
return _vertexCollectionPlanId; return _vertexCollectionPlanId;
} }
inline ShardID const& vertexShardId() const { inline std::vector<ShardID> const& localVertexShardIDs() const {
return _vertexShardID; return _localVertexShardIDs;
} }
inline ShardID const& edgeShardId() const { //inline ShardID const& edgeShardId() const {
return _edgeShardID; // return _edgeShardID;
} //}
inline InMessageCache* readableIncomingCache() { inline InMessageCache* readableIncomingCache() {
return _readCache; return _readCache;
@ -89,7 +89,8 @@ namespace pregel {
std::string _coordinatorId; std::string _coordinatorId;
std::string _database; std::string _database;
std::string _vertexCollectionName, _vertexCollectionPlanId; std::string _vertexCollectionName, _vertexCollectionPlanId;
ShardID _vertexShardID, _edgeShardID; std::vector<ShardID> _localVertexShardIDs;
//ShardID _vertexShardID, _edgeShardID;
InMessageCache *_readCache, *_writeCache; InMessageCache *_readCache, *_writeCache;
void swapIncomingCaches();// only call when message receiving is locked void swapIncomingCaches();// only call when message receiving is locked

142
arangod/Pregel/WorkerJob.cpp Executable file
View File

@ -0,0 +1,142 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Simon Grätzer
////////////////////////////////////////////////////////////////////////////////
#include "WorkerJob.h"
#include "WorkerContext.h"
#include "Worker.h"
#include "Vertex.h"
#include "Utils.h"
#include "InMessageCache.h"
#include "OutMessageCache.h"
#include "VocBase/ticks.h"
#include "Cluster/ClusterInfo.h"
#include "Cluster/ClusterComm.h"
#include "Dispatcher/DispatcherQueue.h"
#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>
using namespace arangodb;
using namespace arangodb::pregel;
WorkerJob::WorkerJob(Worker *worker,
std::shared_ptr<WorkerContext> ctx) : Job("Pregel Job"), _canceled(false), _worker(worker), _ctx(ctx) {
}
void WorkerJob::work() {
LOG(INFO) << "Worker job started";
if (_canceled) {
LOG(INFO) << "Job was canceled before work started";
return;
}
// TODO cache this
OutMessageCache outCache(_ctx);
unsigned int gss = _ctx->globalSuperstep();
bool isDone = true;
if (gss == 0) {
isDone = false;
for (auto const &it : _worker->_vertices) {
Vertex *v = it.second;
//std::string key = v->_data.get(StaticStrings::KeyString).copyString();
//VPackSlice messages = _ctx->readableIncomingCache()->getMessages(key);
v->compute(gss, MessageIterator(), &outCache);
bool active = v->state() == VertexActivationState::ACTIVE;
if (!active) LOG(INFO) << "vertex has halted";
_worker->_activationMap[it.first] = active;
}
} else {
for (auto &it : _worker->_activationMap) {
std::string key = _ctx->vertexCollectionName() + "/" + it.first;
VPackSlice messages = _ctx->readableIncomingCache()->getMessages(key);
MessageIterator iterator(messages);
if (iterator.size() > 0 || it.second) {
isDone = false;
LOG(INFO) << "Processing messages: " << messages.toString();
Vertex *v = _worker->_vertices[it.first];
v->compute(gss, iterator, &outCache);
bool active = v->state() == VertexActivationState::ACTIVE;
it.second = active;
if (!active) LOG(INFO) << "vertex has halted";
}
}
}
LOG(INFO) << "Finished executing vertex programs.";
if (_canceled) {
return;
}
// ==================== send messages to other shards ====================
if (!isDone) {
outCache.sendMessages();
} else {
LOG(INFO) << "Worker job has nothing more to process";
}
// notify the conductor that we are done.
VPackBuilder package;
package.openObject();
package.add(Utils::senderKey, VPackValue(ServerState::instance()->getId()));
package.add(Utils::executionNumberKey, VPackValue(_ctx->executionNumber()));
package.add(Utils::globalSuperstepKey, VPackValue(gss));
package.add(Utils::doneKey, VPackValue(isDone));
package.close();
LOG(INFO) << "Sending finishedGSS to coordinator: " << _ctx->coordinatorId();
// TODO handle communication failures?
ClusterComm *cc = ClusterComm::instance();
std::string baseUrl = Utils::baseUrl(_worker->_vocbase->name());
CoordTransactionID coordinatorTransactionID = TRI_NewTickServer();
auto headers =
std::make_unique<std::unordered_map<std::string, std::string>>();
auto body = std::make_shared<std::string const>(package.toJson());
cc->asyncRequest("", coordinatorTransactionID,
"server:" + _ctx->coordinatorId(),
rest::RequestType::POST,
baseUrl + Utils::finishedGSSPath,
body, headers, nullptr, 90.0);
LOG(INFO) << "Worker job finished sending stuff";
}
bool WorkerJob::cancel() {
LOG(INFO) << "Canceling worker job";
_canceled = true;
return true;
}
void WorkerJob::cleanup(rest::DispatcherQueue* queue) {
queue->removeJob(this);
delete this;
}
void WorkerJob::handleError(basics::Exception const& ex) {}

54
arangod/Pregel/WorkerJob.h Executable file
View File

@ -0,0 +1,54 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Simon Grätzer
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_PREGEL_JOB_H
#define ARANGODB_PREGEL_JOB_H 1
#include "Basics/Common.h"
#include "Dispatcher/Job.h"
namespace arangodb {
class SingleCollectionTransaction;
namespace pregel {
class Worker;
class WorkerContext;
class WorkerJob : public rest::Job {
WorkerJob(WorkerJob const&) = delete;
WorkerJob& operator=(WorkerJob const&) = delete;
public:
WorkerJob(Worker *worker, std::shared_ptr<WorkerContext> ctx);
void work() override;
bool cancel() override;
void cleanup(rest::DispatcherQueue*) override;
void handleError(basics::Exception const& ex) override;
private:
std::atomic<bool> _canceled;
Worker *_worker;
std::shared_ptr<WorkerContext> _ctx;
};
}
}
#endif

1
arangod/RestHandler/RestPregelHandler.cpp
View File

@ -94,6 +94,7 @@ RestHandler::status RestPregelHandler::execute() {
Worker *exe = PregelFeature::instance()->worker(executionNumber); Worker *exe = PregelFeature::instance()->worker(executionNumber);
if (exe) { if (exe) {
exe->writeResults(); exe->writeResults();
PregelFeature::instance()->cleanup(executionNumber);
} }
} }

6
arangod/V8Server/v8-collection.cpp
View File

@ -1854,15 +1854,15 @@ static void JS_Pregel(v8::FunctionCallbackInfo<v8::Value> const& args) {
try { try {
vertexColl = ClusterInfo::instance()->getCollection(vocbase->name(), vName); vertexColl = ClusterInfo::instance()->getCollection(vocbase->name(), vName);
edgeColl = ClusterInfo::instance()->getCollection(vocbase->name(), eName); edgeColl = ClusterInfo::instance()->getCollection(vocbase->name(), eName);
if (edgeColl->isSystem() || edgeColl->isSystem()) { if (vertexColl->isSystem() || edgeColl->isSystem()) {
TRI_V8_THROW_EXCEPTION_USAGE("Cannot use pregel on system collection"); TRI_V8_THROW_EXCEPTION_USAGE("Cannot use pregel on system collection");
} }
if (!vertexColl->usesDefaultShardKeys()) { if (!vertexColl->usesDefaultShardKeys()) {
TRI_V8_THROW_EXCEPTION_USAGE("Vertex collection needs to be shared after '_key'"); TRI_V8_THROW_EXCEPTION_USAGE("Vertex collection needs to be shared after '_key'");
} }
std::vector<std::string> eKeys = edgeColl->shardKeys(); std::vector<std::string> eKeys = edgeColl->shardKeys();
if (eKeys.size() != 1 || eKeys[0] == StaticStrings::FromString) { if (eKeys.size() != 1 || eKeys[0] != "_vertex") {
TRI_V8_THROW_EXCEPTION_USAGE("Edge collection needs to be sharded after '_from'"); TRI_V8_THROW_EXCEPTION_USAGE("Edge collection needs to be sharded after '_vertex', or use smart graphs");
} }
//eName = ci2->cid_as_string(); //eName = ci2->cid_as_string();
} catch (...) { } catch (...) {