1
0
Fork 0

Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

This commit is contained in:
Kaveh Vahedipour 2017-05-17 14:44:28 +02:00
commit cd8e8b8acf
26 changed files with 441 additions and 200 deletions

View File

@ -31,6 +31,8 @@ CMake flags
* *-DUSE_FAILURE_TESTS=1* - adds javascript hook to crash the server for data integrity tests
* *-DUSE_CATCH_TESTS=On (default is On so this is set unless you explicitly disable it)
If you have made changes to errors.dat, remember to use the -DUSE_MAINTAINER_MODE flag.
CFLAGS
------
Add backtraces to cluster requests so you can easily track their origin:

View File

@ -24,7 +24,6 @@
#define ARANGODB_PREGEL_ALGORITHM_H 1
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>
#include <cstdint>
#include <functional>
@ -64,7 +63,8 @@ struct IAlgorithm {
return nullptr;
}
virtual MasterContext* masterContext(VPackSlice userParams) const {
virtual MasterContext* masterContext(
arangodb::velocypack::Slice userParams) const {
return nullptr;
}
@ -85,7 +85,7 @@ struct IAlgorithm {
template <typename V, typename E, typename M>
struct Algorithm : IAlgorithm {
public:
virtual WorkerContext* workerContext(VPackSlice userParams) const {
virtual WorkerContext* workerContext(velocypack::Slice userParams) const {
return new WorkerContext();
}
virtual GraphFormat<V, E>* inputFormat() const = 0;
@ -124,7 +124,7 @@ class SimpleAlgorithm : public Algorithm<V, E, M> {
SimpleAlgorithm(std::string const& name, VPackSlice userParams)
: Algorithm<V, E, M>(name) {
VPackSlice field = userParams.get("sourceField");
arangodb::velocypack::Slice field = userParams.get("sourceField");
_sourceField = field.isString() ? field.copyString() : "value";
field = userParams.get("resultField");
_resultField = field.isString() ? field.copyString() : "result";

View File

@ -81,9 +81,6 @@ struct LPComputation : public VertexComputation<LPValue, int8_t, uint64_t> {
uint64_t newCommunity = mutableVertexData()->currentCommunity;
if (messages.size() == 1) {
newCommunity = std::min(**messages, newCommunity);
//newCommunity = RandomGenerator::interval(UINT32_MAX) % 2 == 0
// ? **messages
// : newCommunity;
}
if (messages.size() > 1) {
newCommunity = mostFrequent(messages);

View File

@ -122,10 +122,14 @@ VertexComputation<SLPAValue, int8_t, uint64_t>* SLPA::createComputation(
struct SLPAGraphFormat : public GraphFormat<SLPAValue, int8_t> {
std::string resField;
std::atomic<uint64_t> vertexIdRange;
float threshold; // TODO
double threshold;
unsigned maxCommunities;
explicit SLPAGraphFormat(std::string const& result, float thr)
: resField(result), vertexIdRange(0), threshold(thr) {}
explicit SLPAGraphFormat(std::string const& result, double thr, unsigned mc)
: resField(result),
vertexIdRange(0),
threshold(thr),
maxCommunities(mc) {}
size_t estimatedVertexSize() const override { return sizeof(LPValue); };
size_t estimatedEdgeSize() const override { return 0; };
@ -159,19 +163,24 @@ struct SLPAGraphFormat : public GraphFormat<SLPAValue, int8_t> {
} else {
std::vector<uint64_t> communities;
for (std::pair<uint64_t, uint64_t> pair : ptr->memory) {
if ((float)pair.second / ptr->numCommunities >= threshold) {
if ((double)pair.second / ptr->numCommunities >= threshold) {
communities.push_back(pair.first);
}
}
std::sort(communities.begin(), communities.end(),
[ptr](uint64_t a, uint64_t b) {
return ptr->memory.at(a) > ptr->memory.at(b);
});
if (communities.empty()) {
b.add(resField, VPackSlice::nullSlice());
} else if (communities.size() == 1) {
b.add(resField, VPackValue(ptr->memory.begin()->second));
} else if (communities.size() == 1 || maxCommunities == 1) {
b.add(resField, VPackValue(communities[0]));
} else if (communities.size() > 1) {
b.add(resField, VPackValue(VPackValueType::Array));
for (uint64_t c : communities) {
b.add(VPackValue(c));
for (unsigned c = 0; c < communities.size() && c < maxCommunities;
c++) {
b.add(VPackValue(communities[c]));
}
b.close();
}
@ -186,5 +195,5 @@ struct SLPAGraphFormat : public GraphFormat<SLPAValue, int8_t> {
};
GraphFormat<SLPAValue, int8_t>* SLPA::inputFormat() const {
return new SLPAGraphFormat(_resultField, 0.15);
return new SLPAGraphFormat(_resultField, _threshold, _maxCommunities);
}

View File

@ -23,6 +23,7 @@
#ifndef ARANGODB_PREGEL_ALGOS_SLPA_H
#define ARANGODB_PREGEL_ALGOS_SLPA_H 1
#include <cmath>
#include "Pregel/Algorithm.h"
#include "Pregel/CommonFormats.h"
@ -40,9 +41,21 @@ namespace algos {
/// overlapping
/// nodes and overlapping communities with different degrees of diversity.
struct SLPA : public SimpleAlgorithm<SLPAValue, int8_t, uint64_t> {
unsigned _maxCommunities = 1;
double _threshold = 0.15;
public:
explicit SLPA(VPackSlice userParams)
: SimpleAlgorithm<SLPAValue, int8_t, uint64_t>("slpa", userParams) {}
: SimpleAlgorithm<SLPAValue, int8_t, uint64_t>("slpa", userParams) {
arangodb::velocypack::Slice field = userParams.get("threshold");
if (field.isNumber()) {
_threshold = std::min(1.0, std::max(field.getDouble(), 0.0));
}
field = userParams.get("maxCommunities");
if (field.isInteger()) {
_threshold = (unsigned)std::min((uint64_t)32, std::max(field.getUInt(), (uint64_t)0));
}
}
GraphFormat<SLPAValue, int8_t>* inputFormat() const override;
MessageFormat<uint64_t>* messageFormat() const override {

View File

@ -146,13 +146,13 @@ class InitGraphFormat : public GraphFormat<V, E> {
virtual bool buildVertexDocument(arangodb::velocypack::Builder& b,
const V* ptr, size_t size) const override {
b.add(_resultField, VPackValue(*ptr));
b.add(_resultField, arangodb::velocypack::Value(*ptr));
return true;
}
virtual bool buildEdgeDocument(arangodb::velocypack::Builder& b, const E* ptr,
size_t size) const override {
b.add(_resultField, VPackValue(*ptr));
b.add(_resultField, arangodb::velocypack::Value(*ptr));
return true;
}
};
@ -184,7 +184,7 @@ class VertexGraphFormat : public GraphFormat<V, E> {
bool buildVertexDocument(arangodb::velocypack::Builder& b, const V* ptr,
size_t size) const override {
b.add(_resultField, VPackValue(*ptr));
b.add(_resultField, arangodb::velocypack::Value(*ptr));
return true;
}

View File

@ -28,9 +28,9 @@
#include "Basics/VelocyPackHelper.h"
#include "Basics/WriteLocker.h"
#include "Cache/CacheManagerFeature.h"
#include "Cache/TransactionalCache.h"
#include "Cache/Common.h"
#include "Cache/Manager.h"
#include "Cache/TransactionalCache.h"
#include "Cluster/ClusterMethods.h"
#include "Cluster/CollectionLockState.h"
#include "Indexes/Index.h"
@ -215,6 +215,7 @@ void RocksDBCollection::open(bool ignoreErrors) {
void RocksDBCollection::prepareIndexes(
arangodb::velocypack::Slice indexesSlice) {
WRITE_LOCKER(guard, _indexesLock);
TRI_ASSERT(indexesSlice.isArray());
if (indexesSlice.length() == 0) {
createInitialIndexes();
@ -339,7 +340,6 @@ void RocksDBCollection::prepareIndexes(
}
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
READ_LOCKER(guard, _indexesLock);
if (_indexes[0]->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX ||
(_logicalCollection->type() == TRI_COL_TYPE_EDGE &&
(_indexes[1]->type() != Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX ||
@ -354,9 +354,9 @@ void RocksDBCollection::prepareIndexes(
#endif
}
/// @brief Find index by definition
std::shared_ptr<Index> RocksDBCollection::lookupIndex(
velocypack::Slice const& info) const {
static std::shared_ptr<Index> findIndex(
velocypack::Slice const& info,
std::vector<std::shared_ptr<Index>> const& indexes) {
TRI_ASSERT(info.isObject());
// extract type
@ -370,7 +370,7 @@ std::shared_ptr<Index> RocksDBCollection::lookupIndex(
std::string tmp = value.copyString();
arangodb::Index::IndexType const type = arangodb::Index::type(tmp.c_str());
for (auto const& idx : _indexes) {
for (auto const& idx : indexes) {
if (idx->type() == type) {
// Only check relevant indexes
if (idx->matchesDefinition(info)) {
@ -382,14 +382,28 @@ std::shared_ptr<Index> RocksDBCollection::lookupIndex(
return nullptr;
}
/// @brief Find index by definition
std::shared_ptr<Index> RocksDBCollection::lookupIndex(
velocypack::Slice const& info) const {
READ_LOCKER(guard, _indexesLock);
return findIndex(info, _indexes);
}
std::shared_ptr<Index> RocksDBCollection::createIndex(
transaction::Methods* trx, arangodb::velocypack::Slice const& info,
bool& created) {
auto idx = lookupIndex(info);
if (idx != nullptr) {
created = false;
// We already have this index.
return idx;
// prevent concurrent dropping
bool isLocked = trx->isLocked(_logicalCollection, AccessMode::Type::EXCLUSIVE);
CONDITIONAL_WRITE_LOCKER(guard, _exclusiveLock, !isLocked);
std::shared_ptr<Index> idx;
{
WRITE_LOCKER(guard, _indexesLock);
idx = findIndex(info, _indexes);
if (idx) {
created = false;
// We already have this index.
return idx;
}
}
StorageEngine* engine = EngineSelectorFeature::ENGINE;
@ -421,33 +435,33 @@ std::shared_ptr<Index> RocksDBCollection::createIndex(
_logicalCollection->vocbase());
// Until here no harm is done if sth fails. The shared ptr will clean up. if
// left before
addIndex(idx);
{
VPackBuilder builder = _logicalCollection->toVelocyPackIgnore(
{"path", "statusString"}, true, /*forPersistence*/ true);
WRITE_LOCKER(guard, _indexesLock);
addIndex(idx);
}
VPackBuilder builder = _logicalCollection->toVelocyPackIgnore(
{"path", "statusString"}, true, /*forPersistence*/ true);
VPackBuilder indexInfo;
idx->toVelocyPack(indexInfo, false, true);
int res = static_cast<RocksDBEngine*>(engine)->writeCreateCollectionMarker(
_logicalCollection->vocbase()->id(), _logicalCollection->cid(),
builder.slice(), RocksDBLogValue::IndexCreate(
_logicalCollection->vocbase()->id(),
_logicalCollection->cid(), indexInfo.slice()));
if (res != TRI_ERROR_NO_ERROR) {
// We could not persist the index creation. Better abort
// Remove the Index in the local list again.
size_t i = 0;
WRITE_LOCKER(guard, _indexesLock);
for (auto index : _indexes) {
if (index == idx) {
_indexes.erase(_indexes.begin() + i);
break;
}
++i;
VPackBuilder indexInfo;
idx->toVelocyPack(indexInfo, false, true);
res = static_cast<RocksDBEngine*>(engine)->writeCreateCollectionMarker(
_logicalCollection->vocbase()->id(), _logicalCollection->cid(),
builder.slice(), RocksDBLogValue::IndexCreate(
_logicalCollection->vocbase()->id(),
_logicalCollection->cid(), indexInfo.slice()));
if (res != TRI_ERROR_NO_ERROR) {
// We could not persist the index creation. Better abort
// Remove the Index in the local list again.
size_t i = 0;
WRITE_LOCKER(guard, _indexesLock);
for (auto index : _indexes) {
if (index == idx) {
_indexes.erase(_indexes.begin() + i);
break;
}
THROW_ARANGO_EXCEPTION(res);
++i;
}
THROW_ARANGO_EXCEPTION(res);
}
created = true;
return idx;
@ -540,6 +554,7 @@ int RocksDBCollection::restoreIndex(transaction::Methods* trx,
/// @brief Drop an index with the given iid.
bool RocksDBCollection::dropIndex(TRI_idx_iid_t iid) {
// usually always called when _exclusiveLock is held
if (iid == 0) {
// invalid index id or primary index
return true;
@ -618,8 +633,8 @@ void RocksDBCollection::truncate(transaction::Methods* trx,
TRI_voc_cid_t cid = _logicalCollection->cid();
RocksDBTransactionState* state = rocksutils::toRocksTransactionState(trx);
RocksDBMethods *mthd = state->rocksdbMethods();
//rocksdb::Transaction* rtrx = state->rocksTransaction();
RocksDBMethods* mthd = state->rocksdbMethods();
// rocksdb::Transaction* rtrx = state->rocksTransaction();
// delete documents
RocksDBKeyBounds documentBounds =
@ -1168,7 +1183,6 @@ void RocksDBCollection::deferDropCollection(
/// @brief return engine-specific figures
void RocksDBCollection::figuresSpecific(
std::shared_ptr<arangodb::velocypack::Builder>& builder) {
rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
RocksDBKeyBounds bounds = RocksDBKeyBounds::CollectionDocuments(_objectId);
rocksdb::Range r(bounds.start(), bounds.end());
@ -1181,11 +1195,9 @@ void RocksDBCollection::figuresSpecific(
/// @brief creates the initial indexes for the collection
void RocksDBCollection::createInitialIndexes() {
{ // addIndex holds an internal write lock
READ_LOCKER(guard, _indexesLock);
if (!_indexes.empty()) {
return;
}
// LOCKED from the outside
if (!_indexes.empty()) {
return;
}
std::vector<std::shared_ptr<arangodb::Index>> systemIndexes;
@ -1200,7 +1212,7 @@ void RocksDBCollection::createInitialIndexes() {
}
void RocksDBCollection::addIndex(std::shared_ptr<arangodb::Index> idx) {
WRITE_LOCKER(guard, _indexesLock);
// LOCKED from the outside
// primary index must be added at position 0
TRI_ASSERT(idx->type() != arangodb::Index::TRI_IDX_TYPE_PRIMARY_INDEX ||
_indexes.empty());
@ -1223,8 +1235,7 @@ void RocksDBCollection::addIndex(std::shared_ptr<arangodb::Index> idx) {
void RocksDBCollection::addIndexCoordinator(
std::shared_ptr<arangodb::Index> idx) {
WRITE_LOCKER(guard, _indexesLock);
// LOCKED from the outside
auto const id = idx->id();
for (auto const& it : _indexes) {
if (it->id() == id) {
@ -1237,6 +1248,7 @@ void RocksDBCollection::addIndexCoordinator(
int RocksDBCollection::saveIndex(transaction::Methods* trx,
std::shared_ptr<arangodb::Index> idx) {
// LOCKED from the outside
TRI_ASSERT(!ServerState::instance()->isCoordinator());
// we cannot persist primary or edge indexes
TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX);
@ -1262,12 +1274,20 @@ int RocksDBCollection::saveIndex(transaction::Methods* trx,
/// from this collection
arangodb::Result RocksDBCollection::fillIndexes(
transaction::Methods* trx, std::shared_ptr<arangodb::Index> added) {
ManagedDocumentResult mmdr;
// LOCKED from the outside, can't use lookupIndex
RocksDBPrimaryIndex* primIndex = nullptr;
for (std::shared_ptr<Index> idx : _indexes) {
if (idx->type() == Index::TRI_IDX_TYPE_PRIMARY_INDEX) {
primIndex = static_cast<RocksDBPrimaryIndex*>(idx.get());
break;
}
}
TRI_ASSERT(primIndex);
ManagedDocumentResult mmdr;
RocksDBIndex* ridx = static_cast<RocksDBIndex*>(added.get());
RocksDBTransactionState* state = rocksutils::toRocksTransactionState(trx);
std::unique_ptr<IndexIterator> iter(
primaryIndex()->allIterator(trx, &mmdr, false));
std::unique_ptr<IndexIterator> it(primIndex->allIterator(trx, &mmdr, false));
rocksdb::TransactionDB* db = globalRocksDB();
uint64_t numDocsWritten = 0;
@ -1291,7 +1311,7 @@ arangodb::Result RocksDBCollection::fillIndexes(
Result r;
bool hasMore = true;
while (hasMore) {
hasMore = iter->next(cb, 250);
hasMore = it->next(cb, 250);
if (_logicalCollection->status() == TRI_VOC_COL_STATUS_DELETED ||
_logicalCollection->deleted()) {
res = TRI_ERROR_INTERNAL;
@ -1312,7 +1332,7 @@ arangodb::Result RocksDBCollection::fillIndexes(
// we will need to remove index elements created before an error
// occured, this needs to happen since we are non transactional
if (!r.ok()) {
iter->reset();
it->reset();
batch.Clear();
res = TRI_ERROR_NO_ERROR;
@ -1330,7 +1350,7 @@ arangodb::Result RocksDBCollection::fillIndexes(
hasMore = true;
while (hasMore && numDocsWritten > 0) {
hasMore = iter->next(removeCb, 5000);
hasMore = it->next(removeCb, 5000);
}
// TODO: if this fails, do we have any recourse?
// Simon: Don't think so
@ -1382,7 +1402,7 @@ RocksDBOperationResult RocksDBCollection::insertDocument(
blackListKey(key.string().data(), static_cast<uint32_t>(key.string().size()));
RocksDBMethods *mthd = rocksutils::toRocksMethods(trx);
RocksDBMethods* mthd = rocksutils::toRocksMethods(trx);
res = mthd->Put(key, value.string());
if (!res.ok()) {
// set keysize that is passed up to the crud operations
@ -1442,12 +1462,12 @@ RocksDBOperationResult RocksDBCollection::removeDocument(
// from the outside. We do not need to DELETE a document from the
// document store, if the doc is overwritten with PUT
// Simon: actually we do, because otherwise the counter recovery is broken
//if (!isUpdate) {
RocksDBMethods *mthd = rocksutils::toRocksMethods(trx);
RocksDBOperationResult res = mthd->Delete(key);
if (!res.ok()) {
return res;
}
// if (!isUpdate) {
RocksDBMethods* mthd = rocksutils::toRocksMethods(trx);
RocksDBOperationResult res = mthd->Delete(key);
if (!res.ok()) {
return res;
}
//}
RocksDBOperationResult resInner;
@ -1557,7 +1577,7 @@ arangodb::Result RocksDBCollection::lookupRevisionVPack(
}
}
RocksDBMethods *mthd = rocksutils::toRocksMethods(trx);
RocksDBMethods* mthd = rocksutils::toRocksMethods(trx);
Result res = mthd->Get(key, &value);
TRI_ASSERT(value.data());
if (res.ok()) {
@ -1775,11 +1795,11 @@ arangodb::Result RocksDBCollection::serializeIndexEstimates(
output.clear();
RocksDBIndex* cindex = static_cast<RocksDBIndex*>(index.get());
TRI_ASSERT(cindex != nullptr);
rocksutils::uint64ToPersistent(output, static_cast<uint64_t>(tdb->GetLatestSequenceNumber()));
rocksutils::uint64ToPersistent(
output, static_cast<uint64_t>(tdb->GetLatestSequenceNumber()));
cindex->serializeEstimate(output);
if (output.size() > sizeof(uint64_t)) {
RocksDBKey key =
RocksDBKey::IndexEstimateValue(cindex->objectId());
RocksDBKey key = RocksDBKey::IndexEstimateValue(cindex->objectId());
rocksdb::Slice value(output);
rocksdb::Status s = rtrx->Put(key.string(), value);
@ -1811,7 +1831,8 @@ void RocksDBCollection::recalculateIndexEstimates() {
recalculateIndexEstimates(idxs);
}
void RocksDBCollection::recalculateIndexEstimates(std::vector<std::shared_ptr<Index>>& indexes) {
void RocksDBCollection::recalculateIndexEstimates(
std::vector<std::shared_ptr<Index>>& indexes) {
// start transaction to get a collection lock
arangodb::SingleCollectionTransaction trx(
arangodb::transaction::StandaloneContext::Create(

View File

@ -247,8 +247,7 @@ class RocksDBCollection final : public PhysicalCollection {
/// upgrade write locks to exclusive locks if this flag is set
bool _hasGeoIndex;
basics::ReadWriteLock _exclusiveLock;
mutable basics::ReadWriteLock _exclusiveLock;
mutable std::shared_ptr<cache::Cache> _cache;
// we use this boolean for testing whether _cache is set.
// it's quicker than accessing the shared_ptr each time

View File

@ -92,10 +92,16 @@ RocksDBIndex::~RocksDBIndex() {
void RocksDBIndex::toVelocyPackFigures(VPackBuilder& builder) const {
TRI_ASSERT(builder.isOpenObject());
Index::toVelocyPackFigures(builder);
builder.add("cacheInUse", VPackValue(useCache()));
if(useCache()){
builder.add("cacheSize", VPackValue(_cache->size()));
builder.add("liftimeHitRate", VPackValue(_cache->hitRates().first));
builder.add("windowHitRate", VPackValue(_cache->hitRates().second));
double rate =_cache->hitRates().first;
rate = std::isnan(rate) ? 0.0 : rate;
builder.add("cacheLiftimeHitRate", VPackValue(rate));
rate =_cache->hitRates().second;
rate = std::isnan(rate) ? 0.0 : rate;
builder.add("cacheWindowHitRate", VPackValue(rate));
} else {
builder.add("cacheSize", VPackValue(0));
}

View File

@ -341,12 +341,6 @@ void RocksDBPrimaryIndex::toVelocyPack(VPackBuilder& builder, bool withFigures,
builder.close();
}
/// @brief return a VelocyPack representation of the index figures
void RocksDBPrimaryIndex::toVelocyPackFigures(VPackBuilder& builder) const {
Index::toVelocyPackFigures(builder);
// TODO: implement
}
RocksDBToken RocksDBPrimaryIndex::lookupKey(transaction::Methods* trx,
arangodb::StringRef keyRef) const {
auto key = RocksDBKey::PrimaryIndexValue(_objectId, keyRef);

View File

@ -161,7 +161,6 @@ class RocksDBPrimaryIndex final : public RocksDBIndex {
size_t memory() const override;
void toVelocyPack(VPackBuilder&, bool, bool) const override;
void toVelocyPackFigures(VPackBuilder&) const override;
RocksDBToken lookupKey(transaction::Methods* trx,
arangodb::StringRef key) const;

View File

@ -32,6 +32,7 @@ const actions = require('@arangodb/actions');
const FoxxManager = require('@arangodb/foxx/manager');
const request = require('@arangodb/request');
const db = require('@arangodb').db;
const ArangoError = require('@arangodb').ArangoError;
const joinPath = require('path').join;
const fs = require('fs');
const fmu = require('@arangodb/foxx/manager-utils');
@ -54,6 +55,9 @@ function proxyLocal (method, url, qs, body, headers = {}) {
headers['content-length'] = body.length;
}
const res = request({method, url, qs, headers, body});
if (res.json && res.json.errorNum) {
throw new ArangoError(res.json);
}
res.throw();
return res.body ? JSON.parse(res.body) : null;
}

View File

@ -46,7 +46,8 @@ function get_api_indexes (req, res) {
return;
}
var list = [], ids = {}, indexes = collection.getIndexes(), i;
var withStats = (req.parameters.withStats === "true" || false);
var list = [], ids = {}, indexes = collection.getIndexes(withStats), i;
for (i = 0; i < indexes.length; ++i) {
var index = indexes[i];

View File

@ -846,7 +846,15 @@
}
var currentP = fmtNumber(self.history[self.server].residentSizePercent * 100, 2);
var data = [prettyBytes(self.history[self.server].physicalMemory)];
var data;
if (self.history[self.server].physicalMemory) {
this.removeEmptyDataLabels();
data = [prettyBytes(self.history[self.server].physicalMemory)];
} else {
this.addEmptyDataLabels();
return;
}
if (self.history[self.server].residentSizeChart === undefined) {
this.addEmptyDataLabels();
@ -966,7 +974,8 @@
if (self.history[self.server].residentSizeChart === undefined) {
self.addEmptyDataLabels();
return;
// initialize with 0 values then
// return;
} else {
self.removeEmptyDataLabels();
}
@ -1041,9 +1050,15 @@
return fmtNumber(((d * 100 * 100) / 100), 0) + '%';
});
d3.select(selector)
.datum(self.history[self.server][k])
.call(self.distributionCharts[k]);
if (self.history[self.server][k]) {
d3.select(selector)
.datum(self.history[self.server][k])
.call(self.distributionCharts[k]);
} else {
d3.select(selector)
.datum([])
.call(self.distributionCharts[k]);
}
nv.utils.windowResize(self.distributionCharts[k].update);
@ -1065,9 +1080,15 @@
self.distributionCharts[k].height(dimensions.height);
// update data
d3.select(selector)
.datum(self.history[self.server][k])
.call(self.distributionCharts[k]);
if (self.history[self.server][k]) {
d3.select(selector)
.datum(self.history[self.server][k])
.call(self.distributionCharts[k]);
} else {
d3.select(selector)
.datum([])
.call(self.distributionCharts[k]);
}
// trigger resize
nv.utils.windowResize(self.distributionCharts[k].update);

View File

@ -21,13 +21,6 @@ module.context.use(router);
const LDJSON = 'application/x-ldjson';
const legacyErrors = new Map([
[errors.ERROR_SERVICE_INVALID_NAME.code, errors.ERROR_SERVICE_SOURCE_NOT_FOUND.code],
[errors.ERROR_SERVICE_INVALID_MOUNT.code, errors.ERROR_INVALID_MOUNTPOINT.code],
[errors.ERROR_SERVICE_DOWNLOAD_FAILED.code, errors.ERROR_SERVICE_SOURCE_ERROR.code],
[errors.ERROR_SERVICE_UPLOAD_FAILED.code, errors.ERROR_SERVICE_SOURCE_ERROR.code]
]);
const serviceToJson = (service) => (
{
mount: service.mount,
@ -70,9 +63,8 @@ router.use((req, res, next) => {
next();
} catch (e) {
if (e.isArangoError) {
const errorNum = legacyErrors.get(e.errorNum) || e.errorNum;
const status = actions.arangoErrorToHttpCode(errorNum);
res.throw(status, e.errorMessage, {errorNum, cause: e});
const status = actions.arangoErrorToHttpCode(e.errorNum);
res.throw(status, e.errorMessage, {errorNum: e.errorNum, cause: e});
}
throw e;
}

View File

@ -286,6 +286,7 @@
"ERROR_SERVICE_SOURCE_ERROR" : { "code" : 3015, "message" : "error resolving source" },
"ERROR_SERVICE_UNKNOWN_SCRIPT" : { "code" : 3016, "message" : "unknown script" },
"ERROR_MODULE_NOT_FOUND" : { "code" : 3100, "message" : "cannot locate module" },
"ERROR_MODULE_SYNTAX_ERROR" : { "code" : 3101, "message" : "syntax error in module" },
"ERROR_MODULE_FAILURE" : { "code" : 3103, "message" : "failed to invoke module" },
"ERROR_NO_SMART_COLLECTION" : { "code" : 4000, "message" : "collection is not smart" },
"ERROR_NO_SMART_GRAPH_ATTRIBUTE" : { "code" : 4001, "message" : "smart graph attribute not given" },

View File

@ -406,17 +406,6 @@ exports.stringPadding = function (str, len, pad, dir) {
return str;
};
// //////////////////////////////////////////////////////////////////////////////
// / @brief throws an error in case a download failed
// //////////////////////////////////////////////////////////////////////////////
exports.throwDownloadError = function (msg) {
throw new exports.ArangoError({
errorNum: exports.errors.ERROR_SERVICE_DOWNLOAD_FAILED.code,
errorMessage: exports.errors.ERROR_SERVICE_DOWNLOAD_FAILED.message + ': ' + String(msg)
});
};
// //////////////////////////////////////////////////////////////////////////////
// / @brief throws an error in case of missing file
// //////////////////////////////////////////////////////////////////////////////

View File

@ -27,12 +27,14 @@
// / @author Copyright 2015, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
const dd = require('dedent');
const arangodb = require('@arangodb');
const plainServerVersion = arangodb.plainServerVersion;
const db = arangodb.db;
const errors = arangodb.errors;
const ArangoError = arangodb.ArangoError;
const download = require('internal').download;
const fs = require('fs');
const throwDownloadError = arangodb.throwDownloadError;
const utils = require('@arangodb/foxx/manager-utils');
const semver = require('semver');
@ -326,7 +328,14 @@ var update = function () {
}, filename);
if (result.code < 200 || result.code > 299) {
throwDownloadError("Github download from '" + url + "' failed with error code " + result.code);
throw new ArangoError({
errorNum: errors.ERROR_SERVICE_SOURCE_ERROR.code,
errorMessage: dd`
${errors.ERROR_SERVICE_SOURCE_ERROR.message}
URL: ${url}
Status Code: ${result.code}
`
});
}
updateFishbowlFromZip(filename);

View File

@ -96,7 +96,7 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'malformed-controller-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_SYNTAX_ERROR.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause').that.is.an.instanceof(SyntaxError);
} else {
@ -110,13 +110,8 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'malformed-controller-path'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause').that.is.an.instanceof(ArangoError)
.with.a.property('errorNum', errors.ERROR_SYS_ERROR.code);
} else {
expect(err).not.to.have.property('cause');
}
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_NOT_FOUND.code);
expect(err).not.to.have.property('cause');
return true;
});
});
@ -125,11 +120,10 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'broken-controller-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_SYNTAX_ERROR.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause');
expect(err.cause).not.to.be.an.instanceof(SyntaxError);
expect(err.cause).not.to.be.an.instanceof(ArangoError);
expect(err).to.have.property('cause')
.that.is.an.instanceof(SyntaxError);
} else {
expect(err).not.to.have.property('cause');
}
@ -173,7 +167,7 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'malformed-exports-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_SYNTAX_ERROR.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(SyntaxError);
@ -188,14 +182,8 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'malformed-exports-path'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(ArangoError)
.with.a.property('errorNum', errors.ERROR_SYS_ERROR.code);
} else {
expect(err).not.to.have.property('cause');
}
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_NOT_FOUND.code);
expect(err).not.to.have.property('cause');
return true;
});
});
@ -204,7 +192,7 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'malformed-setup-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_SYNTAX_ERROR.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(SyntaxError);
@ -219,14 +207,8 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'malformed-setup-path'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(ArangoError)
.with.a.property('errorNum', errors.ERROR_SYS_ERROR.code);
} else {
expect(err).not.to.have.property('cause');
}
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_NOT_FOUND.code);
expect(err).not.to.have.property('cause');
return true;
});
});
@ -235,14 +217,8 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'missing-controller-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(ArangoError)
.with.a.property('errorNum', errors.ERROR_SYS_ERROR.code);
} else {
expect(err).not.to.have.property('cause');
}
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_NOT_FOUND.code);
expect(err).not.to.have.property('cause');
return true;
});
});
@ -251,14 +227,8 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'missing-exports-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(ArangoError)
.with.a.property('errorNum', errors.ERROR_SYS_ERROR.code);
} else {
expect(err).not.to.have.property('cause');
}
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_NOT_FOUND.code);
expect(err).not.to.have.property('cause');
return true;
});
});
@ -267,14 +237,8 @@ describe('Foxx Manager install', function () {
expect(function () {
FoxxManager.install(fs.join(basePath, 'missing-setup-file'), '/unittest/broken');
}).to.throw(ArangoError).that.satisfies(function (err) {
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_FAILURE.code);
if (require('@arangodb').isServer) {
expect(err).to.have.property('cause')
.that.is.an.instanceof(ArangoError)
.with.a.property('errorNum', errors.ERROR_SYS_ERROR.code);
} else {
expect(err).not.to.have.property('cause');
}
expect(err).to.have.property('errorNum', errors.ERROR_MODULE_NOT_FOUND.code);
expect(err).not.to.have.property('cause');
return true;
});
});

View File

@ -36,6 +36,31 @@ var db = internal.db;
var expect = require('chai').expect;
var should = require('chai').should();
function debugOut(item){
print("#############");
print(item);
print("#############");
}
var isRocksDB = db._engine().name === "rocksdb";
var isMMFiles = db._engine().name === "mmfiles";
function verifyMemory(index){
expect(index.figures).to.be.ok;
expect(index.figures.memory).to.be.a('number');
if(isMMFiles){
expect(index.figures.memory).to.be.a.above(0);
}
}
function verifyCache(index){
expect(index.figures.cacheInUse).to.be.true;
expect(index.figures).to.be.ok;
expect(index.figures.cacheSize).to.be.a('number');
expect(index.figures.cacheLiftimeHitRate).to.be.a('number');
expect(index.figures.cacheWindowHitRate).to.be.a('number');
}
describe('Index figures', function () {
beforeEach(function () {
try {
@ -50,7 +75,7 @@ describe('Index figures', function () {
});
// edge index ///////////////////////////////////////////////////////////
describe('edge index', function () {
describe('primar/edge index', function () {
var col;
before('create collection',function(){
col = db._createEdgeCollection("UnitTestEdgar");
@ -58,29 +83,27 @@ describe('Index figures', function () {
col.insert({"_from":"source/1","_to":"sink/"+i});
}
});
it('verify index types', function() {
var indexes = col.getIndexes(true);
expect(indexes.length).to.be.equal(2);
expect(indexes[0].type).to.be.equal("primary");
expect(indexes[1].type).to.be.equal("edge");
});
it.skip('verify index - memory', function() {
it('verify index - memory', function() {
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
expect(i.figure).to.be.defined;
expect(i.figure, "index of type " + i.type + "does not have memory property")
.should.have.property('memory').above(0);
verifyMemory(i);
});
});
it('verify figures - cache', function() {
if(db._engine().name !== "rocksdb"){
if(!isRocksDB){
this.skip();
}
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyCache(i);
});
});
after(function(){
db._drop(col);
});
@ -88,6 +111,153 @@ describe('Index figures', function () {
// hash index ///////////////////////////////////////////////////////////
describe('hash index', function () {
var col;
before('create collection',function(){
col = db._createDocumentCollection("UnitTestDoggyHash");
col.ensureIndex({type: "hash", fields: ["name"]});
for(var i = 0; i < 100; i++){
col.insert({"name":"Harry"+i});
}
});
it('verify index types', function() {
var indexes = col.getIndexes(true);
expect(indexes.length).to.be.equal(2);
expect(indexes[0].type).to.be.equal("primary");
expect(indexes[1].type).to.be.equal("hash");
});
it('verify index - memory', function() {
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyMemory(i);
});
});
// FIXME not implemented
it.skip('verify figures - cache', function() {
if(!isRocksDB){
this.skip();
}
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyCache(i);
});
});
after(function(){
db._drop(col);
});
}); // end hash index
// skiplist index ///////////////////////////////////////////////////////////
describe('skiplist index', function () {
var col;
before('create collection',function(){
col = db._createDocumentCollection("UnitTestDoggySkip");
col.ensureIndex({type: "skiplist", fields: ["name"]});
for(var i = 0; i < 100; i++){
col.insert({"name":"Harry"+i});
}
});
it('verify index types', function() {
var indexes = col.getIndexes(true);
expect(indexes.length).to.be.equal(2);
expect(indexes[0].type).to.be.equal("primary");
expect(indexes[1].type).to.be.equal("skiplist");
});
it('verify index - memory', function() {
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyMemory(i);
});
});
// FIXME not implemented
it.skip('verify figures - cache', function() {
if(!isRocksDB){
this.skip();
}
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyCache(i);
});
});
after(function(){
db._drop(col);
});
}); // end skiplist index
// fulltest index ///////////////////////////////////////////////////////////
describe('fulltext index', function () {
var col;
before('create collection',function(){
col = db._createDocumentCollection("UnitTestDoggyFull");
col.ensureIndex({type: "fulltext", fields: ["name"], minLength : 3});
for(var i = 0; i < 100; i++){
col.insert({"name":"Harry"+i});
}
});
it('verify index types', function() {
var indexes = col.getIndexes(true);
expect(indexes.length).to.be.equal(2);
expect(indexes[0].type).to.be.equal("primary");
expect(indexes[1].type).to.be.equal("fulltext");
});
it('verify index - memory', function() {
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyMemory(i);
});
});
// FIXME not implemented
it.skip('verify figures - cache', function() {
if(!isRocksDB){
this.skip();
}
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyCache(i);
});
});
after(function(){
db._drop(col);
});
}); // end fulltext index
// geo index ///////////////////////////////////////////////////////////
describe('geo index', function () {
var col;
before('create collection',function(){
col = db._createDocumentCollection("UnitTestGeoSpass");
col.ensureIndex({type: "geo", fields: ["loc"]});
for(var i = 0; i < 10; i++){
for(var j = 0; j < 10; j++){
col.insert({ "name": "place" + i + "/" + j,
loc : [i, j]
});
}
}
});
it('verify index types', function() {
var indexes = col.getIndexes(true);
expect(indexes.length).to.be.equal(2);
expect(indexes[0].type).to.be.equal("primary");
expect(indexes[1].type).to.be.equal("geo1");
});
it('verify index - memory', function() {
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyMemory(i);
});
});
// FIXME not implemented
it.skip('verify figures - cache', function() {
if(!isRocksDB){
this.skip();
}
var indexes = col.getIndexes(true);
indexes.forEach((i) => {
verifyCache(i);
});
});
after(function(){
db._drop(col);
});
}); // end fulltext index
}); // end Index figures

View File

@ -791,6 +791,9 @@ function createServiceBundle (mount, bundlePath = FoxxService.bundlePath(mount))
function downloadServiceBundleFromRemote (url) {
try {
const res = request.get(url, {encoding: null});
if (res.json && res.json.errorNum) {
throw new ArangoError(res.json);
}
res.throw();
const tempFile = fs.getTempFile('bundles', false);
fs.writeFileSync(tempFile, res.body);

View File

@ -61,6 +61,36 @@ const LEGACY_ALIASES = [
['@arangodb/foxx', '@arangodb/foxx/legacy']
];
function parseFile (servicePath, filename) {
const filepath = path.resolve(servicePath, filename);
if (!fs.isFile(filepath)) {
throw new ArangoError({
errorNum: errors.ERROR_MODULE_NOT_FOUND.code,
errorMessage: dd`
${errors.ERROR_MODULE_NOT_FOUND.message}
File: ${filepath}
`
});
}
try {
internal.parseFile(filepath);
} catch (e) {
if (e instanceof SyntaxError) {
throw Object.assign(
new ArangoError({
errorNum: errors.ERROR_MODULE_SYNTAX_ERROR.code,
errorMessage: dd`
${errors.ERROR_MODULE_SYNTAX_ERROR.message}
File: ${filepath}
`
}),
{cause: e}
);
}
throw e;
}
}
module.exports =
class FoxxService {
static validatedManifest (definition) {
@ -80,25 +110,25 @@ module.exports =
static validateServiceFiles (mount, manifest) {
const servicePath = FoxxService.basePath(mount);
if (manifest.main) {
internal.parseFile(path.resolve(servicePath, manifest.main));
parseFile(servicePath, manifest.main);
}
for (const name of Object.keys(manifest.scripts)) {
const scriptFilename = manifest.scripts[name];
internal.parseFile(path.resolve(servicePath, scriptFilename));
parseFile(servicePath, scriptFilename);
}
if (manifest.controllers) {
for (const name of Object.keys(manifest.controllers)) {
const controllerFilename = manifest.controllers[name];
internal.parseFile(path.resolve(servicePath, controllerFilename));
parseFile(servicePath, controllerFilename);
}
}
if (manifest.exports) {
if (typeof manifest.exports === 'string') {
internal.parseFile(path.resolve(servicePath, manifest.exports));
parseFile(servicePath, manifest.exports);
} else {
for (const name of Object.keys(manifest.exports)) {
const exportFilename = manifest.exports[name];
internal.parseFile(path.resolve(servicePath, exportFilename));
parseFile(servicePath, exportFilename);
}
}
}

View File

@ -399,6 +399,7 @@ ERROR_SERVICE_UNKNOWN_SCRIPT,3016,"unknown script","The service does not have a
################################################################################
ERROR_MODULE_NOT_FOUND,3100,"cannot locate module","The module path could not be resolved."
ERROR_MODULE_SYNTAX_ERROR,3101,"syntax error in module","The module could not be parsed because of a syntax error."
ERROR_MODULE_FAILURE,3103,"failed to invoke module","Failed to invoke the module in its context."
################################################################################

View File

@ -282,6 +282,7 @@ void TRI_InitializeErrorMessages () {
REG_ERROR(ERROR_SERVICE_SOURCE_ERROR, "error resolving source");
REG_ERROR(ERROR_SERVICE_UNKNOWN_SCRIPT, "unknown script");
REG_ERROR(ERROR_MODULE_NOT_FOUND, "cannot locate module");
REG_ERROR(ERROR_MODULE_SYNTAX_ERROR, "syntax error in module");
REG_ERROR(ERROR_MODULE_FAILURE, "failed to invoke module");
REG_ERROR(ERROR_NO_SMART_COLLECTION, "collection is not smart");
REG_ERROR(ERROR_NO_SMART_GRAPH_ATTRIBUTE, "smart graph attribute not given");

View File

@ -671,6 +671,8 @@
/// The service does not have a script with this name.
/// - 3100: @LIT{cannot locate module}
/// The module path could not be resolved.
/// - 3101: @LIT{syntax error in module}
/// The module could not be parsed because of a syntax error.
/// - 3103: @LIT{failed to invoke module}
/// Failed to invoke the module in its context.
/// - 4000: @LIT{collection is not smart}
@ -3583,6 +3585,16 @@ void TRI_InitializeErrorMessages ();
#define TRI_ERROR_MODULE_NOT_FOUND (3100)
////////////////////////////////////////////////////////////////////////////////
/// @brief 3101: ERROR_MODULE_SYNTAX_ERROR
///
/// syntax error in module
///
/// The module could not be parsed because of a syntax error.
////////////////////////////////////////////////////////////////////////////////
#define TRI_ERROR_MODULE_SYNTAX_ERROR (3101)
////////////////////////////////////////////////////////////////////////////////
/// @brief 3103: ERROR_MODULE_FAILURE
///

View File

@ -9,17 +9,19 @@ main(){
local suite=${1:-all}
local tasks="suite_$suite"
type -t function $tasks || die "suite $suite not defined"
# ceck task
type -t function $tasks &>/dev/null|| die "suite $suite not defined"
echo "using function name: $tasks"
local session_name="$($tasks 'name')"
echo "using session name: $session_name"
local panes=$($tasks 'num')
echo "using $panes panes"
kill_old_session "$session_name"
tmux new-session -d -s "$session_name" || die "unable to spawn session"
local rows=$(( (panes+1) / 2 ))
local cols=$((((panes>1)) * 2))
spawn_panes "$session_name" $rows $cols
tmux select-pane -t $session_name.0
execute_tasks "$(pwd)" $tasks
execute_tasks "$(pwd)" "$tasks"
tmux -2 attach-session -t $session_name
}
@ -32,7 +34,7 @@ suite_all(){
local args_default=( )
local tests=""
case $1 in
case $count in
num)
echo "6"
return
@ -74,7 +76,7 @@ suite_all_rocksdb(){
shift 1
local args="$@"
case $1 in
case $count in
name)
echo "test_all_rocksdb"
return
@ -129,6 +131,7 @@ spawn_panes(){
execute_tasks(){
cd $1 || die
local tasks="$2"
shift 2
local args="$@"
local count=0
while (( count < $($tasks 'num') )); do