From 5d20b79c0ffd59834b27f84d1eda787c55e35510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Tue, 25 Apr 2017 12:01:58 +0200 Subject: [PATCH 01/58] Incremental Sync --- arangod/Pregel/Graph.h | 2 +- arangod/Pregel/Worker.cpp | 3 +- arangod/Replication/InitialSyncer.cpp | 1123 ++++++++++++----- arangod/Replication/InitialSyncer.h | 26 +- .../RocksDBReplicationContext.cpp | 8 +- .../RocksDBRestReplicationHandler.cpp | 7 +- etc/testing/arangod-common.conf | 3 + 7 files changed, 853 insertions(+), 319 deletions(-) diff --git a/arangod/Pregel/Graph.h b/arangod/Pregel/Graph.h index f8223c3155..b96040759d 100644 --- a/arangod/Pregel/Graph.h +++ b/arangod/Pregel/Graph.h @@ -75,7 +75,7 @@ class Edge { // EdgeEntry() : _nextEntryOffset(0), _dataSize(0), _vertexIDSize(0) {} Edge() : _targetShard(InvalidPregelShard) {} Edge(PregelShard target, PregelKey const& key) - : _targetShard(target), _toKey(key) {} + : _targetShard(target), _toKey(key), _data(0) {} // size_t getSize() { return sizeof(EdgeEntry) + _vertexIDSize + _dataSize; } PregelKey const& toKey() const { return _toKey; } diff --git a/arangod/Pregel/Worker.cpp b/arangod/Pregel/Worker.cpp index 119e079db7..c832a1c426 100644 --- a/arangod/Pregel/Worker.cpp +++ b/arangod/Pregel/Worker.cpp @@ -62,7 +62,8 @@ Worker::Worker(TRI_vocbase_t* vocbase, Algorithm* algo, : _state(WorkerState::IDLE), _config(vocbase, initConfig), _algorithm(algo), - _nextGSSSendMessageCount(0) { + _nextGSSSendMessageCount(0), + _requestedNextGSS(false) { MUTEX_LOCKER(guard, _commandMutex); VPackSlice userParams = initConfig.get(Utils::userParametersKey); diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 7869617947..6cd266128f 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -33,14 +33,19 @@ #include "Basics/VelocyPackHelper.h" #include "Indexes/Index.h" #include "Logger/Logger.h" +#include "StorageEngine/EngineSelectorFeature.h" #include "MMFiles/MMFilesCollection.h" //TODO -- Remove -- ditches #include "MMFiles/MMFilesDatafileHelper.h" #include "MMFiles/MMFilesDitch.h" +#include "MMFiles/MMFilesIndexElement.h" +#include "MMFiles/MMFilesPrimaryIndex.h" #include "RestServer/DatabaseFeature.h" #include "SimpleHttpClient/SimpleHttpClient.h" #include "SimpleHttpClient/SimpleHttpResult.h" #include "StorageEngine/EngineSelectorFeature.h" +#include "Indexes/IndexIterator.h" #include "StorageEngine/StorageEngine.h" +#include "Transaction/Helpers.h" #include "Utils/CollectionGuard.h" #include "Utils/OperationOptions.h" #include "VocBase/LogicalCollection.h" @@ -53,6 +58,7 @@ #include #include #include +#include using namespace arangodb; using namespace arangodb::basics; @@ -1020,8 +1026,15 @@ int InitialSyncer::handleCollectionSync(arangodb::LogicalCollection* col, // now we can fetch the complete chunk information from the master try { - res = handleSyncKeys(col, id.copyString(), cid, collectionName, maxTick, - errorMsg); + if (std::strcmp("mmfiles", EngineSelectorFeature::engineName()) == 0) { + res = handleSyncKeysMMFiles(col, id.copyString(), cid, collectionName, maxTick, + errorMsg); + } else if (std::strcmp("rocksdb", EngineSelectorFeature::engineName()) == 0) { + res = handleSyncKeysRocksDB(col, id.copyString(), cid, collectionName, maxTick, + errorMsg); + } else { + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + } } catch (arangodb::basics::Exception const& ex) { res = ex.code(); } catch (std::exception const& ex) { @@ -1038,7 +1051,7 @@ int InitialSyncer::handleCollectionSync(arangodb::LogicalCollection* col, /// @brief incrementally fetch data from a collection //////////////////////////////////////////////////////////////////////////////// -int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, +int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, std::string const& keysId, std::string const& cid, std::string const& collectionName, @@ -1048,134 +1061,6 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, "collecting local keys for collection '" + collectionName + "'"; setProgress(progress); - // fetch all local keys from primary index - VPackBuilder dataLake; - std::vector markers; - - MMFilesDocumentDitch* ditch = nullptr; - - // acquire a replication ditch so no datafiles are thrown away from now on - // note: the ditch also protects against unloading the collection - { - SingleCollectionTransaction trx( - transaction::StandaloneContext::Create(_vocbase), col->cid(), - AccessMode::Type::READ); - - Result res = trx.begin(); - - if (!res.ok()) { - errorMsg = - std::string("unable to start transaction: ") + res.errorMessage(); - res.reset(res.errorNumber(), errorMsg); - return res.errorNumber(); - } - - MMFilesCollection* mmcol = - dynamic_cast(col->getPhysical()); - if (mmcol != nullptr) { - ditch = mmcol->ditches()->createMMFilesDocumentDitch(false, __FILE__, - __LINE__); - - if (ditch == nullptr) { - return TRI_ERROR_OUT_OF_MEMORY; - } - } - } - - // TRI_ASSERT(ditch != nullptr); - TRI_DEFER(if (ditch != nullptr) { - arangodb::MMFilesCollection::toMMFilesCollection(col)->ditches()->freeDitch( - ditch); - }); - - { - SingleCollectionTransaction trx( - transaction::StandaloneContext::Create(_vocbase), col->cid(), - AccessMode::Type::READ); - - Result res = trx.begin(); - - if (!res.ok()) { - errorMsg = - std::string("unable to start transaction: ") + res.errorMessage(); - res.reset(res.errorNumber(), errorMsg); - return res.errorNumber(); - } - - // We do not take responsibility for the index. - // The LogicalCollection is protected by trx. - // Neither it nor it's indexes can be invalidated - - markers.reserve(trx.documentCollection()->numberDocuments(&trx)); - - uint64_t iterations = 0; - ManagedDocumentResult mmdr; - dataLake.openArray(); - trx.invokeOnAllElements( - trx.name(), [this, &trx, &mmdr, &markers, &iterations, - &dataLake](DocumentIdentifierToken const& token) { - if (trx.documentCollection()->readDocument(&trx, token, mmdr)) { - VPackSlice doc(mmdr.vpack()); - dataLake.openObject(); - dataLake.add(StaticStrings::KeyString, - doc.get(StaticStrings::KeyString)); - dataLake.add(StaticStrings::RevString, - doc.get(StaticStrings::RevString)); - dataLake.close(); - - if (++iterations % 10000 == 0) { - if (checkAborted()) { - return false; - } - } - } - return true; - }); - dataLake.close(); - // because live is hard - for (VPackSlice const& slice : VPackArrayIterator(dataLake.slice())) { - markers.emplace_back(slice.start()); - } - - if (checkAborted()) { - return TRI_ERROR_REPLICATION_APPLIER_STOPPED; - } - - sendExtendBatch(); - sendExtendBarrier(); - - std::string progress = "sorting " + std::to_string(markers.size()) + - " local key(s) for collection '" + collectionName + - "'"; - setProgress(progress); - - // sort all our local keys - std::sort(markers.begin(), markers.end(), [](uint8_t const* lhs, - uint8_t const* rhs) -> bool { - VPackSlice const l(lhs); - VPackSlice const r(rhs); - - VPackValueLength lLength, rLength; - char const* lKey = l.get(StaticStrings::KeyString).getString(lLength); - char const* rKey = r.get(StaticStrings::KeyString).getString(rLength); - - size_t const length = - static_cast(lLength < rLength ? lLength : rLength); - int res = memcmp(lKey, rKey, length); - - if (res < 0) { - // left is smaller than right - return true; - } - if (res == 0 && lLength < rLength) { - // left is equal to right, but of shorter length - return true; - } - - return false; - }); - } - if (checkAborted()) { return TRI_ERROR_REPLICATION_APPLIER_STOPPED; } @@ -1225,25 +1110,26 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - VPackSlice const slice = builder->slice(); + VPackSlice const chunkSlice = builder->slice(); - if (!slice.isArray()) { + if (!chunkSlice.isArray()) { errorMsg = "got invalid response from master at " + _masterInfo._endpoint + ": response is no array"; return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } + ManagedDocumentResult mmdr; OperationOptions options; options.silent = true; options.ignoreRevs = true; options.isRestore = true; VPackBuilder keyBuilder; - size_t const n = static_cast(slice.length()); + size_t const numChunks = static_cast(chunkSlice.length()); // remove all keys that are below first remote key or beyond last remote key - if (n > 0) { + if (numChunks > 0) { // first chunk SingleCollectionTransaction trx( transaction::StandaloneContext::Create(_vocbase), col->cid(), @@ -1258,64 +1144,42 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, return res.errorNumber(); } - VPackSlice chunk = slice.at(0); + VPackSlice chunk = chunkSlice.at(0); TRI_ASSERT(chunk.isObject()); auto lowSlice = chunk.get("low"); TRI_ASSERT(lowSlice.isString()); - - std::string const lowKey(lowSlice.copyString()); - - for (size_t i = 0; i < markers.size(); ++i) { - VPackSlice const k(markers[i]); - - std::string const key(k.get(StaticStrings::KeyString).copyString()); - - if (key.compare(lowKey) >= 0) { - break; - } - - keyBuilder.clear(); - keyBuilder.openObject(); - keyBuilder.add(StaticStrings::KeyString, VPackValue(key)); - keyBuilder.close(); - - trx.remove(collectionName, keyBuilder.slice(), options); - } - + // last high - chunk = slice.at(n - 1); + chunk = chunkSlice.at(numChunks - 1); TRI_ASSERT(chunk.isObject()); - + auto highSlice = chunk.get("high"); TRI_ASSERT(highSlice.isString()); + std::string const lowKey(lowSlice.copyString()); std::string const highKey(highSlice.copyString()); - for (size_t i = markers.size(); i >= 1; --i) { - VPackSlice const k(markers[i - 1]); - - std::string const key(k.get(StaticStrings::KeyString).copyString()); - - if (key.compare(highKey) <= 0) { - break; + + LogicalCollection* coll = trx.documentCollection(); + std::unique_ptr iterator = coll->getAllIterator(&trx, &mmdr, false); + iterator->next([&] (DocumentIdentifierToken const& token) { + if (coll->readDocument(&trx, token, mmdr) == false) { + return; } - - keyBuilder.clear(); - keyBuilder.openObject(); - keyBuilder.add(StaticStrings::KeyString, VPackValue(key)); - keyBuilder.close(); - - trx.remove(collectionName, keyBuilder.slice(), options); - } + VPackSlice doc(mmdr.vpack()); + VPackSlice key = doc.get(StaticStrings::KeyString); + if (key.compareString(lowKey.data(), lowKey.length()) < 0) { + trx.remove(collectionName, key, options); + } else if (key.compareString(highKey.data(), highKey.length()) > 0) { + trx.remove(collectionName, key, options); + } + }, UINT64_MAX); trx.commit(); } - size_t nextStart = 0; - - // now process each chunk - for (size_t i = 0; i < n; ++i) { + { if (checkAborted()) { return TRI_ERROR_REPLICATION_APPLIER_STOPPED; } @@ -1333,69 +1197,708 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, return res.errorNumber(); } - trx.pinData(col->cid()); // will throw when it fails // We do not take responsibility for the index. // The LogicalCollection is protected by trx. // Neither it nor it's indexes can be invalidated - // TODO Move to MMFiles - PhysicalCollection* physical = trx.documentCollection()->getPhysical(); + size_t currentChunkId = 0; + + std::string lowKey; + std::string highKey; + std::string hashString; + uint64_t localHash = 0x012345678; + // chunk keys + revisionId + std::vector> markers; + + auto resetChunk = [&] () -> void { + sendExtendBatch(); + sendExtendBarrier(); + + progress = "processing keys chunk " + std::to_string(currentChunkId) + + " for collection '" + collectionName + "'"; + setProgress(progress); + + // read remote chunk + VPackSlice chunk = chunkSlice.at(currentChunkId); + if (!chunk.isObject()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": chunk is no object"; + THROW_ARANGO_EXCEPTION(TRI_ERROR_REPLICATION_INVALID_RESPONSE); + } + + VPackSlice const lowSlice = chunk.get("low"); + VPackSlice const highSlice = chunk.get("high"); + VPackSlice const hashSlice = chunk.get("hash"); + if (!lowSlice.isString() || !highSlice.isString() || + !hashSlice.isString()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + + ": chunks in response have an invalid format"; + THROW_ARANGO_EXCEPTION(TRI_ERROR_REPLICATION_INVALID_RESPONSE); + } + + // now reset chunk information + markers.clear(); + lowKey = lowSlice.copyString(); + highKey = highSlice.copyString(); + hashString = hashSlice.copyString(); + localHash = 0x012345678; + }; + + LogicalCollection* coll = trx.documentCollection(); + std::unique_ptr iterator = coll->getAllIterator(&trx, &mmdr, false); + iterator->next([&] (DocumentIdentifierToken const& token) { + if (coll->readDocument(&trx, token, mmdr) == false) { + return; + } + VPackSlice doc(mmdr.vpack()); + VPackSlice key = doc.get(StaticStrings::KeyString); + + int cmp1 = key.compareString(lowKey.data(), lowKey.length()); + int cmp2 = key.compareString(highKey.data(), highKey.length()); + if (cmp1 >= 0 && cmp2 <= 0) { + VPackSlice revision = doc.get(StaticStrings::RevString); + localHash ^= key.hashString(); + localHash ^= revision.hash(); + + VPackValueLength revLength; + char const* revCharPtr = revision.getString(revLength); + markers.emplace_back(key.copyString(), + StringUtils::uint64(revCharPtr, revLength)); + + if (cmp2 == 0) {// found highKey + if (std::to_string(localHash) != hashString) { + // TODO do something + int res = syncChunkRocksDB(&trx, keysId, currentChunkId, + lowKey, highKey, + markers, errorMsg); + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + } + // else: we will jump into if (cmp2 > 0) next + return; + } + } else if (cmp1 < 0) { + // smaller values than lowKey should never happen, unless + // we hit the + trx.remove(collectionName, key, options); + return; + } else if (cmp2 > 0) { // higher than highKey + if (currentChunkId+1 < numChunks) { + currentChunkId++;// we are out of range, see next chunk + resetChunk(); + } else { + LOG_TOPIC(ERR, Logger::FIXME) << "WTF iterator position beyond last chunk"; + } + } + }, UINT64_MAX); - size_t const currentChunkId = i; - progress = "processing keys chunk " + std::to_string(currentChunkId) + - " for collection '" + collectionName + "'"; - setProgress(progress); + res = trx.commit(); + if (!res.ok()) { + return res.errorNumber(); + } + } - sendExtendBatch(); - sendExtendBarrier(); + return res; +} - // read remote chunk - VPackSlice chunk = slice.at(i); - if (!chunk.isObject()) { +int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, + std::string const& keysId, + uint64_t chunkId, + std::string const& lowString, + std::string const& highString, + std::vector> markers, + std::string& errorMsg) { + std::string const baseUrl = BaseUrl + "/keys"; + TRI_voc_tick_t const chunkSize = 5000; + std::string const& collectionName = trx->documentCollection()->name(); + PhysicalCollection* physical = trx->documentCollection()->getPhysical(); + OperationOptions options; + options.silent = true; + options.ignoreRevs = true; + options.isRestore = true; + + // no match + // must transfer keys for non-matching range + std::string url = baseUrl + "/" + keysId + "?type=keys&chunk=" + + std::to_string(chunkId) + "&chunkSize=" + + std::to_string(chunkSize); + + std::string progress = "fetching keys chunk " + std::to_string(chunkId) + "' from " + url; + setProgress(progress); + + std::unique_ptr response( + _client->retryRequest(rest::RequestType::PUT, url, nullptr, 0)); + + if (response == nullptr || !response->isComplete()) { + errorMsg = "could not connect to master at " + _masterInfo._endpoint + + ": " + _client->getErrorMessage(); + + return TRI_ERROR_REPLICATION_NO_RESPONSE; + } + + TRI_ASSERT(response != nullptr); + + if (response->wasHttpError()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + + return TRI_ERROR_REPLICATION_MASTER_ERROR; + } + + auto builder = std::make_shared(); + int res = parseResponse(builder, response.get()); + + if (res != TRI_ERROR_NO_ERROR) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": response is no array"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + VPackSlice const responseBody = builder->slice(); + if (!responseBody.isArray()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": response is no array"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + transaction::BuilderLeaser keyBuilder(trx); + /*size_t nextStart = 0; + // delete all keys at start of the range + while (nextStart < markers.size()) { + std::string const& localKey = markers[nextStart].first; + + if ( localKey.compare(lowString) < 0) { + // we have a local key that is not present remotely + keyBuilder.clear(); + keyBuilder.openObject(); + keyBuilder.add(StaticStrings::KeyString, VPackValue(localKey)); + keyBuilder.close(); + + trx.remove(collectionName, keyBuilder.slice(), options); + ++nextStart; + } else { + break; + } + }*/ + + std::vector toFetch; + + size_t const numKeys = static_cast(responseBody.length()); + TRI_ASSERT(numKeys > 0); + + size_t i = 0; + for (VPackSlice const& pair : VPackArrayIterator(responseBody)) { + + if (!pair.isArray() || pair.length() != 2) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": chunk is no object"; - + _masterInfo._endpoint + + ": response key pair is no valid array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } + + // key + VPackSlice const keySlice = pair.at(0); + if (!keySlice.isString()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": response key is no string"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + // rid + if (markers.empty()) { + // no local markers + toFetch.emplace_back(i); + continue; + } + + std::string const keyString = keySlice.copyString(); + + size_t nextStart = 0; + while (nextStart < markers.size()) { + std::string const& localKey = markers[nextStart].first; + + int res = localKey.compare(keyString); + if (res != 0) { + // we have a local key that is not present remotely + keyBuilder->clear(); + keyBuilder->openObject(); + keyBuilder->add(StaticStrings::KeyString, VPackValue(localKey)); + keyBuilder->close(); + + trx->remove(collectionName, keyBuilder->slice(), options); + ++nextStart; + } else { + // key match + break; + } + } + + DocumentIdentifierToken token = physical->lookupKey(trx, keySlice); + if (!token._data) { + // key not found locally + toFetch.emplace_back(i); + } else if (TRI_RidToString(token._data) != pair.at(1).copyString()) { + // key found, but revision id differs + toFetch.emplace_back(i); + ++nextStart; + } else { + // a match - nothing to do! + ++nextStart; + } + } + + + if (!toFetch.empty()) { + VPackBuilder keysBuilder; + keysBuilder.openArray(); + for (auto& it : toFetch) { + keysBuilder.add(VPackValue(it)); + } + keysBuilder.close(); + + std::string url = baseUrl + "/" + keysId + "?type=docs&chunk=" + + std::to_string(chunkId) + "&chunkSize=" + + std::to_string(chunkSize); + progress = "fetching documents chunk " + + std::to_string(chunkId) + " for collection '" + + collectionName + "' from " + url; + setProgress(progress); + + std::string const keyJsonString(keysBuilder.slice().toJson()); + + std::unique_ptr response( + _client->retryRequest(rest::RequestType::PUT, url, + keyJsonString.c_str(), keyJsonString.size())); + + if (response == nullptr || !response->isComplete()) { + errorMsg = "could not connect to master at " + _masterInfo._endpoint + + ": " + _client->getErrorMessage(); + + return TRI_ERROR_REPLICATION_NO_RESPONSE; + } + + TRI_ASSERT(response != nullptr); + + if (response->wasHttpError()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + + return TRI_ERROR_REPLICATION_MASTER_ERROR; + } + + auto builder = std::make_shared(); + int res = parseResponse(builder, response.get()); + + if (res != TRI_ERROR_NO_ERROR) { + errorMsg = "got invalid response from master at " + + std::string(_masterInfo._endpoint) + + ": response is no array"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + VPackSlice const slice = builder->slice(); + if (!slice.isArray()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": response is no array"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + for (auto const& it : VPackArrayIterator(slice)) { + if (!it.isObject()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": document is no object"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + VPackSlice const keySlice = it.get(StaticStrings::KeyString); + + if (!keySlice.isString()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": document key is invalid"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + VPackSlice const revSlice = it.get(StaticStrings::RevString); + + if (!revSlice.isString()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": document revision is invalid"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + DocumentIdentifierToken token = physical->lookupKey(trx, keySlice); + + if (!token._data) { + // INSERT + OperationResult opRes = trx->insert(collectionName, it, options); + res = opRes.code; + } else { + // UPDATE + OperationResult opRes = trx->update(collectionName, it, options); + res = opRes.code; + } + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + } + } + return TRI_ERROR_NO_ERROR; +} +//////////////////////////////////////////////////////////////////////////////// +/// @brief incrementally fetch data from a collection +//////////////////////////////////////////////////////////////////////////////// + +int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, + std::string const& keysId, std::string const& cid, + std::string const& collectionName, TRI_voc_tick_t maxTick, + std::string& errorMsg) { + + std::string progress = + "collecting local keys for collection '" + collectionName + "'"; + setProgress(progress); + + // fetch all local keys from primary index + std::vector markers; + + MMFilesDocumentDitch* ditch = nullptr; + + // acquire a replication ditch so no datafiles are thrown away from now on + // note: the ditch also protects against unloading the collection + { + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ); + + Result res = trx.begin(); + + if (!res.ok()) { + errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(),errorMsg); + return res.errorNumber(); + } + + ditch = arangodb::MMFilesCollection::toMMFilesCollection(col) + ->ditches() + ->createMMFilesDocumentDitch(false, __FILE__, __LINE__); + + if (ditch == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + } + + TRI_ASSERT(ditch != nullptr); + + TRI_DEFER(arangodb::MMFilesCollection::toMMFilesCollection(col) + ->ditches() + ->freeDitch(ditch)); + + { + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ); + + Result res = trx.begin(); + + if (!res.ok()) { + errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(),errorMsg); + return res.errorNumber(); + } + + // We do not take responsibility for the index. + // The LogicalCollection is protected by trx. + // Neither it nor it's indexes can be invalidated + + markers.reserve(trx.documentCollection()->numberDocuments(&trx)); + + uint64_t iterations = 0; + ManagedDocumentResult mmdr; + trx.invokeOnAllElements(trx.name(), [this, &trx, &mmdr, &markers, &iterations](DocumentIdentifierToken const& token) { + if (trx.documentCollection()->readDocument(&trx, token, mmdr)) { + markers.emplace_back(mmdr.vpack()); + + if (++iterations % 10000 == 0) { + if (checkAborted()) { + return false; + } + } + } + return true; + }); + + if (checkAborted()) { + return TRI_ERROR_REPLICATION_APPLIER_STOPPED; + } + + sendExtendBatch(); + sendExtendBarrier(); + + std::string progress = "sorting " + std::to_string(markers.size()) + + " local key(s) for collection '" + collectionName + + "'"; + setProgress(progress); + + // sort all our local keys + std::sort( + markers.begin(), markers.end(), + [](uint8_t const* lhs, uint8_t const* rhs) -> bool { + VPackSlice const l(lhs); + VPackSlice const r(rhs); + + VPackValueLength lLength, rLength; + char const* lKey = l.get(StaticStrings::KeyString).getString(lLength); + char const* rKey = r.get(StaticStrings::KeyString).getString(rLength); + + size_t const length = static_cast(lLength < rLength ? lLength : rLength); + int res = memcmp(lKey, rKey, length); + + if (res < 0) { + // left is smaller than right + return true; + } + if (res == 0 && lLength < rLength) { + // left is equal to right, but of shorter length + return true; + } + + return false; + }); + } + + if (checkAborted()) { + return TRI_ERROR_REPLICATION_APPLIER_STOPPED; + } + + sendExtendBatch(); + sendExtendBarrier(); + + std::vector toFetch; + + TRI_voc_tick_t const chunkSize = 5000; + std::string const baseUrl = BaseUrl + "/keys"; + + std::string url = + baseUrl + "/" + keysId + "?chunkSize=" + std::to_string(chunkSize); + progress = "fetching remote keys chunks for collection '" + collectionName + + "' from " + url; + setProgress(progress); + + std::unique_ptr response( + _client->retryRequest(rest::RequestType::GET, url, nullptr, 0)); + + if (response == nullptr || !response->isComplete()) { + errorMsg = "could not connect to master at " + + _masterInfo._endpoint + ": " + + _client->getErrorMessage(); + + return TRI_ERROR_REPLICATION_NO_RESPONSE; + } + + TRI_ASSERT(response != nullptr); + + if (response->wasHttpError()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + + return TRI_ERROR_REPLICATION_MASTER_ERROR; + } + + auto builder = std::make_shared(); + int res = parseResponse(builder, response.get()); + + if (res != TRI_ERROR_NO_ERROR) { + errorMsg = "got invalid response from master at " + + std::string(_masterInfo._endpoint) + + ": invalid response is no array"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + VPackSlice const slice = builder->slice(); + + if (!slice.isArray()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": response is no array"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + + OperationOptions options; + options.silent = true; + options.ignoreRevs = true; + options.isRestore = true; + + VPackBuilder keyBuilder; + size_t const n = static_cast(slice.length()); + + // remove all keys that are below first remote key or beyond last remote key + if (n > 0) { + // first chunk + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE); + + Result res = trx.begin(); + + if (!res.ok()) { + errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(),errorMsg); + return res.errorNumber(); + } + + VPackSlice chunk = slice.at(0); + + TRI_ASSERT(chunk.isObject()); + auto lowSlice = chunk.get("low"); + TRI_ASSERT(lowSlice.isString()); + + std::string const lowKey(lowSlice.copyString()); + + for (size_t i = 0; i < markers.size(); ++i) { + VPackSlice const k(markers[i]); + + std::string const key(k.get(StaticStrings::KeyString).copyString()); + + if (key.compare(lowKey) >= 0) { + break; + } + + keyBuilder.clear(); + keyBuilder.openObject(); + keyBuilder.add(StaticStrings::KeyString, VPackValue(key)); + keyBuilder.close(); + + trx.remove(collectionName, keyBuilder.slice(), options); + } + + // last high + chunk = slice.at(n - 1); + TRI_ASSERT(chunk.isObject()); + + auto highSlice = chunk.get("high"); + TRI_ASSERT(highSlice.isString()); + + std::string const highKey(highSlice.copyString()); + + for (size_t i = markers.size(); i >= 1; --i) { + VPackSlice const k(markers[i - 1]); + + std::string const key(k.get(StaticStrings::KeyString).copyString()); + + if (key.compare(highKey) <= 0) { + break; + } + + keyBuilder.clear(); + keyBuilder.openObject(); + keyBuilder.add(StaticStrings::KeyString, VPackValue(key)); + keyBuilder.close(); + + trx.remove(collectionName, keyBuilder.slice(), options); + } + + trx.commit(); + } + + size_t nextStart = 0; + + // now process each chunk + for (size_t i = 0; i < n; ++i) { + if (checkAborted()) { + return TRI_ERROR_REPLICATION_APPLIER_STOPPED; + } + + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE); + + Result res = trx.begin(); + + if (!res.ok()) { + errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(),res.errorMessage()); + return res.errorNumber(); + } + + trx.pinData(col->cid()); // will throw when it fails + + // We do not take responsibility for the index. + // The LogicalCollection is protected by trx. + // Neither it nor it's indexes can be invalidated + + // TODO Move to MMFiles + auto physical = static_cast( + trx.documentCollection()->getPhysical()); + auto idx = physical->primaryIndex(); + + size_t const currentChunkId = i; + progress = "processing keys chunk " + std::to_string(currentChunkId) + + " for collection '" + collectionName + "'"; + setProgress(progress); + + sendExtendBatch(); + sendExtendBarrier(); + + // read remote chunk + VPackSlice chunk = slice.at(i); + + if (!chunk.isObject()) { + errorMsg = "got invalid response from master at " + + _masterInfo._endpoint + ": chunk is no object"; + + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; + } + VPackSlice const lowSlice = chunk.get("low"); VPackSlice const highSlice = chunk.get("high"); VPackSlice const hashSlice = chunk.get("hash"); - + if (!lowSlice.isString() || !highSlice.isString() || !hashSlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": chunks in response have an invalid format"; - + _masterInfo._endpoint + + ": chunks in response have an invalid format"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + std::string const lowString = lowSlice.copyString(); std::string const highString = highSlice.copyString(); - + size_t localFrom; size_t localTo; - bool match = FindRange(markers, lowString, highString, localFrom, localTo); - + bool match = + FindRange(markers, lowString, highString, localFrom, localTo); + if (match) { // now must hash the range uint64_t hash = 0x012345678; - + for (size_t i = localFrom; i <= localTo; ++i) { TRI_ASSERT(i < markers.size()); VPackSlice const current(markers.at(i)); hash ^= current.get(StaticStrings::KeyString).hashString(); hash ^= current.get(StaticStrings::RevString).hash(); } - + if (std::to_string(hash) != hashSlice.copyString()) { match = false; } } - + if (match) { // match nextStart = localTo + 1; @@ -1403,120 +1906,123 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, // no match // must transfer keys for non-matching range std::string url = baseUrl + "/" + keysId + "?type=keys&chunk=" + - std::to_string(i) + "&chunkSize=" + - std::to_string(chunkSize); + std::to_string(i) + "&chunkSize=" + + std::to_string(chunkSize); progress = "fetching keys chunk " + std::to_string(currentChunkId) + - " for collection '" + collectionName + "' from " + url; + " for collection '" + collectionName + "' from " + url; setProgress(progress); - - std::unique_ptr response( - _client->retryRequest(rest::RequestType::PUT, url, nullptr, 0)); - + + std::unique_ptr response(_client->retryRequest( + rest::RequestType::PUT, url, nullptr, 0)); + if (response == nullptr || !response->isComplete()) { - errorMsg = "could not connect to master at " + _masterInfo._endpoint + - ": " + _client->getErrorMessage(); - + errorMsg = "could not connect to master at " + + _masterInfo._endpoint + ": " + + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + _masterInfo._endpoint + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const slice = builder->slice(); if (!slice.isArray()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + _masterInfo._endpoint + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + + // delete all keys at start of the range while (nextStart < markers.size()) { VPackSlice const keySlice(markers[nextStart]); - std::string const localKey( - keySlice.get(StaticStrings::KeyString).copyString()); - + std::string const localKey(keySlice.get(StaticStrings::KeyString).copyString()); + if (localKey.compare(lowString) < 0) { // we have a local key that is not present remotely keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(localKey)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); ++nextStart; } else { break; } } - + toFetch.clear(); - + size_t const n = static_cast(slice.length()); TRI_ASSERT(n > 0); - + for (size_t i = 0; i < n; ++i) { VPackSlice const pair = slice.at(i); - + if (!pair.isArray() || pair.length() != 2) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response key pair is no valid array"; - + _masterInfo._endpoint + + ": response key pair is no valid array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + // key VPackSlice const keySlice = pair.at(0); - + if (!keySlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response key is no string"; - + _masterInfo._endpoint + + ": response key is no string"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + // rid if (markers.empty()) { // no local markers toFetch.emplace_back(i); continue; } - + std::string const keyString = keySlice.copyString(); - + while (nextStart < markers.size()) { VPackSlice const localKeySlice(markers[nextStart]); - std::string const localKey( - localKeySlice.get(StaticStrings::KeyString).copyString()); - + std::string const localKey(localKeySlice.get(StaticStrings::KeyString).copyString()); + int res = localKey.compare(keyString); - + if (res != 0) { // we have a local key that is not present remotely keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(localKey)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); ++nextStart; } else { @@ -1524,12 +2030,13 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, break; } } - - DocumentIdentifierToken token = physical->lookupKey(&trx, keySlice); - if (!token._data) { + + MMFilesSimpleIndexElement element = idx->lookupKey(&trx, keySlice); + + if (!element) { // key not found locally toFetch.emplace_back(i); - } else if (TRI_RidToString(token._data) != pair.at(1).copyString()) { + } else if (TRI_RidToString(element.revisionId()) != pair.at(1).copyString()) { // key found, but revision id differs toFetch.emplace_back(i); ++nextStart; @@ -1538,18 +2045,17 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, ++nextStart; } } - + // calculate next starting point if (!markers.empty()) { BinarySearch(markers, highString, nextStart); - + while (nextStart < markers.size()) { VPackSlice const localKeySlice(markers[nextStart]); - std::string const localKey( - localKeySlice.get(StaticStrings::KeyString).copyString()); - + std::string const localKey(localKeySlice.get(StaticStrings::KeyString).copyString()); + int res = localKey.compare(highString); - + if (res <= 0) { ++nextStart; } else { @@ -1557,7 +2063,7 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, } } } - + if (!toFetch.empty()) { VPackBuilder keysBuilder; keysBuilder.openArray(); @@ -1565,87 +2071,92 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, keysBuilder.add(VPackValue(it)); } keysBuilder.close(); - + std::string url = baseUrl + "/" + keysId + "?type=docs&chunk=" + - std::to_string(currentChunkId) + "&chunkSize=" + - std::to_string(chunkSize); + std::to_string(currentChunkId) + "&chunkSize=" + + std::to_string(chunkSize); progress = "fetching documents chunk " + - std::to_string(currentChunkId) + " for collection '" + - collectionName + "' from " + url; + std::to_string(currentChunkId) + " for collection '" + + collectionName + "' from " + url; setProgress(progress); - + std::string const keyJsonString(keysBuilder.slice().toJson()); - + std::unique_ptr response( - _client->retryRequest(rest::RequestType::PUT, url, - keyJsonString.c_str(), keyJsonString.size())); - + _client->retryRequest(rest::RequestType::PUT, url, + keyJsonString.c_str(), keyJsonString.size())); + if (response == nullptr || !response->isComplete()) { - errorMsg = "could not connect to master at " + _masterInfo._endpoint + - ": " + _client->getErrorMessage(); - + errorMsg = "could not connect to master at " + + _masterInfo._endpoint + ": " + + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { errorMsg = "got invalid response from master at " + - std::string(_masterInfo._endpoint) + - ": response is no array"; - + std::string(_masterInfo._endpoint) + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const slice = builder->slice(); if (!slice.isArray()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + _masterInfo._endpoint + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + for (auto const& it : VPackArrayIterator(slice)) { if (!it.isObject()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": document is no object"; - + _masterInfo._endpoint + + ": document is no object"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const keySlice = it.get(StaticStrings::KeyString); - + if (!keySlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": document key is invalid"; - + _masterInfo._endpoint + + ": document key is invalid"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const revSlice = it.get(StaticStrings::RevString); - + if (!revSlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": document revision is invalid"; - + _masterInfo._endpoint + + ": document revision is invalid"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - - DocumentIdentifierToken token = physical->lookupKey(&trx, keySlice); - - if (!token._data) { + + MMFilesSimpleIndexElement element = idx->lookupKey(&trx, keySlice); + + if (!element) { // INSERT OperationResult opRes = trx.insert(collectionName, it, options); res = opRes.code; @@ -1654,21 +2165,21 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col, OperationResult opRes = trx.update(collectionName, it, options); res = opRes.code; } - + if (res != TRI_ERROR_NO_ERROR) { return res; } } } } - + res = trx.commit(); - + if (!res.ok()) { return res.errorNumber(); } } - + return res; } diff --git a/arangod/Replication/InitialSyncer.h b/arangod/Replication/InitialSyncer.h index 0dccf981a2..0769ecc80f 100644 --- a/arangod/Replication/InitialSyncer.h +++ b/arangod/Replication/InitialSyncer.h @@ -185,14 +185,34 @@ class InitialSyncer : public Syncer { int handleCollectionSync(arangodb::LogicalCollection*, std::string const&, std::string const&, TRI_voc_tick_t, std::string&); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief incrementally fetch data from a collection + ////////////////////////////////////////////////////////////////////////////// + + int handleSyncKeysRocksDB(arangodb::LogicalCollection* col, + std::string const& keysId, std::string const& cid, + std::string const& collectionName, TRI_voc_tick_t maxTick, + std::string& errorMsg); + ////////////////////////////////////////////////////////////////////////////// + /// @brief incrementally fetch chunk data from a collection + ////////////////////////////////////////////////////////////////////////////// + + int syncChunkRocksDB(SingleCollectionTransaction* trx, + std::string const& keysId, + uint64_t chunkId, + std::string const& lowKey, std::string const& highKey, + std::vector> markers, + std::string& errorMsg); ////////////////////////////////////////////////////////////////////////////// /// @brief incrementally fetch data from a collection ////////////////////////////////////////////////////////////////////////////// - int handleSyncKeys(arangodb::LogicalCollection*, std::string const&, - std::string const&, std::string const&, TRI_voc_tick_t, - std::string&); + int handleSyncKeysMMFiles(arangodb::LogicalCollection* col, + std::string const& keysId, std::string const& cid, + std::string const& collectionName, TRI_voc_tick_t maxTick, + std::string& errorMsg); ////////////////////////////////////////////////////////////////////////////// /// @brief changes the properties of a collection, based on the VelocyPack diff --git a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp index b8e6f649b0..e9474d77ef 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp @@ -88,7 +88,6 @@ int RocksDBReplicationContext::bindCollection( if (_collection == nullptr) { return TRI_ERROR_BAD_PARAMETER; - RocksDBReplicationResult(TRI_ERROR_BAD_PARAMETER, _lastTick); } _trx->addCollectionAtRuntime(collectionName); _iter = _collection->getAllIterator(_trx.get(), &_mdr, @@ -127,7 +126,10 @@ RocksDBReplicationResult RocksDBReplicationContext::dump( if (_trx.get() == nullptr) { return RocksDBReplicationResult(TRI_ERROR_BAD_PARAMETER, _lastTick); } - bindCollection(collectionName); + int res = bindCollection(collectionName); + if (res != TRI_ERROR_NO_ERROR) { + return RocksDBReplicationResult(res, _lastTick); + } // set type int type = 2300; // documents @@ -195,7 +197,6 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b, VPackSlice highKey; // FIXME: no good keeping this uint64_t hash = 0x012345678; - // auto cb = [&](DocumentIdentifierToken const& token, StringRef const& key) { auto cb = [&](DocumentIdentifierToken const& token) { bool ok = _collection->readDocument(_trx.get(), token, _mdr); if (!ok) { @@ -220,7 +221,6 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b, b.openArray(); while (_hasMore && true /*sizelimit*/) { try { - //_hasMore = primary->nextWithKey(cb, chunkSize); _hasMore = primary->next(cb, chunkSize); b.add(VPackValue(VPackValueType::Object)); diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index fb6a869bfa..5729ccc320 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -342,14 +342,13 @@ void RocksDBRestReplicationHandler::handleCommandLoggerState() { res.errorMessage()); return; } - rocksdb::SequenceNumber lastTick = db->GetLatestSequenceNumber(); - + rocksdb::SequenceNumber lastTick = latestSequenceNumber(); // "state" part builder.add("state", VPackValue(VPackValueType::Object)); builder.add("running", VPackValue(true)); - builder.add("lastLogTick", VPackValue(std::to_string(lastTick))); + builder.add("lastLogTick", VPackValue(StringUtils::itoa(lastTick))); builder.add("lastUncommittedLogTick", - VPackValue(std::to_string(lastTick + 1))); + VPackValue(StringUtils::itoa(lastTick + 1))); builder.add("totalEvents", VPackValue(0)); // s.numEvents + s.numEventsSync builder.add("time", VPackValue(utilities::timeString())); builder.close(); diff --git a/etc/testing/arangod-common.conf b/etc/testing/arangod-common.conf index 8cb85c7d25..0a65c87a4f 100644 --- a/etc/testing/arangod-common.conf +++ b/etc/testing/arangod-common.conf @@ -19,3 +19,6 @@ threads = 20 [ssl] keyfile = @TOP_DIR@/UnitTests/server.pem + +[cluster] + system-replication-factor = 1 From bd5737c5e7ab7d6d23dac37444d8e3b8d96c389d Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 25 Apr 2017 14:20:00 +0200 Subject: [PATCH 02/58] try to fix shutdown issues when collection opening thread is waiting forever for scheduler to finish --- arangod/GeneralServer/GeneralServer.cpp | 1 - .../GeneralServer/GeneralServerFeature.cpp | 10 +++--- arangod/GeneralServer/GeneralServerFeature.h | 1 - arangod/MMFiles/MMFilesCollection.cpp | 2 -- lib/Basics/LocalTaskQueue.cpp | 36 ++++++++++++++++++- lib/Basics/LocalTaskQueue.h | 9 +++++ 6 files changed, 48 insertions(+), 11 deletions(-) diff --git a/arangod/GeneralServer/GeneralServer.cpp b/arangod/GeneralServer/GeneralServer.cpp index bb7cf127c5..751ed1b192 100644 --- a/arangod/GeneralServer/GeneralServer.cpp +++ b/arangod/GeneralServer/GeneralServer.cpp @@ -48,7 +48,6 @@ using namespace arangodb::rest; GeneralServer::~GeneralServer() { for (auto& task : _listenTasks) { - task->stop(); delete task; } } diff --git a/arangod/GeneralServer/GeneralServerFeature.cpp b/arangod/GeneralServer/GeneralServerFeature.cpp index f40eced8df..a67dcde5cc 100644 --- a/arangod/GeneralServer/GeneralServerFeature.cpp +++ b/arangod/GeneralServer/GeneralServerFeature.cpp @@ -107,12 +107,6 @@ GeneralServerFeature::GeneralServerFeature( startsAfter("Upgrade"); } -GeneralServerFeature::~GeneralServerFeature() { - for (auto& server : _servers) { - delete server; - } -} - void GeneralServerFeature::collectOptions( std::shared_ptr options) { options->addSection("server", "Server features"); @@ -273,6 +267,10 @@ void GeneralServerFeature::stop() { } void GeneralServerFeature::unprepare() { + for (auto& server : _servers) { + delete server; + } + GENERAL_SERVER = nullptr; JOB_MANAGER = nullptr; HANDLER_FACTORY = nullptr; diff --git a/arangod/GeneralServer/GeneralServerFeature.h b/arangod/GeneralServer/GeneralServerFeature.h index 1cf5da37d7..43b1f41663 100644 --- a/arangod/GeneralServer/GeneralServerFeature.h +++ b/arangod/GeneralServer/GeneralServerFeature.h @@ -84,7 +84,6 @@ class GeneralServerFeature final public: explicit GeneralServerFeature(application_features::ApplicationServer*); - ~GeneralServerFeature(); public: void collectOptions(std::shared_ptr) override final; diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index 8bb08b6009..9ea7272eee 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -1585,8 +1585,6 @@ int MMFilesCollection::fillIndexes( blockSize = 1; } - ManagedDocumentResult mmdr; - std::vector> documents; documents.reserve(blockSize); diff --git a/lib/Basics/LocalTaskQueue.cpp b/lib/Basics/LocalTaskQueue.cpp index e20a3e1307..d11dae928f 100644 --- a/lib/Basics/LocalTaskQueue.cpp +++ b/lib/Basics/LocalTaskQueue.cpp @@ -23,8 +23,10 @@ #include "LocalTaskQueue.h" #include "Basics/ConditionLocker.h" +#include "Basics/Exceptions.h" #include "Basics/MutexLocker.h" #include "Basics/asio-helper.h" +#include "Logger/Logger.h" using namespace arangodb::basics; @@ -40,7 +42,16 @@ LocalTask::LocalTask(LocalTaskQueue* queue) : _queue(queue) {} void LocalTask::dispatch() { auto self = shared_from_this(); - _queue->ioService()->post([self, this]() { run(); }); + _queue->ioService()->post([self, this]() { + _queue->startTask(); + try { + run(); + _queue->stopTask(); + } catch (...) { + _queue->stopTask(); + throw; + } + }); } //////////////////////////////////////////////////////////////////////////////// @@ -83,6 +94,7 @@ LocalTaskQueue::LocalTaskQueue(boost::asio::io_service* ioService) _condition(), _mutex(), _missing(0), + _started(0), _status(TRI_ERROR_NO_ERROR) { TRI_ASSERT(_ioService != nullptr); } @@ -98,6 +110,16 @@ boost::asio::io_service* LocalTaskQueue::ioService() { return _ioService; } //////////////////////////////////////////////////////////////////////////////// LocalTaskQueue::~LocalTaskQueue() {} + +void LocalTaskQueue::startTask() { + CONDITION_LOCKER(guard, _condition); + ++_started; +} + +void LocalTaskQueue::stopTask() { + CONDITION_LOCKER(guard, _condition); + --_started; +} ////////////////////////////////////////////////////////////////////////////// /// @brief enqueue a task to be run @@ -159,6 +181,12 @@ void LocalTaskQueue::dispatchAndWait() { break; } + if (_missing > 0 && + _started == 0 && + _ioService->stopped()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN); + } + guard.wait(100000); } } @@ -182,6 +210,12 @@ void LocalTaskQueue::dispatchAndWait() { if (_missing == 0) { break; } + + if (_missing > 0 && + _started == 0 && + _ioService->stopped()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN); + } guard.wait(100000); } diff --git a/lib/Basics/LocalTaskQueue.h b/lib/Basics/LocalTaskQueue.h index 9dc957f81a..92de02e939 100644 --- a/lib/Basics/LocalTaskQueue.h +++ b/lib/Basics/LocalTaskQueue.h @@ -94,6 +94,9 @@ class LocalTaskQueue { ~LocalTaskQueue(); + void startTask(); + void stopTask(); + ////////////////////////////////////////////////////////////////////////////// /// @brief exposes underlying io_service ////////////////////////////////////////////////////////////////////////////// @@ -176,6 +179,12 @@ class LocalTaskQueue { ////////////////////////////////////////////////////////////////////////////// size_t _missing; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief number of dispatched and started tasks + ////////////////////////////////////////////////////////////////////////////// + + size_t _started; ////////////////////////////////////////////////////////////////////////////// /// @brief overall status of queue tasks From 69c2faf59254b11d69fc4d30aa5e083de1b84afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Tue, 25 Apr 2017 14:27:34 +0200 Subject: [PATCH 03/58] Fixed synced --- arangod/Replication/InitialSyncer.cpp | 126 ++++++++++++++++---------- 1 file changed, 80 insertions(+), 46 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 6cd266128f..938177157e 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -42,7 +42,7 @@ #include "RestServer/DatabaseFeature.h" #include "SimpleHttpClient/SimpleHttpClient.h" #include "SimpleHttpClient/SimpleHttpResult.h" -#include "StorageEngine/EngineSelectorFeature.h" +#include "StorageEngine/EngineSelectorFeature.h" b #include "Indexes/IndexIterator.h" #include "StorageEngine/StorageEngine.h" #include "Transaction/Helpers.h" @@ -1210,6 +1210,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, uint64_t localHash = 0x012345678; // chunk keys + revisionId std::vector> markers; + bool foundLowKey = false; auto resetChunk = [&] () -> void { sendExtendBatch(); @@ -1244,55 +1245,83 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, highKey = highSlice.copyString(); hashString = hashSlice.copyString(); localHash = 0x012345678; + foundLowKey = false; + }; + // set to first chunk + resetChunk(); + + std::function parseDoc = + [&] (VPackSlice doc, VPackSlice key) { + + bool rangeUneqal = false; + bool nextChunk = false; + + int cmp1 = key.compareString(lowKey.data(), lowKey.length()); + int cmp2 = key.compareString(highKey.data(), highKey.length()); + + if (cmp1 < 0) { + // smaller values than lowKey mean they don't exist remotely + trx.remove(collectionName, key, options); + return; + } if (cmp1 >= 0 && cmp2 <= 0) { + // we only need to hash we are in the range + if (cmp1 == 0) { + foundLowKey = true; + } else if (!foundLowKey && cmp1 > 0) { + rangeUneqal = true; + nextChunk = true; + } + + if (foundLowKey) { + VPackSlice revision = doc.get(StaticStrings::RevString); + localHash ^= key.hashString(); + localHash ^= revision.hash(); + + markers.emplace_back(key.copyString(), TRI_ExtractRevisionId(doc)); + + if (cmp2 == 0) {// found highKey + rangeUneqal = std::to_string(localHash) != hashString; + nextChunk = true; + } + } else if (cmp2 == 0) { + rangeUneqal = true; + nextChunk = true; + } + } else if (cmp2 > 0) { // higher than highKey + // current range was unequal and we did not find the + // high key. Load range and skip to next + rangeUneqal = true; + nextChunk = true; + } + + if (rangeUneqal) { + int res = syncChunkRocksDB(&trx, keysId, currentChunkId, + lowKey, highKey, + markers, errorMsg); + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + } + TRI_ASSERT(!rangeUneqal || rangeUneqal && nextChunk); // A => B + if (nextChunk && currentChunkId+1 < numChunks) { + currentChunkId++;// we are out of range, see next chunk + resetChunk(); + + // key is higher than upper bound, recheck the current document + if (cmp2 > 0) { + parseDoc(doc, key); + } + } }; - LogicalCollection* coll = trx.documentCollection(); - std::unique_ptr iterator = coll->getAllIterator(&trx, &mmdr, false); + std::unique_ptr iterator = col->getAllIterator(&trx, &mmdr, false); iterator->next([&] (DocumentIdentifierToken const& token) { - if (coll->readDocument(&trx, token, mmdr) == false) { + if (col->readDocument(&trx, token, mmdr) == false) { return; } VPackSlice doc(mmdr.vpack()); VPackSlice key = doc.get(StaticStrings::KeyString); - - int cmp1 = key.compareString(lowKey.data(), lowKey.length()); - int cmp2 = key.compareString(highKey.data(), highKey.length()); - if (cmp1 >= 0 && cmp2 <= 0) { - VPackSlice revision = doc.get(StaticStrings::RevString); - localHash ^= key.hashString(); - localHash ^= revision.hash(); - - VPackValueLength revLength; - char const* revCharPtr = revision.getString(revLength); - markers.emplace_back(key.copyString(), - StringUtils::uint64(revCharPtr, revLength)); - - if (cmp2 == 0) {// found highKey - if (std::to_string(localHash) != hashString) { - // TODO do something - int res = syncChunkRocksDB(&trx, keysId, currentChunkId, - lowKey, highKey, - markers, errorMsg); - if (res != TRI_ERROR_NO_ERROR) { - THROW_ARANGO_EXCEPTION(res); - } - } - // else: we will jump into if (cmp2 > 0) next - return; - } - } else if (cmp1 < 0) { - // smaller values than lowKey should never happen, unless - // we hit the - trx.remove(collectionName, key, options); - return; - } else if (cmp2 > 0) { // higher than highKey - if (currentChunkId+1 < numChunks) { - currentChunkId++;// we are out of range, see next chunk - resetChunk(); - } else { - LOG_TOPIC(ERR, Logger::FIXME) << "WTF iterator position beyond last chunk"; - } - } + parseDoc(doc, key); }, UINT64_MAX); res = trx.commit(); @@ -1327,7 +1356,7 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, std::to_string(chunkId) + "&chunkSize=" + std::to_string(chunkSize); - std::string progress = "fetching keys chunk " + std::to_string(chunkId) + "' from " + url; + std::string progress = "fetching keys chunk '" + std::to_string(chunkId) + "' from " + url; setProgress(progress); std::unique_ptr response( @@ -1395,6 +1424,8 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, TRI_ASSERT(numKeys > 0); size_t i = 0; + size_t nextStart = 0; + for (VPackSlice const& pair : VPackArrayIterator(responseBody)) { if (!pair.isArray() || pair.length() != 2) { @@ -1418,12 +1449,12 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, if (markers.empty()) { // no local markers toFetch.emplace_back(i); + i++; continue; } std::string const keyString = keySlice.copyString(); - - size_t nextStart = 0; + // remove keys not present anymore while (nextStart < markers.size()) { std::string const& localKey = markers[nextStart].first; @@ -1443,6 +1474,7 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, } } + // see if key exists DocumentIdentifierToken token = physical->lookupKey(trx, keySlice); if (!token._data) { // key not found locally @@ -1455,6 +1487,8 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, // a match - nothing to do! ++nextStart; } + + i++; } From 09e3a19a5b4978d807e8b4f3dd68bdf1af8ef5af Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 25 Apr 2017 14:41:04 +0200 Subject: [PATCH 04/58] try to fix shutdown issue --- arangod/Scheduler/ListenTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp index d3a3b71595..ea2fcb544e 100644 --- a/arangod/Scheduler/ListenTask.cpp +++ b/arangod/Scheduler/ListenTask.cpp @@ -106,4 +106,5 @@ void ListenTask::stop() { _bound = false; _acceptor->close(); + _acceptor.reset(); } From ab56530f764274680ba17b7912a0d066bd8b48c6 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Tue, 25 Apr 2017 14:49:19 +0200 Subject: [PATCH 05/58] Hand out distributeShardsLike in properties in arangosh. --- js/client/modules/@arangodb/arango-collection.js | 1 + 1 file changed, 1 insertion(+) diff --git a/js/client/modules/@arangodb/arango-collection.js b/js/client/modules/@arangodb/arango-collection.js index d50633156d..462b004685 100644 --- a/js/client/modules/@arangodb/arango-collection.js +++ b/js/client/modules/@arangodb/arango-collection.js @@ -336,6 +336,7 @@ ArangoCollection.prototype.properties = function (properties) { 'keyOptions': false, 'indexBuckets': true, 'replicationFactor': false, + 'distributeShardsLike': false, }; var a; From 3b17d9cdbc610b63db232b293a3e1fcc4cac3406 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 25 Apr 2017 14:50:58 +0200 Subject: [PATCH 06/58] add missing signaling of crashed state --- js/client/modules/@arangodb/process-utils.js | 1 + 1 file changed, 1 insertion(+) diff --git a/js/client/modules/@arangodb/process-utils.js b/js/client/modules/@arangodb/process-utils.js index d632956ede..a52de9c220 100644 --- a/js/client/modules/@arangodb/process-utils.js +++ b/js/client/modules/@arangodb/process-utils.js @@ -790,6 +790,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) { } else if (arangod.exitStatus.status !== 'TERMINATED') { if (arangod.exitStatus.hasOwnProperty('signal')) { analyzeServerCrash(arangod, options, 'instance Shutdown - ' + arangod.exitStatus.signal); + serverCrashed = true; } } else { print('Server shutdown: Success: pid', arangod.pid); From 2f968d649abace99aa6346e700aabab6268d12d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Tue, 25 Apr 2017 14:53:59 +0200 Subject: [PATCH 07/58] Fixed build --- arangod/Replication/InitialSyncer.cpp | 37 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 938177157e..186a3e84ff 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -42,7 +42,7 @@ #include "RestServer/DatabaseFeature.h" #include "SimpleHttpClient/SimpleHttpClient.h" #include "SimpleHttpClient/SimpleHttpResult.h" -#include "StorageEngine/EngineSelectorFeature.h" b +#include "StorageEngine/EngineSelectorFeature.h" #include "Indexes/IndexIterator.h" #include "StorageEngine/StorageEngine.h" #include "Transaction/Helpers.h" @@ -1211,6 +1211,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, // chunk keys + revisionId std::vector> markers; bool foundLowKey = false; + bool syncedRange = false; auto resetChunk = [&] () -> void { sendExtendBatch(); @@ -1246,6 +1247,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, hashString = hashSlice.copyString(); localHash = 0x012345678; foundLowKey = false; + syncedRange = false; }; // set to first chunk resetChunk(); @@ -1253,7 +1255,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, std::function parseDoc = [&] (VPackSlice doc, VPackSlice key) { - bool rangeUneqal = false; + bool rangeUneqal = true; bool nextChunk = false; int cmp1 = key.compareString(lowKey.data(), lowKey.length()); @@ -1268,7 +1270,6 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, if (cmp1 == 0) { foundLowKey = true; } else if (!foundLowKey && cmp1 > 0) { - rangeUneqal = true; nextChunk = true; } @@ -1284,7 +1285,6 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, nextChunk = true; } } else if (cmp2 == 0) { - rangeUneqal = true; nextChunk = true; } } else if (cmp2 > 0) { // higher than highKey @@ -1294,23 +1294,30 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, nextChunk = true; } - if (rangeUneqal) { - int res = syncChunkRocksDB(&trx, keysId, currentChunkId, - lowKey, highKey, - markers, errorMsg); - if (res != TRI_ERROR_NO_ERROR) { - THROW_ARANGO_EXCEPTION(res); - } - } TRI_ASSERT(!rangeUneqal || rangeUneqal && nextChunk); // A => B - if (nextChunk && currentChunkId+1 < numChunks) { - currentChunkId++;// we are out of range, see next chunk - resetChunk(); + if (nextChunk) { + if (rangeUneqal && !syncedRange) { + syncedRange = true; + int res = syncChunkRocksDB(&trx, keysId, currentChunkId, + lowKey, highKey, + markers, errorMsg); + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + } + + if (currentChunkId+1 < numChunks) { + currentChunkId++;// we are out of range, see next chunk + resetChunk(); + } // key is higher than upper bound, recheck the current document if (cmp2 > 0) { parseDoc(doc, key); } + } else { + LOG_TOPIC(ERR, Logger::FIXME) << "Did not expect keys higher" + << "than teh highest chunk"; } }; From 26d4fd61db87aba7dfe27bdc95d8ab1bc68aba46 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 25 Apr 2017 14:56:00 +0200 Subject: [PATCH 08/58] fix compile warnings --- arangod/Replication/InitialSyncer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 186a3e84ff..92456b3f09 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -1255,7 +1255,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, std::function parseDoc = [&] (VPackSlice doc, VPackSlice key) { - bool rangeUneqal = true; + bool rangeUnequal = true; bool nextChunk = false; int cmp1 = key.compareString(lowKey.data(), lowKey.length()); @@ -1281,7 +1281,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, markers.emplace_back(key.copyString(), TRI_ExtractRevisionId(doc)); if (cmp2 == 0) {// found highKey - rangeUneqal = std::to_string(localHash) != hashString; + rangeUnequal = std::to_string(localHash) != hashString; nextChunk = true; } } else if (cmp2 == 0) { @@ -1290,13 +1290,13 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, } else if (cmp2 > 0) { // higher than highKey // current range was unequal and we did not find the // high key. Load range and skip to next - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } - TRI_ASSERT(!rangeUneqal || rangeUneqal && nextChunk); // A => B + TRI_ASSERT(!rangeUnequal || (rangeUnequal && nextChunk)); // A => B if (nextChunk) { - if (rangeUneqal && !syncedRange) { + if (rangeUnequal && !syncedRange) { syncedRange = true; int res = syncChunkRocksDB(&trx, keysId, currentChunkId, lowKey, highKey, From fca8f01df6e6164b0ab4e21e85ef3198dcc9227b Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Tue, 25 Apr 2017 14:58:32 +0200 Subject: [PATCH 09/58] create logger state just in one place --- arangod/Replication/InitialSyncer.cpp | 16 +++--- arangod/RocksDBEngine/RocksDBEngine.cpp | 56 ++++++++++++++++++- arangod/RocksDBEngine/RocksDBEngine.h | 2 + .../RocksDBRestReplicationHandler.cpp | 49 +--------------- arangod/V8Server/v8-replication.cpp | 46 ++++++++------- 5 files changed, 89 insertions(+), 80 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 938177157e..08d4491423 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -42,7 +42,7 @@ #include "RestServer/DatabaseFeature.h" #include "SimpleHttpClient/SimpleHttpClient.h" #include "SimpleHttpClient/SimpleHttpResult.h" -#include "StorageEngine/EngineSelectorFeature.h" b +#include "StorageEngine/EngineSelectorFeature.h" #include "Indexes/IndexIterator.h" #include "StorageEngine/StorageEngine.h" #include "Transaction/Helpers.h" @@ -1253,7 +1253,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, std::function parseDoc = [&] (VPackSlice doc, VPackSlice key) { - bool rangeUneqal = false; + bool rangeUnequal = false; bool nextChunk = false; int cmp1 = key.compareString(lowKey.data(), lowKey.length()); @@ -1268,7 +1268,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, if (cmp1 == 0) { foundLowKey = true; } else if (!foundLowKey && cmp1 > 0) { - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } @@ -1280,21 +1280,21 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, markers.emplace_back(key.copyString(), TRI_ExtractRevisionId(doc)); if (cmp2 == 0) {// found highKey - rangeUneqal = std::to_string(localHash) != hashString; + rangeUnequal = std::to_string(localHash) != hashString; nextChunk = true; } } else if (cmp2 == 0) { - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } } else if (cmp2 > 0) { // higher than highKey // current range was unequal and we did not find the // high key. Load range and skip to next - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } - if (rangeUneqal) { + if (rangeUnequal) { int res = syncChunkRocksDB(&trx, keysId, currentChunkId, lowKey, highKey, markers, errorMsg); @@ -1302,7 +1302,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, THROW_ARANGO_EXCEPTION(res); } } - TRI_ASSERT(!rangeUneqal || rangeUneqal && nextChunk); // A => B + TRI_ASSERT(!rangeUnequal || (rangeUnequal && nextChunk)); // A => B if (nextChunk && currentChunkId+1 < numChunks) { currentChunkId++;// we are out of range, see next chunk resetChunk(); diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 9271bafd98..7b7f439d12 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -30,12 +30,14 @@ #include "Basics/StaticStrings.h" #include "Basics/Thread.h" #include "Basics/VelocyPackHelper.h" +#include "Basics/build.h" #include "GeneralServer/RestHandlerFactory.h" #include "Logger/Logger.h" #include "ProgramOptions/ProgramOptions.h" #include "ProgramOptions/Section.h" #include "RestHandler/RestHandlerCreator.h" #include "RestServer/DatabasePathFeature.h" +#include "RestServer/ServerIdFeature.h" #include "RestServer/ViewTypesFeature.h" #include "RocksDBEngine/RocksDBBackgroundThread.h" #include "RocksDBEngine/RocksDBCollection.h" @@ -54,6 +56,7 @@ #include "RocksDBEngine/RocksDBV8Functions.h" #include "RocksDBEngine/RocksDBValue.h" #include "RocksDBEngine/RocksDBView.h" +#include "VocBase/replication-applier.h" #include "VocBase/ticks.h" #include @@ -300,7 +303,6 @@ void RocksDBEngine::getDatabases(arangodb::velocypack::Builder& result) { rocksdb::ReadOptions readOptions; std::unique_ptr iter(_db->NewIterator(readOptions)); - result.openArray(); auto rSlice = rocksDBSlice(RocksDBEntryType::Database); for (iter->Seek(rSlice); iter->Valid() && iter->key().starts_with(rSlice); @@ -849,6 +851,58 @@ std::pair RocksDBEngine::mapObjectToCollection( return it->second; } +Result RocksDBEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder){ + Result res; + + rocksdb::Status status = _db->GetBaseDB()->SyncWAL(); + if (!status.ok()) { + res = rocksutils::convertStatus(status).errorNumber(); + return res; + } + + builder.add(VPackValue(VPackValueType::Object)); // Base + rocksdb::SequenceNumber lastTick = _db->GetLatestSequenceNumber(); + + // "state" part + builder.add("state", VPackValue(VPackValueType::Object)); //open + builder.add("running", VPackValue(true)); + builder.add("lastLogTick", VPackValue(std::to_string(lastTick))); + builder.add("lastUncommittedLogTick", VPackValue(std::to_string(lastTick))); + builder.add("totalEvents", VPackValue(0)); // s.numEvents + s.numEventsSync + builder.add("time", VPackValue(utilities::timeString())); + builder.close(); + + // "server" part + builder.add("server", VPackValue(VPackValueType::Object)); //open + builder.add("version", VPackValue(ARANGODB_VERSION)); + builder.add("serverId", VPackValue(std::to_string(ServerIdFeature::getId()))); + builder.close(); + + // "clients" part + builder.add("clients", VPackValue(VPackValueType::Array)); //open + if(vocbase != nullptr) { //add clients + auto allClients = vocbase->getReplicationClients(); + for (auto& it : allClients) { + // One client + builder.add(VPackValue(VPackValueType::Object)); + builder.add("serverId", VPackValue(std::to_string(std::get<0>(it)))); + + char buffer[21]; + TRI_GetTimeStampReplication(std::get<1>(it), &buffer[0], sizeof(buffer)); + builder.add("time", VPackValue(buffer)); + + builder.add("lastServedTick", VPackValue(std::to_string(std::get<2>(it)))); + + builder.close(); + } + } + builder.close(); // clients + + builder.close(); // base + + return res; +} + Result RocksDBEngine::dropDatabase(TRI_voc_tick_t id) { using namespace rocksutils; Result res; diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index 53fa0c4336..6019a99940 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -248,6 +248,8 @@ class RocksDBEngine final : public StorageEngine { void addCollectionMapping(uint64_t, TRI_voc_tick_t, TRI_voc_cid_t); std::pair mapObjectToCollection(uint64_t); + Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder); + private: Result dropDatabase(TRI_voc_tick_t); bool systemDatabaseExists(); diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index b1994ecf4f..dc36f31ec6 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -328,57 +328,12 @@ bool RocksDBRestReplicationHandler::isCoordinatorError() { void RocksDBRestReplicationHandler::handleCommandLoggerState() { VPackBuilder builder; - builder.add(VPackValue(VPackValueType::Object)); // Base - - // MMFilesLogfileManager::instance()->waitForSync(10.0); - // MMFilesLogfileManagerState const s = - // MMFilesLogfileManager::instance()->state(); - rocksdb::TransactionDB* db = - static_cast(EngineSelectorFeature::ENGINE)->db(); - rocksdb::Status status = db->GetBaseDB()->SyncWAL(); - if (!status.ok()) { - Result res = rocksutils::convertStatus(status).errorNumber(); + auto res = globalRocksEngine()->createLoggerState(_vocbase, builder); + if (res.fail()) { generateError(rest::ResponseCode::BAD, res.errorNumber(), res.errorMessage()); return; } - rocksdb::SequenceNumber lastTick = latestSequenceNumber(); - // "state" part - builder.add("state", VPackValue(VPackValueType::Object)); - builder.add("running", VPackValue(true)); - builder.add("lastLogTick", VPackValue(StringUtils::itoa(lastTick))); - builder.add("lastUncommittedLogTick", - VPackValue(StringUtils::itoa(lastTick + 1))); - builder.add("totalEvents", VPackValue(0)); // s.numEvents + s.numEventsSync - builder.add("time", VPackValue(utilities::timeString())); - builder.close(); - - // "server" part - builder.add("server", VPackValue(VPackValueType::Object)); - builder.add("version", VPackValue(ARANGODB_VERSION)); - builder.add("serverId", VPackValue(std::to_string(ServerIdFeature::getId()))); - builder.close(); - - // "clients" part - builder.add("clients", VPackValue(VPackValueType::Array)); - auto allClients = _vocbase->getReplicationClients(); - for (auto& it : allClients) { - // One client - builder.add(VPackValue(VPackValueType::Object)); - builder.add("serverId", VPackValue(std::to_string(std::get<0>(it)))); - - char buffer[21]; - TRI_GetTimeStampReplication(std::get<1>(it), &buffer[0], sizeof(buffer)); - builder.add("time", VPackValue(buffer)); - - builder.add("lastServedTick", VPackValue(std::to_string(std::get<2>(it)))); - - builder.close(); - } - builder.close(); // clients - - builder.close(); // base - generateResult(rest::ResponseCode::OK, builder.slice()); } diff --git a/arangod/V8Server/v8-replication.cpp b/arangod/V8Server/v8-replication.cpp index 2afc396872..df5dd33355 100644 --- a/arangod/V8Server/v8-replication.cpp +++ b/arangod/V8Server/v8-replication.cpp @@ -63,46 +63,44 @@ static void JS_StateLoggerReplication( v8::HandleScope scope(isolate); std::string engineName = EngineSelectorFeature::ENGINE->typeName(); + v8::Handle result = v8::Object::New(isolate); - v8::Handle state = v8::Object::New(isolate); - state->Set(TRI_V8_ASCII_STRING("running"), v8::True(isolate)); if(engineName == "mmfiles"){ + v8::Handle state = v8::Object::New(isolate); MMFilesLogfileManagerState const s = MMFilesLogfileManager::instance()->state(); + state->Set(TRI_V8_ASCII_STRING("running"), v8::True(isolate)); state->Set(TRI_V8_ASCII_STRING("lastLogTick"), TRI_V8UInt64String(isolate, s.lastCommittedTick)); state->Set(TRI_V8_ASCII_STRING("lastUncommittedLogTick"), TRI_V8UInt64String(isolate, s.lastAssignedTick)); state->Set(TRI_V8_ASCII_STRING("totalEvents"), v8::Number::New(isolate, static_cast(s.numEvents + s.numEventsSync))); state->Set(TRI_V8_ASCII_STRING("time"), TRI_V8_STD_STRING(s.timeString)); + result->Set(TRI_V8_ASCII_STRING("state"), state); + + v8::Handle server = v8::Object::New(isolate); + server->Set(TRI_V8_ASCII_STRING("version"), + TRI_V8_ASCII_STRING(ARANGODB_VERSION)); + server->Set(TRI_V8_ASCII_STRING("serverId"), + TRI_V8_STD_STRING(StringUtils::itoa(ServerIdFeature::getId()))); + result->Set(TRI_V8_ASCII_STRING("server"), server); + + v8::Handle clients = v8::Object::New(isolate); + result->Set(TRI_V8_ASCII_STRING("clients"), clients); } else if (engineName == "rocksdb") { - rocksdb::TransactionDB* db = rocksutils::globalRocksDB(); - uint64_t lastTick = db->GetLatestSequenceNumber(); - state->Set(TRI_V8_ASCII_STRING("lastLogTick"), - TRI_V8UInt64String(isolate, lastTick)); - state->Set(TRI_V8_ASCII_STRING("lastUncommittedLogTick"), - TRI_V8UInt64String(isolate, lastTick)); - state->Set(TRI_V8_ASCII_STRING("totalEvents"), - v8::Number::New(isolate, static_cast(0))); //s.numEvents + s.numEventsSync))); - state->Set(TRI_V8_ASCII_STRING("time"), TRI_V8_STD_STRING(utilities::timeString())); + VPackBuilder builder; + auto res = rocksutils::globalRocksEngine()->createLoggerState(nullptr,builder); + if(res.fail()){ + TRI_V8_THROW_EXCEPTION(res); + return; + } + v8::HandleresultValue = TRI_VPackToV8(isolate, builder.slice()); + result = v8::Handle::Cast(resultValue); } else { TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine"); return; } - v8::Handle result = v8::Object::New(isolate); - result->Set(TRI_V8_ASCII_STRING("state"), state); - - v8::Handle server = v8::Object::New(isolate); - server->Set(TRI_V8_ASCII_STRING("version"), - TRI_V8_ASCII_STRING(ARANGODB_VERSION)); - server->Set(TRI_V8_ASCII_STRING("serverId"), - TRI_V8_STD_STRING(StringUtils::itoa(ServerIdFeature::getId()))); - result->Set(TRI_V8_ASCII_STRING("server"), server); - - v8::Handle clients = v8::Object::New(isolate); - result->Set(TRI_V8_ASCII_STRING("clients"), clients); - TRI_V8_RETURN(result); TRI_V8_TRY_CATCH_END } From da5102ce6a48c217764eed4da0f4e6f8ac861346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Tue, 25 Apr 2017 15:05:18 +0200 Subject: [PATCH 10/58] fixed warning --- arangod/Replication/InitialSyncer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 186a3e84ff..d17120e5e5 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -1294,7 +1294,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, nextChunk = true; } - TRI_ASSERT(!rangeUneqal || rangeUneqal && nextChunk); // A => B + TRI_ASSERT(!rangeUneqal || (rangeUneqal && nextChunk)); // A => B if (nextChunk) { if (rangeUneqal && !syncedRange) { syncedRange = true; From d49e22f691dc3c4d6e932a7b73fd403b87326e1c Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 25 Apr 2017 15:08:14 +0200 Subject: [PATCH 11/58] HTTP collection API now finally exposes distributeShardsLike --- js/actions/_api/collection/app.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/js/actions/_api/collection/app.js b/js/actions/_api/collection/app.js index a5b57fc103..ae5b0a0fe3 100644 --- a/js/actions/_api/collection/app.js +++ b/js/actions/_api/collection/app.js @@ -64,10 +64,11 @@ function collectionRepresentation(collection, showProperties, showCount, showFig result.indexBuckets = properties.indexBuckets; if (cluster.isCoordinator()) { - result.shardKeys = properties.shardKeys; + result.avoidServers = properties.avoidServers; + result.distributeShardsLike = properties.distributeShardsLike; result.numberOfShards = properties.numberOfShards; result.replicationFactor = properties.replicationFactor; - result.avoidServers = properties.avoidServers; + result.shardKeys = properties.shardKeys; } } From e5f7303dade718086ee95db6ac3833577bb757dd Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 25 Apr 2017 15:08:45 +0200 Subject: [PATCH 12/58] Fixed drop in general-graph it now honors distributeShardsLike --- js/common/modules/@arangodb/general-graph.js | 88 ++++++++++++++------ 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/js/common/modules/@arangodb/general-graph.js b/js/common/modules/@arangodb/general-graph.js index 6e68f272ee..befb51ddbf 100644 --- a/js/common/modules/@arangodb/general-graph.js +++ b/js/common/modules/@arangodb/general-graph.js @@ -644,6 +644,10 @@ var checkIfMayBeDropped = function (colName, graphName, graphs) { var result = true; graphs.forEach( function (graph) { + if (result == false) { + // Short circuit + return; + } if (graph._key === graphName) { return; } @@ -2008,31 +2012,7 @@ exports._drop = function (graphId, dropCollections) { if (dropCollections === true) { graphs = exports._listObjects(); - var edgeDefinitions = graph.edgeDefinitions; - edgeDefinitions.forEach( - function (edgeDefinition) { - var from = edgeDefinition.from; - var to = edgeDefinition.to; - var collection = edgeDefinition.collection; - if (checkIfMayBeDropped(collection, graph._key, graphs)) { - db._drop(collection); - } - from.forEach( - function (col) { - if (checkIfMayBeDropped(col, graph._key, graphs)) { - db._drop(col); - } - } - ); - to.forEach( - function (col) { - if (checkIfMayBeDropped(col, graph._key, graphs)) { - db._drop(col); - } - } - ); - } - ); + var initialCollections = new Set(); // Here we collect the initial collection(s) // drop orphans if (!graph.orphanCollections) { graph.orphanCollections = []; @@ -2041,11 +2021,67 @@ exports._drop = function (graphId, dropCollections) { function (oC) { if (checkIfMayBeDropped(oC, graph._key, graphs)) { try { - db._drop(oC); + let colObj = db[oC]; + if (colObj.properties().distributeShardsLike !== undefined) { + db._drop(oC); + } else { + initialCollections.add(oC); + } } catch (ignore) {} } } ); + var edgeDefinitions = graph.edgeDefinitions; + edgeDefinitions.forEach( + function (edgeDefinition) { + var from = edgeDefinition.from; + var to = edgeDefinition.to; + var collection = edgeDefinition.collection; + if (checkIfMayBeDropped(collection, graph._key, graphs)) { + let colObj = db[collection]; + if (colObj.properties().distributeShardsLike !== undefined) { + db._drop(collection); + } else { + initialCollections.add(collection); + } + } + from.forEach( + function (col) { + if (checkIfMayBeDropped(col, graph._key, graphs)) { + try { + let colObj = db[col]; + if (colObj.properties().distributeShardsLike !== undefined) { + db._drop(col); + } else { + initialCollections.add(col); + } + } catch (ignore) {} + } + } + ); + to.forEach( + function (col) { + if (checkIfMayBeDropped(col, graph._key, graphs)) { + try { + let colObj = db[col]; + if (colObj.properties().distributeShardsLike !== undefined) { + db._drop(col); + } else { + initialCollections.add(col); + } + } catch (ignore2) {} + } + } + ); + } + ); + for (let c of initialCollections) { + try { + db._drop(c); + } catch (e) { + console.error("Failed to Drop: '" + c + "' reason: " + e.message); + } + } } gdb.remove(graphId); From e4b3f5c4ac7ee17fad741f33a39b3d9663233ebd Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Tue, 25 Apr 2017 15:12:22 +0200 Subject: [PATCH 13/58] fix warning --- arangod/Replication/InitialSyncer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index a21eeb7647..9eaf426430 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -1302,7 +1302,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, THROW_ARANGO_EXCEPTION(res); } } - TRI_ASSERT(!rangeUneqal || rangeUneqal && nextChunk); // A => B + TRI_ASSERT(!rangeUneqal || (rangeUneqal && nextChunk)); // A => B if (nextChunk && currentChunkId+1 < numChunks) { currentChunkId++;// we are out of range, see next chunk resetChunk(); From 9ab0ee817acdf4d07e8041faf81f5cb5bcc7094f Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 25 Apr 2017 15:17:33 +0200 Subject: [PATCH 14/58] jslint --- js/common/modules/@arangodb/general-graph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/common/modules/@arangodb/general-graph.js b/js/common/modules/@arangodb/general-graph.js index befb51ddbf..2064054250 100644 --- a/js/common/modules/@arangodb/general-graph.js +++ b/js/common/modules/@arangodb/general-graph.js @@ -644,7 +644,7 @@ var checkIfMayBeDropped = function (colName, graphName, graphs) { var result = true; graphs.forEach( function (graph) { - if (result == false) { + if (result === false) { // Short circuit return; } From 2b4038953e6afea43e816b95a0cf4857d4991835 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 25 Apr 2017 15:19:42 +0200 Subject: [PATCH 15/58] Remove unused CXX checks (were not in use and cmake has built in support for these) --- CMakeLists.txt | 2 +- arangod/CMakeLists.txt | 2 + cmake/CheckCXX11Features.cmake | 160 ------------------ .../cxx11-test-__func__.cpp | 8 - .../cxx11-test-atomic_uint_fast16_t.cpp | 6 - cmake/CheckCXX11Features/cxx11-test-auto.cpp | 12 -- .../cxx11-test-auto_fail_compile.cpp | 7 - .../cxx11-test-auto_ret_type.cpp | 8 - .../cxx11-test-class_override_final.cpp | 28 --- ...test-class_override_final_fail_compile.cpp | 25 --- .../cxx11-test-conditional.cpp | 17 -- .../cxx11-test-constexpr.cpp | 19 --- .../CheckCXX11Features/cxx11-test-cstdint.cpp | 11 -- .../cxx11-test-decltype.cpp | 10 -- .../cxx11-test-initializer_list.cpp | 27 --- .../CheckCXX11Features/cxx11-test-lambda.cpp | 5 - .../cxx11-test-long_long.cpp | 7 - cmake/CheckCXX11Features/cxx11-test-mutex.cpp | 6 - .../cxx11-test-noexcept.cpp | 8 - .../CheckCXX11Features/cxx11-test-nullptr.cpp | 6 - .../cxx11-test-nullptr_fail_compile.cpp | 6 - .../cxx11-test-range_based_for_loop.cpp | 15 -- cmake/CheckCXX11Features/cxx11-test-regex.cpp | 26 --- .../cxx11-test-rvalue-references.cpp | 57 ------- .../cxx11-test-shared_ptr.cpp | 6 - .../cxx11-test-sizeof_member.cpp | 14 -- .../cxx11-test-sizeof_member_fail.cpp | 9 - .../cxx11-test-static_assert.cpp | 5 - .../cxx11-test-static_assert_fail_compile.cpp | 5 - .../CheckCXX11Features/cxx11-test-thread.cpp | 6 - cmake/CheckCXX11Features/cxx11-test-tuple.cpp | 10 -- .../cxx11-test-unique_ptr.cpp | 6 - .../cxx11-test-variadic_templates.cpp | 23 --- .../cxx11-test-weak_ptr.cpp | 6 - 34 files changed, 3 insertions(+), 565 deletions(-) delete mode 100644 cmake/CheckCXX11Features.cmake delete mode 100644 cmake/CheckCXX11Features/cxx11-test-__func__.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-atomic_uint_fast16_t.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-auto.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-auto_fail_compile.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-auto_ret_type.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-class_override_final.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-class_override_final_fail_compile.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-conditional.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-constexpr.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-cstdint.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-decltype.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-initializer_list.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-lambda.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-long_long.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-mutex.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-noexcept.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-nullptr.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-nullptr_fail_compile.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-range_based_for_loop.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-regex.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-rvalue-references.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-shared_ptr.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-sizeof_member.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-sizeof_member_fail.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-static_assert.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-static_assert_fail_compile.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-thread.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-tuple.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-unique_ptr.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-variadic_templates.cpp delete mode 100644 cmake/CheckCXX11Features/cxx11-test-weak_ptr.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cc911ba39d..faa2ec65d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,8 +340,8 @@ if (CMAKE_COMPILER_IS_CLANG) endif () # need c++11 +# XXX this should really be set on a per target level using cmake compile_features capabilties set(CMAKE_CXX_STANDARD 11) -include(CheckCXX11Features) # need threads find_package(Threads REQUIRED) diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 1f4bbd3f04..cd3e5a34d8 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -414,6 +414,8 @@ target_link_libraries(${BIN_ARANGOD} arangoserver ) +target_compile_features(${BIN_ARANGOD} PRIVATE cxx_constexpr) + install( TARGETS ${BIN_ARANGOD} RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} diff --git a/cmake/CheckCXX11Features.cmake b/cmake/CheckCXX11Features.cmake deleted file mode 100644 index 0d334d52c4..0000000000 --- a/cmake/CheckCXX11Features.cmake +++ /dev/null @@ -1,160 +0,0 @@ -# - Check which parts of the C++11 standard the compiler supports -# -# When found it will set the following variables -# -# CXX11_COMPILER_FLAGS - the compiler flags needed to get C++11 features -# -# HAS_CXX11_AUTO - auto keyword -# HAS_CXX11_AUTO_RET_TYPE - function declaration with deduced return types -# HAS_CXX11_CLASS_OVERRIDE - override and final keywords for classes and methods -# HAS_CXX11_CONSTEXPR - constexpr keyword -# HAS_CXX11_CSTDINT_H - cstdint header -# HAS_CXX11_DECLTYPE - decltype keyword -# HAS_CXX11_FUNC - __func__ preprocessor constant -# HAS_CXX11_INITIALIZER_LIST - initializer list -# HAS_CXX11_LAMBDA - lambdas -# HAS_CXX11_LIB_REGEX - regex library -# HAS_CXX11_LONG_LONG - long long signed & unsigned types -# HAS_CXX11_NULLPTR - nullptr -# HAS_CXX11_RVALUE_REFERENCES - rvalue references -# HAS_CXX11_SIZEOF_MEMBER - sizeof() non-static members -# HAS_CXX11_STATIC_ASSERT - static_assert() -# HAS_CXX11_VARIADIC_TEMPLATES - variadic templates -# HAS_CXX11_SHARED_PTR - Shared Pointer -# HAS_CXX11_THREAD - thread -# HAS_CXX11_MUTEX - mutex -# HAS_CXX11_NOEXCEPT - noexcept -# HAS_CXX11_CONDITIONAL - conditional type definitions - -#============================================================================= -# Copyright 2011,2012 Rolf Eike Beer -# Copyright 2012 Andreas Weis -# Copyright 2014 Kaveh Vahedipour -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) - -# -# Each feature may have up to 3 checks, every one of them in it's own file -# FEATURE.cpp - example that must build and return 0 when run -# FEATURE_fail.cpp - example that must build, but may not return 0 when run -# FEATURE_fail_compile.cpp - example that must fail compilation -# -# The first one is mandatory, the latter 2 are optional and do not depend on -# each other (i.e. only one may be present). -# -# Modification for std::thread (Kaveh Vahdipour, Forschungszentrum Juelich) -# - -IF (NOT CMAKE_CXX_COMPILER_LOADED) - message(FATAL_ERROR "CheckCXX11Features modules only works if language CXX is enabled") -endif () - -cmake_minimum_required(VERSION 2.8.3) - -# -### Check for needed compiler flags -# -include(CheckCXXCompilerFlag) -check_cxx_compiler_flag("-std=c++11" _HAS_CXX11_FLAG) -if (NOT _HAS_CXX11_FLAG) - check_cxx_compiler_flag("-std=c++0x" _HAS_CXX0X_FLAG) -endif () - -if (_HAS_CXX11_FLAG) - set(CXX11_COMPILER_FLAGS "-std=c++11") -elseif (_HAS_CXX0X_FLAG) - set(CXX11_COMPILER_FLAGS "-std=c++0x") -endif () - -function(cxx11_check_feature FEATURE_NAME RESULT_VAR) - if (NOT DEFINED ${RESULT_VAR}) - set(_bindir "${CMAKE_CURRENT_BINARY_DIR}/cxx11/${FEATURE_NAME}") - - set(_SRCFILE_BASE ${CMAKE_CURRENT_LIST_DIR}/CheckCXX11Features/cxx11-test-${FEATURE_NAME}) - set(_LOG_NAME "\"${FEATURE_NAME}\"") - message(STATUS "Checking C++11 support for ${_LOG_NAME}") - - set(_SRCFILE "${_SRCFILE_BASE}.cpp") - set(_SRCFILE_FAIL "${_SRCFILE_BASE}_fail.cpp") - set(_SRCFILE_FAIL_COMPILE "${_SRCFILE_BASE}_fail_compile.cpp") - - if (CROSS_COMPILING) - try_compile(${RESULT_VAR} "${_bindir}" "${_SRCFILE}" - COMPILE_DEFINITIONS "${CXX11_COMPILER_FLAGS}") - if (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL}) - try_compile(${RESULT_VAR} "${_bindir}_fail" "${_SRCFILE_FAIL}" - COMPILE_DEFINITIONS "${CXX11_COMPILER_FLAGS}") - endif (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL}) - else (CROSS_COMPILING) - try_run(_RUN_RESULT_VAR _COMPILE_RESULT_VAR - "${_bindir}" "${_SRCFILE}" - COMPILE_DEFINITIONS "${CXX11_COMPILER_FLAGS}") - if (_COMPILE_RESULT_VAR AND NOT _RUN_RESULT_VAR) - set(${RESULT_VAR} TRUE) - else (_COMPILE_RESULT_VAR AND NOT _RUN_RESULT_VAR) - set(${RESULT_VAR} FALSE) - endif (_COMPILE_RESULT_VAR AND NOT _RUN_RESULT_VAR) - if (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL}) - try_run(_RUN_RESULT_VAR _COMPILE_RESULT_VAR - "${_bindir}_fail" "${_SRCFILE_FAIL}" - COMPILE_DEFINITIONS "${CXX11_COMPILER_FLAGS}") - if (_COMPILE_RESULT_VAR AND _RUN_RESULT_VAR) - set(${RESULT_VAR} TRUE) - else (_COMPILE_RESULT_VAR AND _RUN_RESULT_VAR) - set(${RESULT_VAR} FALSE) - endif (_COMPILE_RESULT_VAR AND _RUN_RESULT_VAR) - endif (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL}) - endif (CROSS_COMPILING) - if (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL_COMPILE}) - try_compile(_TMP_RESULT "${_bindir}_fail_compile" "${_SRCFILE_FAIL_COMPILE}" - COMPILE_DEFINITIONS "${CXX11_COMPILER_FLAGS}") - if (_TMP_RESULT) - set(${RESULT_VAR} FALSE) - else (_TMP_RESULT) - set(${RESULT_VAR} TRUE) - endif (_TMP_RESULT) - endif (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL_COMPILE}) - - if (${RESULT_VAR}) - message(STATUS "Checking C++11 support for ${_LOG_NAME}: works") - else (${RESULT_VAR}) - message(FATAL_ERROR "Checking C++11 support for ${_LOG_NAME}: not supported") - endif (${RESULT_VAR}) - set(${RESULT_VAR} ${${RESULT_VAR}} CACHE INTERNAL "C++11 support for ${_LOG_NAME}") - endif (NOT DEFINED ${RESULT_VAR}) -endfunction(cxx11_check_feature) - -cxx11_check_feature("__func__" HAS_CXX11_FUNC) -cxx11_check_feature("auto" HAS_CXX11_AUTO) -cxx11_check_feature("auto_ret_type" HAS_CXX11_AUTO_RET_TYPE) -#cxx11_check_feature("atomic_uint_fast16_t" HAS_CXX11_ATOMIC_UINT_FAST16_T) -cxx11_check_feature("class_override_final" HAS_CXX11_CLASS_OVERRIDE) -cxx11_check_feature("constexpr" HAS_CXX11_CONSTEXPR) -cxx11_check_feature("conditional" HAS_CXX11_CONDITIONAL) -#cxx11_check_feature("cstdint" HAS_CXX11_CSTDINT_H) -cxx11_check_feature("decltype" HAS_CXX11_DECLTYPE) -cxx11_check_feature("initializer_list" HAS_CXX11_INITIALIZER_LIST) -cxx11_check_feature("lambda" HAS_CXX11_LAMBDA) -cxx11_check_feature("range_based_for_loop" HAS_CXX11_RANGE_BASED_FOR_LOOP) -#cxx11_check_feature("long_long" HAS_CXX11_LONG_LONG) -cxx11_check_feature("nullptr" HAS_CXX11_NULLPTR) -cxx11_check_feature("tuple" HAS_CXX11_TUPLE) -cxx11_check_feature("regex" HAS_CXX11_LIB_REGEX) -cxx11_check_feature("rvalue-references" HAS_CXX11_RVALUE_REFERENCES) -cxx11_check_feature("sizeof_member" HAS_CXX11_SIZEOF_MEMBER) -cxx11_check_feature("static_assert" HAS_CXX11_STATIC_ASSERT) -cxx11_check_feature("variadic_templates" HAS_CXX11_VARIADIC_TEMPLATES) -cxx11_check_feature("shared_ptr" HAS_CXX11_SHARED_PTR) -cxx11_check_feature("unique_ptr" HAS_CXX11_UNIQUE_PTR) -cxx11_check_feature("weak_ptr" HAS_CXX11_WEAK_PTR) -cxx11_check_feature("thread" HAS_CXX11_THREAD) -cxx11_check_feature("mutex" HAS_CXX11_MUTEX) -cxx11_check_feature("noexcept" HAS_CXX11_NOEXCEPT) diff --git a/cmake/CheckCXX11Features/cxx11-test-__func__.cpp b/cmake/CheckCXX11Features/cxx11-test-__func__.cpp deleted file mode 100644 index 3bfd8a8935..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-__func__.cpp +++ /dev/null @@ -1,8 +0,0 @@ -int main(void) -{ - if (!__func__) - return 1; - if (!(*__func__)) - return 1; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-atomic_uint_fast16_t.cpp b/cmake/CheckCXX11Features/cxx11-test-atomic_uint_fast16_t.cpp deleted file mode 100644 index d95ea4781c..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-atomic_uint_fast16_t.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main () { - std::atomic_uint_fast16_t a; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-auto.cpp b/cmake/CheckCXX11Features/cxx11-test-auto.cpp deleted file mode 100644 index 948648e993..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-auto.cpp +++ /dev/null @@ -1,12 +0,0 @@ - -int main() -{ - auto i = 5; - auto f = 3.14159f; - auto d = 3.14159; - bool ret = ( - (sizeof(f) < sizeof(d)) && - (sizeof(i) == sizeof(int)) - ); - return ret ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-auto_fail_compile.cpp b/cmake/CheckCXX11Features/cxx11-test-auto_fail_compile.cpp deleted file mode 100644 index 3c0e3f2e01..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-auto_fail_compile.cpp +++ /dev/null @@ -1,7 +0,0 @@ -int main(void) -{ - // must fail because there is no initializer - auto i; - - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-auto_ret_type.cpp b/cmake/CheckCXX11Features/cxx11-test-auto_ret_type.cpp deleted file mode 100644 index 937b683555..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-auto_ret_type.cpp +++ /dev/null @@ -1,8 +0,0 @@ -auto foo(int i) -> int { - return i - 1; -} - -int main() -{ - return foo(1); -} diff --git a/cmake/CheckCXX11Features/cxx11-test-class_override_final.cpp b/cmake/CheckCXX11Features/cxx11-test-class_override_final.cpp deleted file mode 100644 index bcdc9e17c3..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-class_override_final.cpp +++ /dev/null @@ -1,28 +0,0 @@ -class base { -public: - virtual int foo(int a) - { return 4 + a; } - int bar(int a) - { return a - 2; } -}; - -class sub final : public base { -public: - virtual int foo(int a) override - { return 8 + 2 * a; }; -}; - -class sub2 final : public base { -public: - virtual int foo(int a) override final - { return 8 + 2 * a; }; -}; - -int main(void) -{ - base b; - sub s; - sub2 t; - - return (b.foo(2) * 2 == s.foo(2) && b.foo(2) * 2 == t.foo(2) ) ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-class_override_final_fail_compile.cpp b/cmake/CheckCXX11Features/cxx11-test-class_override_final_fail_compile.cpp deleted file mode 100644 index bc00b278ed..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-class_override_final_fail_compile.cpp +++ /dev/null @@ -1,25 +0,0 @@ -class base { -public: - virtual int foo(int a) - { return 4 + a; } - virtual int bar(int a) final - { return a - 2; } -}; - -class sub final : public base { -public: - virtual int foo(int a) override - { return 8 + 2 * a; }; - virtual int bar(int a) - { return a; } -}; - -class impossible : public sub { }; - -int main(void) -{ - base b; - sub s; - - return 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-conditional.cpp b/cmake/CheckCXX11Features/cxx11-test-conditional.cpp deleted file mode 100644 index 485a3138cf..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-conditional.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - -template class A { -public: - typedef typename std::conditional::type StringType; - A() : s(""), t(0) {} - virtual ~A () {} -private: - StringType s; - T t; -}; - -int main() { - A a; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-constexpr.cpp b/cmake/CheckCXX11Features/cxx11-test-constexpr.cpp deleted file mode 100644 index ed624512d8..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-constexpr.cpp +++ /dev/null @@ -1,19 +0,0 @@ -constexpr int square(int x) -{ - return x*x; -} - -constexpr int the_answer() -{ - return 42; -} - -int main() -{ - int test_arr[square(3)]; - bool ret = ( - (square(the_answer()) == 1764) && - (sizeof(test_arr)/sizeof(test_arr[0]) == 9) - ); - return ret ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-cstdint.cpp b/cmake/CheckCXX11Features/cxx11-test-cstdint.cpp deleted file mode 100644 index ca2c72ddd0..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-cstdint.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include - -int main() -{ - bool test = - (sizeof(int8_t) == 1) && - (sizeof(int16_t) == 2) && - (sizeof(int32_t) == 4) && - (sizeof(int64_t) == 8); - return test ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-decltype.cpp b/cmake/CheckCXX11Features/cxx11-test-decltype.cpp deleted file mode 100644 index 0dbb1cc520..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-decltype.cpp +++ /dev/null @@ -1,10 +0,0 @@ -bool check_size(int i) -{ - return sizeof(int) == sizeof(decltype(i)); -} - -int main() -{ - bool ret = check_size(42); - return ret ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-initializer_list.cpp b/cmake/CheckCXX11Features/cxx11-test-initializer_list.cpp deleted file mode 100644 index 35e6c38429..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-initializer_list.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include - -class seq { -public: - seq(std::initializer_list list); - - int length() const; -private: - std::vector m_v; -}; - -seq::seq(std::initializer_list list) - : m_v(list) -{ -} - -int seq::length() const -{ - return m_v.size(); -} - -int main(void) -{ - seq a = {18, 20, 2, 0, 4, 7}; - - return (a.length() == 6) ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-lambda.cpp b/cmake/CheckCXX11Features/cxx11-test-lambda.cpp deleted file mode 100644 index 4c33ed58dd..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-lambda.cpp +++ /dev/null @@ -1,5 +0,0 @@ -int main() -{ - int ret = 0; - return ([&ret]() -> int { return ret; })(); -} diff --git a/cmake/CheckCXX11Features/cxx11-test-long_long.cpp b/cmake/CheckCXX11Features/cxx11-test-long_long.cpp deleted file mode 100644 index 091112756a..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-long_long.cpp +++ /dev/null @@ -1,7 +0,0 @@ -int main(void) -{ - long long l; - unsigned long long ul; - - return ((sizeof(l) >= 8) && (sizeof(ul) >= 8)) ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-mutex.cpp b/cmake/CheckCXX11Features/cxx11-test-mutex.cpp deleted file mode 100644 index d4245c9dde..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-mutex.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::mutex _mutex; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-noexcept.cpp b/cmake/CheckCXX11Features/cxx11-test-noexcept.cpp deleted file mode 100644 index 59ac0f8f06..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-noexcept.cpp +++ /dev/null @@ -1,8 +0,0 @@ -volatile void dummy () noexcept { - int a = 0; -} - -int main () { - dummy(); - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-nullptr.cpp b/cmake/CheckCXX11Features/cxx11-test-nullptr.cpp deleted file mode 100644 index 9f41071531..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-nullptr.cpp +++ /dev/null @@ -1,6 +0,0 @@ -int main(void) -{ - void *v = nullptr; - - return v ? 1 : 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-nullptr_fail_compile.cpp b/cmake/CheckCXX11Features/cxx11-test-nullptr_fail_compile.cpp deleted file mode 100644 index 6a002bcb78..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-nullptr_fail_compile.cpp +++ /dev/null @@ -1,6 +0,0 @@ -int main(void) -{ - int i = nullptr; - - return 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-range_based_for_loop.cpp b/cmake/CheckCXX11Features/cxx11-test-range_based_for_loop.cpp deleted file mode 100644 index a3a293c718..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-range_based_for_loop.cpp +++ /dev/null @@ -1,15 +0,0 @@ - - -int main() { - int my_array[5] = {1, 2, 3, 4, 5}; - - for (int &x : my_array) { - x *= 2; - } - - for (auto &x : my_array) { - x *= 2; - } - -} - diff --git a/cmake/CheckCXX11Features/cxx11-test-regex.cpp b/cmake/CheckCXX11Features/cxx11-test-regex.cpp deleted file mode 100644 index 2fe01c4fc9..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-regex.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include - -int parse_line(std::string const& line) -{ - std::string tmp; - if(std::regex_search(line, std::regex("(\\s)+(-)?(\\d)+//(-)?(\\d)+(\\s)+"))) { - tmp = std::regex_replace(line, std::regex("(-)?(\\d)+//(-)?(\\d)+"), std::string("V")); - } else if(std::regex_search(line, std::regex("(\\s)+(-)?(\\d)+/(-)?(\\d)+(\\s)+"))) { - tmp = std::regex_replace(line, std::regex("(-)?(\\d)+/(-)?(\\d)+"), std::string("V")); - } else if(std::regex_search(line, std::regex("(\\s)+(-)?(\\d)+/(-)?(\\d)+/(-)?(\\d)+(\\s)+"))) { - tmp = std::regex_replace(line, std::regex("(-)?(\\d)+/(-)?(\\d)+/(-)?(\\d)+"), std::string("V")); - } else { - tmp = std::regex_replace(line, std::regex("(-)?(\\d)+"), std::string("V")); - } - return static_cast(std::count(tmp.begin(), tmp.end(), 'V')); -} - -int main() -{ - bool test = (parse_line("f 7/7/7 -3/3/-3 2/-2/2") == 3) && - (parse_line("f 7//7 3//-3 -2//2") == 3) && - (parse_line("f 7/7 3/-3 -2/2") == 3) && - (parse_line("f 7 3 -2") == 3); - return test ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-rvalue-references.cpp b/cmake/CheckCXX11Features/cxx11-test-rvalue-references.cpp deleted file mode 100644 index e6e7e5a98a..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-rvalue-references.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include - -class rvmove { -public: - void *ptr; - char *array; - - rvmove() - : ptr(0), - array(new char[10]) - { - ptr = this; - } - - rvmove(rvmove &&other) - : ptr(other.ptr), - array(other.array) - { - other.array = 0; - other.ptr = 0; - } - - ~rvmove() - { - assert(((ptr != 0) && (array != 0)) || ((ptr == 0) && (array == 0))); - delete[] array; - } - - rvmove &operator=(rvmove &&other) - { - delete[] array; - ptr = other.ptr; - array = other.array; - other.array = 0; - other.ptr = 0; - return *this; - } - - static rvmove create() - { - return rvmove(); - } -private: - rvmove(const rvmove &); - rvmove &operator=(const rvmove &); -}; - -int main() -{ - rvmove mine; - if (mine.ptr != &mine) - return 1; - mine = rvmove::create(); - if (mine.ptr == &mine) - return 1; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-shared_ptr.cpp b/cmake/CheckCXX11Features/cxx11-test-shared_ptr.cpp deleted file mode 100644 index ddde711ef7..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-shared_ptr.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::shared_ptr test; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-sizeof_member.cpp b/cmake/CheckCXX11Features/cxx11-test-sizeof_member.cpp deleted file mode 100644 index 4902fc73ee..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-sizeof_member.cpp +++ /dev/null @@ -1,14 +0,0 @@ -struct foo { - char bar; - int baz; -}; - -int main(void) -{ - bool ret = ( - (sizeof(foo::bar) == 1) && - (sizeof(foo::baz) >= sizeof(foo::bar)) && - (sizeof(foo) >= sizeof(foo::bar) + sizeof(foo::baz)) - ); - return ret ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-sizeof_member_fail.cpp b/cmake/CheckCXX11Features/cxx11-test-sizeof_member_fail.cpp deleted file mode 100644 index 0348c2cea8..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-sizeof_member_fail.cpp +++ /dev/null @@ -1,9 +0,0 @@ -struct foo { - int baz; - double bar; -}; - -int main(void) -{ - return (sizeof(foo::bar) == 4) ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-static_assert.cpp b/cmake/CheckCXX11Features/cxx11-test-static_assert.cpp deleted file mode 100644 index 47c2fefb8b..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-static_assert.cpp +++ /dev/null @@ -1,5 +0,0 @@ -int main(void) -{ - static_assert(0 < 1, "your ordering of integers is screwed"); - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-static_assert_fail_compile.cpp b/cmake/CheckCXX11Features/cxx11-test-static_assert_fail_compile.cpp deleted file mode 100644 index 362fcdde9c..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-static_assert_fail_compile.cpp +++ /dev/null @@ -1,5 +0,0 @@ -int main(void) -{ - static_assert(1 < 0, "your ordering of integers is screwed"); - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-thread.cpp b/cmake/CheckCXX11Features/cxx11-test-thread.cpp deleted file mode 100644 index 9a4488c1cc..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-thread.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::thread test; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-tuple.cpp b/cmake/CheckCXX11Features/cxx11-test-tuple.cpp deleted file mode 100644 index 9d9d02dcdd..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-tuple.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include - -int main () { - typedef std::tuple test_tuple; - long lengthy = 12; - test_tuple proof (18, 6.5, lengthy, "Ciao!"); - lengthy = std::get<0>(proof); - std::get<3>(proof) = " Beautiful!"; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-unique_ptr.cpp b/cmake/CheckCXX11Features/cxx11-test-unique_ptr.cpp deleted file mode 100644 index 04c2fc8dd9..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-unique_ptr.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::unique_ptr test; - return 0; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-variadic_templates.cpp b/cmake/CheckCXX11Features/cxx11-test-variadic_templates.cpp deleted file mode 100644 index 4518e886fd..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-variadic_templates.cpp +++ /dev/null @@ -1,23 +0,0 @@ -int Accumulate() -{ - return 0; -} - -template -int Accumulate(T v, Ts... vs) -{ - return v + Accumulate(vs...); -} - -template -int CountElements() -{ - return sizeof...(Is); -} - -int main() -{ - int acc = Accumulate(1, 2, 3, 4, -5); - int count = CountElements<1,2,3,4,5>(); - return ((acc == 5) && (count == 5)) ? 0 : 1; -} diff --git a/cmake/CheckCXX11Features/cxx11-test-weak_ptr.cpp b/cmake/CheckCXX11Features/cxx11-test-weak_ptr.cpp deleted file mode 100644 index 6398d3f7e3..0000000000 --- a/cmake/CheckCXX11Features/cxx11-test-weak_ptr.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::weak_ptr test; - return 0; -} From b0b919d9cc04211cadcb5324f3b6551f26a5d483 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 25 Apr 2017 15:31:10 +0200 Subject: [PATCH 16/58] portabilize the output of the debugger hint command --- js/client/modules/@arangodb/crash-utils.js | 27 ++++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/js/client/modules/@arangodb/crash-utils.js b/js/client/modules/@arangodb/crash-utils.js index 39b9385d7b..09cd099048 100644 --- a/js/client/modules/@arangodb/crash-utils.js +++ b/js/client/modules/@arangodb/crash-utils.js @@ -80,6 +80,15 @@ function analyzeCoreDump (instanceInfo, options, storeArangodPath, pid) { executeExternalAndWait('/bin/bash', args); GDB_OUTPUT = fs.read(gdbOutputFile); print(GDB_OUTPUT); + + command = 'gdb ' + storeArangodPath + ' '; + + if (options.coreDirectory === '') { + command += 'core'; + } else { + command += options.coreDirectory; + } + return command; } // ////////////////////////////////////////////////////////////////////////////// @@ -112,6 +121,7 @@ function analyzeCoreDumpMac (instanceInfo, options, storeArangodPath, pid) { executeExternalAndWait('/bin/bash', args); GDB_OUTPUT = fs.read(lldbOutputFile); print(GDB_OUTPUT); + return 'lldb ' + storeArangodPath + ' -c /cores/core.' + pid; } // ////////////////////////////////////////////////////////////////////////////// @@ -144,6 +154,8 @@ function analyzeCoreDumpWindows (instanceInfo) { print('running cdb ' + JSON.stringify(args)); executeExternalAndWait('cdb', args); + + return 'cdb ' + args.join(' '); } // ////////////////////////////////////////////////////////////////////////////// @@ -189,24 +201,19 @@ function analyzeCrash (binary, arangod, options, checkStr) { yaml.safeDump(arangod) + 'marking build as crashy.' + RESET); - let corePath = (options.coreDirectory === '') - ? 'core' - : options.coreDirectory; - - arangod.exitStatus.gdbHint = 'Run debugger with "gdb ' + - storeArangodPath + ' ' + corePath; - + let hint = ''; if (platform.substr(0, 3) === 'win') { // Windows: wait for procdump to do its job... statusExternal(arangod.monitor, true); - analyzeCoreDumpWindows(arangod); + hint = analyzeCoreDumpWindows(arangod); } else if (platform === 'darwin') { fs.copyFile(binary, storeArangodPath); - analyzeCoreDumpMac(arangod, options, storeArangodPath, arangod.pid); + hint = analyzeCoreDumpMac(arangod, options, storeArangodPath, arangod.pid); } else { fs.copyFile(binary, storeArangodPath); - analyzeCoreDump(arangod, options, storeArangodPath, arangod.pid); + hint = analyzeCoreDump(arangod, options, storeArangodPath, arangod.pid); } + arangod.exitStatus.gdbHint = 'Run debugger with "' + hint + '"'; print(RESET); } From 7344a036237d3a9ca1bbf49dc33928b45058e851 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 25 Apr 2017 15:47:11 +0200 Subject: [PATCH 17/58] fix crashes --- arangod/Indexes/Index.cpp | 2 +- arangod/Indexes/Index.h | 2 +- arangod/MMFiles/MMFilesCollection.cpp | 31 +++++++++++----------- arangod/MMFiles/MMFilesCollection.h | 2 +- arangod/MMFiles/MMFilesEdgeIndex.cpp | 2 +- arangod/MMFiles/MMFilesEdgeIndex.h | 2 +- arangod/MMFiles/MMFilesHashIndex.cpp | 6 ++--- arangod/MMFiles/MMFilesHashIndex.h | 6 ++--- arangod/RocksDBEngine/RocksDBEdgeIndex.cpp | 2 +- arangod/RocksDBEngine/RocksDBEdgeIndex.h | 2 +- lib/Basics/AssocMulti.h | 2 +- lib/Basics/AssocMultiHelpers.h | 4 +-- lib/Basics/AssocUnique.h | 2 +- lib/Basics/AssocUniqueHelpers.h | 4 +-- lib/Basics/LocalTaskQueue.cpp | 4 +-- lib/Basics/LocalTaskQueue.h | 8 +++--- 16 files changed, 41 insertions(+), 40 deletions(-) diff --git a/arangod/Indexes/Index.cpp b/arangod/Indexes/Index.cpp index b34f480615..98381953df 100644 --- a/arangod/Indexes/Index.cpp +++ b/arangod/Indexes/Index.cpp @@ -514,7 +514,7 @@ void Index::batchInsert( transaction::Methods* trx, std::vector> const& documents, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { for (auto const& it : documents) { int status = insert(trx, it.first, it.second, false); if (status != TRI_ERROR_NO_ERROR) { diff --git a/arangod/Indexes/Index.h b/arangod/Indexes/Index.h index 9481f512ce..831dc7fef1 100644 --- a/arangod/Indexes/Index.h +++ b/arangod/Indexes/Index.h @@ -250,7 +250,7 @@ class Index { virtual void batchInsert( transaction::Methods*, std::vector> const&, - arangodb::basics::LocalTaskQueue* queue = nullptr); + std::shared_ptr queue); virtual int unload() = 0; diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index 9ea7272eee..484b4708cb 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -73,7 +73,7 @@ namespace { class MMFilesIndexFillerTask : public basics::LocalTask { public: MMFilesIndexFillerTask( - basics::LocalTaskQueue* queue, transaction::Methods* trx, Index* idx, + std::shared_ptr queue, transaction::Methods* trx, Index* idx, std::vector> const& documents) : LocalTask(queue), _trx(trx), _idx(idx), _documents(documents) {} @@ -1464,7 +1464,7 @@ bool MMFilesCollection::openIndex(VPackSlice const& description, /// @brief initializes an index with a set of existing documents void MMFilesCollection::fillIndex( - arangodb::basics::LocalTaskQueue* queue, transaction::Methods* trx, + std::shared_ptr queue, transaction::Methods* trx, arangodb::Index* idx, std::vector> const& documents, bool skipPersistent) { @@ -1554,12 +1554,13 @@ int MMFilesCollection::fillIndexes( TRI_ASSERT(SchedulerFeature::SCHEDULER != nullptr); auto ioService = SchedulerFeature::SCHEDULER->ioService(); TRI_ASSERT(ioService != nullptr); - arangodb::basics::LocalTaskQueue queue(ioService); - + PerformanceLogScope logScope( std::string("fill-indexes-document-collection { collection: ") + _logicalCollection->vocbase()->name() + "/" + _logicalCollection->name() + " }, indexes: " + std::to_string(n - 1)); + + auto queue = std::make_shared(ioService); try { TRI_ASSERT(!ServerState::instance()->isCoordinator()); @@ -1594,12 +1595,12 @@ int MMFilesCollection::fillIndexes( if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { continue; } - fillIndex(&queue, trx, idx.get(), documents, skipPersistent); + fillIndex(queue, trx, idx.get(), documents, skipPersistent); } - queue.dispatchAndWait(); + queue->dispatchAndWait(); - if (queue.status() != TRI_ERROR_NO_ERROR) { + if (queue->status() != TRI_ERROR_NO_ERROR) { rollbackAll(); rolledBack = true; } @@ -1626,7 +1627,7 @@ int MMFilesCollection::fillIndexes( if (documents.size() == blockSize) { // now actually fill the secondary indexes insertInAllIndexes(); - if (queue.status() != TRI_ERROR_NO_ERROR) { + if (queue->status() != TRI_ERROR_NO_ERROR) { break; } documents.clear(); @@ -1636,33 +1637,33 @@ int MMFilesCollection::fillIndexes( } // process the remainder of the documents - if (queue.status() == TRI_ERROR_NO_ERROR && !documents.empty()) { + if (queue->status() == TRI_ERROR_NO_ERROR && !documents.empty()) { insertInAllIndexes(); } } catch (arangodb::basics::Exception const& ex) { - queue.setStatus(ex.code()); + queue->setStatus(ex.code()); LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "caught exception while filling indexes: " << ex.what(); } catch (std::bad_alloc const&) { - queue.setStatus(TRI_ERROR_OUT_OF_MEMORY); + queue->setStatus(TRI_ERROR_OUT_OF_MEMORY); } catch (std::exception const& ex) { LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "caught exception while filling indexes: " << ex.what(); - queue.setStatus(TRI_ERROR_INTERNAL); + queue->setStatus(TRI_ERROR_INTERNAL); } catch (...) { LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "caught unknown exception while filling indexes"; - queue.setStatus(TRI_ERROR_INTERNAL); + queue->setStatus(TRI_ERROR_INTERNAL); } - if (queue.status() != TRI_ERROR_NO_ERROR && !rolledBack) { + if (queue->status() != TRI_ERROR_NO_ERROR && !rolledBack) { try { rollbackAll(); } catch (...) { } } - return queue.status(); + return queue->status(); } /// @brief opens an existing collection diff --git a/arangod/MMFiles/MMFilesCollection.h b/arangod/MMFiles/MMFilesCollection.h index dd186b0796..6d0d38b998 100644 --- a/arangod/MMFiles/MMFilesCollection.h +++ b/arangod/MMFiles/MMFilesCollection.h @@ -398,7 +398,7 @@ class MMFilesCollection final : public PhysicalCollection { bool openIndex(VPackSlice const& description, transaction::Methods* trx); /// @brief initializes an index with all existing documents - void fillIndex(basics::LocalTaskQueue*, transaction::Methods*, Index*, + void fillIndex(std::shared_ptr, transaction::Methods*, Index*, std::vector> const&, bool); diff --git a/arangod/MMFiles/MMFilesEdgeIndex.cpp b/arangod/MMFiles/MMFilesEdgeIndex.cpp index dd70f6e5cd..c9c15b485a 100644 --- a/arangod/MMFiles/MMFilesEdgeIndex.cpp +++ b/arangod/MMFiles/MMFilesEdgeIndex.cpp @@ -329,7 +329,7 @@ int MMFilesEdgeIndex::remove(transaction::Methods* trx, void MMFilesEdgeIndex::batchInsert( transaction::Methods* trx, std::vector> const& documents, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { if (documents.empty()) { return; } diff --git a/arangod/MMFiles/MMFilesEdgeIndex.h b/arangod/MMFiles/MMFilesEdgeIndex.h index e42a84a22e..86bfdd6883 100644 --- a/arangod/MMFiles/MMFilesEdgeIndex.h +++ b/arangod/MMFiles/MMFilesEdgeIndex.h @@ -111,7 +111,7 @@ class MMFilesEdgeIndex final : public Index { void batchInsert(transaction::Methods*, std::vector> const&, - arangodb::basics::LocalTaskQueue*) override; + std::shared_ptr) override; int unload() override; diff --git a/arangod/MMFiles/MMFilesHashIndex.cpp b/arangod/MMFiles/MMFilesHashIndex.cpp index 74c3a2c2da..38a1490481 100644 --- a/arangod/MMFiles/MMFilesHashIndex.cpp +++ b/arangod/MMFiles/MMFilesHashIndex.cpp @@ -644,7 +644,7 @@ int MMFilesHashIndex::remove(transaction::Methods* trx, void MMFilesHashIndex::batchInsert( transaction::Methods* trx, std::vector> const& documents, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { TRI_ASSERT(queue != nullptr); if (_unique) { batchInsertUnique(trx, documents, queue); @@ -760,7 +760,7 @@ int MMFilesHashIndex::insertUnique(transaction::Methods* trx, void MMFilesHashIndex::batchInsertUnique( transaction::Methods* trx, std::vector> const& documents, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { TRI_ASSERT(queue != nullptr); std::shared_ptr> elements; elements.reset(new std::vector()); @@ -880,7 +880,7 @@ int MMFilesHashIndex::insertMulti(transaction::Methods* trx, void MMFilesHashIndex::batchInsertMulti( transaction::Methods* trx, std::vector> const& documents, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { TRI_ASSERT(queue != nullptr); std::shared_ptr> elements; elements.reset(new std::vector()); diff --git a/arangod/MMFiles/MMFilesHashIndex.h b/arangod/MMFiles/MMFilesHashIndex.h index 95a02cf287..73561faccf 100644 --- a/arangod/MMFiles/MMFilesHashIndex.h +++ b/arangod/MMFiles/MMFilesHashIndex.h @@ -173,7 +173,7 @@ class MMFilesHashIndex final : public MMFilesPathBasedIndex { void batchInsert( transaction::Methods*, std::vector> const&, - arangodb::basics::LocalTaskQueue* queue = nullptr) override; + std::shared_ptr queue) override; int unload() override; @@ -205,7 +205,7 @@ class MMFilesHashIndex final : public MMFilesPathBasedIndex { void batchInsertUnique( transaction::Methods*, std::vector> const&, - arangodb::basics::LocalTaskQueue* queue = nullptr); + std::shared_ptr queue); int insertMulti(transaction::Methods*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback); @@ -213,7 +213,7 @@ class MMFilesHashIndex final : public MMFilesPathBasedIndex { void batchInsertMulti( transaction::Methods*, std::vector> const&, - arangodb::basics::LocalTaskQueue* queue = nullptr); + std::shared_ptr queue); int removeUniqueElement(transaction::Methods*, MMFilesHashIndexElement*, bool); diff --git a/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp b/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp index df9e6c8213..c1ade6cf77 100644 --- a/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp @@ -251,7 +251,7 @@ int RocksDBEdgeIndex::remove(transaction::Methods* trx, void RocksDBEdgeIndex::batchInsert( transaction::Methods* trx, std::vector> const& documents, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { // acquire rocksdb transaction RocksDBTransactionState* state = rocksutils::toRocksTransactionState(trx); rocksdb::Transaction* rtrx = state->rocksTransaction(); diff --git a/arangod/RocksDBEngine/RocksDBEdgeIndex.h b/arangod/RocksDBEngine/RocksDBEdgeIndex.h index af771a60a0..180def9ed1 100644 --- a/arangod/RocksDBEngine/RocksDBEdgeIndex.h +++ b/arangod/RocksDBEngine/RocksDBEdgeIndex.h @@ -111,7 +111,7 @@ class RocksDBEdgeIndex final : public RocksDBIndex { void batchInsert( transaction::Methods*, std::vector> const&, - arangodb::basics::LocalTaskQueue* queue = nullptr) override; + std::shared_ptr queue) override; int drop() override; diff --git a/lib/Basics/AssocMulti.h b/lib/Basics/AssocMulti.h index 061b11f2ee..900fb24889 100644 --- a/lib/Basics/AssocMulti.h +++ b/lib/Basics/AssocMulti.h @@ -292,7 +292,7 @@ class AssocMulti { void batchInsert(std::function const& contextCreator, std::function const& contextDestroyer, std::shared_ptr const> data, - LocalTaskQueue* queue) { + std::shared_ptr queue) { if (data->empty()) { // nothing to do return; diff --git a/lib/Basics/AssocMultiHelpers.h b/lib/Basics/AssocMultiHelpers.h index 1548ebadec..600c11f203 100644 --- a/lib/Basics/AssocMultiHelpers.h +++ b/lib/Basics/AssocMultiHelpers.h @@ -94,7 +94,7 @@ class MultiInserterTask : public LocalTask { public: MultiInserterTask( - LocalTaskQueue* queue, std::function contextDestroyer, + std::shared_ptr queue, std::function contextDestroyer, std::vector* buckets, std::function @@ -168,7 +168,7 @@ class MultiPartitionerTask : public LocalTask { public: MultiPartitionerTask( - LocalTaskQueue* queue, + std::shared_ptr queue, std::function hashElement, std::function const& contextDestroyer, std::shared_ptr const> data, size_t lower, diff --git a/lib/Basics/AssocUnique.h b/lib/Basics/AssocUnique.h index 8e686fe4da..c4d6111d99 100644 --- a/lib/Basics/AssocUnique.h +++ b/lib/Basics/AssocUnique.h @@ -553,7 +553,7 @@ class AssocUnique { void batchInsert(std::function const& contextCreator, std::function const& contextDestroyer, std::shared_ptr const> data, - arangodb::basics::LocalTaskQueue* queue) { + std::shared_ptr queue) { TRI_ASSERT(queue != nullptr); if (data->empty()) { // nothing to do diff --git a/lib/Basics/AssocUniqueHelpers.h b/lib/Basics/AssocUniqueHelpers.h index b51e0cb40a..25148f1caf 100644 --- a/lib/Basics/AssocUniqueHelpers.h +++ b/lib/Basics/AssocUniqueHelpers.h @@ -70,7 +70,7 @@ class UniqueInserterTask : public LocalTask { public: UniqueInserterTask( - LocalTaskQueue* queue, std::function contextDestroyer, + std::shared_ptr queue, std::function contextDestroyer, std::vector* buckets, std::function doInsert, std::function checkResize, size_t i, @@ -140,7 +140,7 @@ class UniquePartitionerTask : public LocalTask { public: UniquePartitionerTask( - LocalTaskQueue* queue, + std::shared_ptr queue, std::function hashElement, std::function const& contextDestroyer, std::shared_ptr const> data, size_t lower, diff --git a/lib/Basics/LocalTaskQueue.cpp b/lib/Basics/LocalTaskQueue.cpp index d11dae928f..214e5b86d6 100644 --- a/lib/Basics/LocalTaskQueue.cpp +++ b/lib/Basics/LocalTaskQueue.cpp @@ -34,7 +34,7 @@ using namespace arangodb::basics; /// @brief create a task tied to the specified queue //////////////////////////////////////////////////////////////////////////////// -LocalTask::LocalTask(LocalTaskQueue* queue) : _queue(queue) {} +LocalTask::LocalTask(std::shared_ptr queue) : _queue(queue) {} //////////////////////////////////////////////////////////////////////////////// /// @brief dispatch this task to the underlying io_service @@ -58,7 +58,7 @@ void LocalTask::dispatch() { /// @brief create a callback task tied to the specified queue //////////////////////////////////////////////////////////////////////////////// -LocalCallbackTask::LocalCallbackTask(LocalTaskQueue* queue, +LocalCallbackTask::LocalCallbackTask(std::shared_ptr queue, std::function cb) : _queue(queue), _cb(cb) {} diff --git a/lib/Basics/LocalTaskQueue.h b/lib/Basics/LocalTaskQueue.h index 92de02e939..d48ae2cc1d 100644 --- a/lib/Basics/LocalTaskQueue.h +++ b/lib/Basics/LocalTaskQueue.h @@ -43,7 +43,7 @@ class LocalTask : public std::enable_shared_from_this { LocalTask(LocalTask const&) = delete; LocalTask& operator=(LocalTask const&) = delete; - explicit LocalTask(LocalTaskQueue* queue); + explicit LocalTask(std::shared_ptr queue); virtual ~LocalTask() {} virtual void run() = 0; @@ -54,7 +54,7 @@ class LocalTask : public std::enable_shared_from_this { /// @brief the underlying queue ////////////////////////////////////////////////////////////////////////////// - LocalTaskQueue* _queue; + std::shared_ptr _queue; }; class LocalCallbackTask @@ -64,7 +64,7 @@ class LocalCallbackTask LocalCallbackTask(LocalCallbackTask const&) = delete; LocalCallbackTask& operator=(LocalCallbackTask const&) = delete; - LocalCallbackTask(LocalTaskQueue* queue, std::function cb); + LocalCallbackTask(std::shared_ptr queue, std::function cb); virtual ~LocalCallbackTask() {} virtual void run(); @@ -75,7 +75,7 @@ class LocalCallbackTask /// @brief the underlying queue ////////////////////////////////////////////////////////////////////////////// - LocalTaskQueue* _queue; + std::shared_ptr _queue; ////////////////////////////////////////////////////////////////////////////// /// @brief the callback executed by run() (any exceptions will be caught and From f21fd3c0fe9459fcfe1757892143f00bf28be151 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 25 Apr 2017 15:56:39 +0200 Subject: [PATCH 18/58] Made dropping of graphs more resilient to errors --- js/common/modules/@arangodb/general-graph.js | 73 ++++++-------------- 1 file changed, 22 insertions(+), 51 deletions(-) diff --git a/js/common/modules/@arangodb/general-graph.js b/js/common/modules/@arangodb/general-graph.js index 2064054250..219858fdcf 100644 --- a/js/common/modules/@arangodb/general-graph.js +++ b/js/common/modules/@arangodb/general-graph.js @@ -2012,67 +2012,38 @@ exports._drop = function (graphId, dropCollections) { if (dropCollections === true) { graphs = exports._listObjects(); - var initialCollections = new Set(); // Here we collect the initial collection(s) + // Here we collect all collections + // that are leading for distribution + var initialCollections = new Set(); + let dropColCB = (name) => { + if (checkIfMayBeDropped(name, graph._key, graphs)) { + try { + let colObj = db[name]; + if (colObj !== undefined) { + // If it is undefined the collection is gone already + if (colObj.properties().distributeShardsLike !== undefined) { + db._drop(name); + } else { + initialCollections.add(name); + } + } + } catch (ignore) {} + } + }; // drop orphans if (!graph.orphanCollections) { graph.orphanCollections = []; } - graph.orphanCollections.forEach( - function (oC) { - if (checkIfMayBeDropped(oC, graph._key, graphs)) { - try { - let colObj = db[oC]; - if (colObj.properties().distributeShardsLike !== undefined) { - db._drop(oC); - } else { - initialCollections.add(oC); - } - } catch (ignore) {} - } - } - ); + graph.orphanCollections.forEach(dropColCB); var edgeDefinitions = graph.edgeDefinitions; edgeDefinitions.forEach( function (edgeDefinition) { var from = edgeDefinition.from; var to = edgeDefinition.to; var collection = edgeDefinition.collection; - if (checkIfMayBeDropped(collection, graph._key, graphs)) { - let colObj = db[collection]; - if (colObj.properties().distributeShardsLike !== undefined) { - db._drop(collection); - } else { - initialCollections.add(collection); - } - } - from.forEach( - function (col) { - if (checkIfMayBeDropped(col, graph._key, graphs)) { - try { - let colObj = db[col]; - if (colObj.properties().distributeShardsLike !== undefined) { - db._drop(col); - } else { - initialCollections.add(col); - } - } catch (ignore) {} - } - } - ); - to.forEach( - function (col) { - if (checkIfMayBeDropped(col, graph._key, graphs)) { - try { - let colObj = db[col]; - if (colObj.properties().distributeShardsLike !== undefined) { - db._drop(col); - } else { - initialCollections.add(col); - } - } catch (ignore2) {} - } - } - ); + dropColCB(edgeDefinition.collection); + from.forEach(dropColCB); + to.forEach(dropColCB); } ); for (let c of initialCollections) { From 4289105eb319bd54f3cf6ef28aa1b27361ca3678 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 25 Apr 2017 16:09:01 +0200 Subject: [PATCH 19/58] fix shutdown issue --- arangod/Agency/Agent.cpp | 14 +------------- arangod/Agency/Supervision.cpp | 6 +++++- lib/Basics/Thread.cpp | 3 ++- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp index b064aa5277..94e3ad2fb1 100644 --- a/arangod/Agency/Agent.cpp +++ b/arangod/Agency/Agent.cpp @@ -100,20 +100,8 @@ Agent::~Agent() { FATAL_ERROR_EXIT(); } } - - if (!isStopping()) { - - { - CONDITION_LOCKER(guardW, _waitForCV); - guardW.broadcast(); - } - { - CONDITION_LOCKER(guardA, _appendCV); - guardA.broadcast(); - } - shutdown(); - } + shutdown(); } diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 1f2db43b27..ba4daf322b 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -483,7 +483,11 @@ void Supervision::run() { // that running the supervision does not make sense and will indeed // lead to horrible errors: while (!this->isStopping()) { - std::this_thread::sleep_for(std::chrono::duration(5.0)); + { + CONDITION_LOCKER(guard, _cv); + _cv.wait(static_cast(1000000 * _frequency)); + } + MUTEX_LOCKER(locker, _lock); try { _snapshot = _agent->readDB().get(_agencyPrefix); diff --git a/lib/Basics/Thread.cpp b/lib/Basics/Thread.cpp index 1f25d8d547..88ca01b821 100644 --- a/lib/Basics/Thread.cpp +++ b/lib/Basics/Thread.cpp @@ -179,12 +179,13 @@ Thread::~Thread() { } _state.store(ThreadState::DETACHED); + return; } state = _state.load(); if (state != ThreadState::DETACHED && state != ThreadState::CREATED) { - LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "thread is not detached but " << stringify(state) + LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "thread '" << _name << "' is not detached but " << stringify(state) << ". shutting down hard"; FATAL_ERROR_ABORT(); } From 6e7843897b6f677a6275ae43ecd6b463c050d11b Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Tue, 25 Apr 2017 16:55:40 +0200 Subject: [PATCH 20/58] distributeShardsLike port from 3.1 app.js --- js/actions/_api/collection/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/js/actions/_api/collection/app.js b/js/actions/_api/collection/app.js index a5b57fc103..db1f547b03 100644 --- a/js/actions/_api/collection/app.js +++ b/js/actions/_api/collection/app.js @@ -68,6 +68,7 @@ function collectionRepresentation(collection, showProperties, showCount, showFig result.numberOfShards = properties.numberOfShards; result.replicationFactor = properties.replicationFactor; result.avoidServers = properties.avoidServers; + result.distributeShardsLike = properties.distributeShardsLike; } } From cd819e0b269186042276a934a6becd36a88a99c8 Mon Sep 17 00:00:00 2001 From: hkernbach Date: Tue, 25 Apr 2017 19:27:30 +0200 Subject: [PATCH 21/58] node info view --- .../APP/frontend/js/routers/router.js | 44 +++++-- .../frontend/js/templates/nodeInfoView.ejs | 27 +++++ .../APP/frontend/js/templates/nodesView.ejs | 38 +++--- .../APP/frontend/js/views/nodeInfoView.js | 108 ++++++++++++++++++ .../APP/frontend/js/views/nodeView.js | 8 ++ .../APP/frontend/js/views/nodesView.js | 15 ++- .../aardvark/APP/frontend/scss/_nodes.scss | 3 +- 7 files changed, 214 insertions(+), 29 deletions(-) create mode 100644 js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodeInfoView.ejs create mode 100644 js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeInfoView.js diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/routers/router.js b/js/apps/system/_admin/aardvark/APP/frontend/js/routers/router.js index 92b3fcd1f0..ae643647c6 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/routers/router.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/routers/router.js @@ -39,6 +39,7 @@ 'nodes': 'nodes', 'shards': 'shards', 'node/:name': 'node', + 'nodeInfo/:id': 'nodeInfo', 'logs': 'logger', 'helpus': 'helpUs', 'graph/:name': 'graph', @@ -327,16 +328,40 @@ return; } - if (!this.nodeView) { - this.nodeView = new window.NodeView({ - coordname: name, - coordinators: this.coordinatorCollection, - dbServers: this.dbServers - }); + if (this.nodeView) { + this.nodeView.remove(); } + this.nodeView = new window.NodeView({ + coordname: name, + coordinators: this.coordinatorCollection, + dbServers: this.dbServers + }); this.nodeView.render(); }, + nodeInfo: function (id, initialized) { + this.checkUser(); + if (!initialized || this.isCluster === undefined) { + this.waitForInit(this.nodeInfo.bind(this), id); + return; + } + if (this.isCluster === false) { + this.routes[''] = 'dashboard'; + this.navigate('#dashboard', {trigger: true}); + return; + } + + if (this.nodeInfoView) { + this.nodeInfoView.remove(); + } + this.nodeInfoView = new window.NodeInfoView({ + nodeId: id, + coordinators: this.coordinatorCollection, + dbServers: this.dbServers[0] + }); + this.nodeInfoView.render(); + }, + shards: function (initialized) { this.checkUser(); if (!initialized || this.isCluster === undefined) { @@ -367,10 +392,11 @@ this.navigate('#dashboard', {trigger: true}); return; } - if (!this.nodesView) { - this.nodesView = new window.NodesView({ - }); + if (this.nodesView) { + this.nodesView.remove(); } + this.nodesView = new window.NodesView({ + }); this.nodesView.render(); }, diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodeInfoView.ejs b/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodeInfoView.ejs new file mode 100644 index 0000000000..2889431ccc --- /dev/null +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodeInfoView.ejs @@ -0,0 +1,27 @@ + diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodesView.ejs b/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodesView.ejs index ed1ba839fa..6ed7b1fda7 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodesView.ejs +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/templates/nodesView.ejs @@ -47,10 +47,10 @@
Name
-
Endpoint
-
Heartbeat
-
Status
-
+
Endpoint
+
Since
+
Info
+
Status
@@ -67,16 +67,17 @@ <% } %> -
<%= node.Endpoint %>
+
<%= node.Endpoint %>
<% var formatted = (node.LastHeartbeatAcked).substr(11, 18).slice(0, -1); %> -
<%= formatted %>
-
<%= node.LastHeartbeatStatus %>
+
<%= formatted %>
+ +
<% if(node.Status === 'GOOD') { %> -
+
<% } else { %> -
+
<% } %> @@ -128,10 +129,10 @@
Name
-
Endpoint
-
Heartbeat
-
Status
-
+
Endpoint
+
Since
+
Info
+
Status
<% } %> @@ -143,16 +144,17 @@
<%= node.ShortName %>
-
<%= node.Endpoint %>
+
<%= node.Endpoint %>
<% var formatted = (node.LastHeartbeatAcked).substr(11, 18).slice(0, -1); %> -
<%= formatted %>
-
<%= node.LastHeartbeatStatus %>
+
<%= formatted %>
+ +
<% if(node.Status === 'GOOD') { %> -
+
<% } else { %> -
+
<% } %>
diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeInfoView.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeInfoView.js new file mode 100644 index 0000000000..39bbff0939 --- /dev/null +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeInfoView.js @@ -0,0 +1,108 @@ +/* jshint browser: true */ +/* jshint unused: false */ +/* global arangoHelper, $, Backbone, templateEngine, window */ +(function () { + 'use strict'; + + window.NodeInfoView = Backbone.View.extend({ + el: '#content', + + template: templateEngine.createTemplate('nodeInfoView.ejs'), + + initialize: function (options) { + if (window.App.isCluster) { + this.nodeId = options.nodeId; + this.dbServers = options.dbServers; + this.coordinators = options.coordinators; + } + }, + + remove: function () { + this.$el.empty().off(); /* off to unbind the events */ + this.stopListening(); + this.unbind(); + delete this.el; + return this; + }, + + render: function () { + this.$el.html(this.template.render({entries: []})); + + var callback = function () { + this.continueRender(); + this.breadcrumb(arangoHelper.getCoordinatorShortName(this.nodeId)); + $(window).trigger('resize'); + }.bind(this); + + if (!this.initCoordDone) { + this.waitForCoordinators(); + } + + if (!this.initDBDone) { + this.waitForDBServers(callback); + } else { + this.nodeId = window.location.hash.split('/')[1]; + this.coordinator = this.coordinators.findWhere({name: this.coordname}); + callback(); + } + }, + + continueRender: function () { + var model; + if (this.coordinator) { + model = this.coordinator.toJSON(); + } else { + model = this.dbServer.toJSON(); + } + + var renderObj = {}; + renderObj.Name = model.name; + renderObj.Address = model.address; + renderObj.Status = model.status; + renderObj.Protocol = model.protocol; + renderObj.Role = model.role; + this.$el.html(this.template.render({entries: renderObj})); + }, + + breadcrumb: function (name) { + $('#subNavigationBar .breadcrumb').html('Node: ' + name); + }, + + waitForCoordinators: function (callback) { + var self = this; + + window.setTimeout(function () { + if (self.coordinators.length === 0) { + self.waitForCoordinators(callback); + } else { + self.coordinator = self.coordinators.findWhere({name: self.nodeId}); + self.initCoordDone = true; + if (callback) { + callback(); + } + } + }, 200); + }, + + waitForDBServers: function (callback) { + var self = this; + + window.setTimeout(function () { + if (self.dbServers.length === 0) { + self.waitForDBServers(callback); + } else { + self.initDBDone = true; + + self.dbServers.each(function (model) { + if (model.get('id') === self.nodeId) { + self.dbServer = model; + } + }); + + callback(); + } + }, 200); + } + + }); +}()); diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeView.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeView.js index 5f81d42c10..cf48f40205 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeView.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodeView.js @@ -30,6 +30,14 @@ } }, + remove: function () { + this.$el.empty().off(); /* off to unbind the events */ + this.stopListening(); + this.unbind(); + delete this.el; + return this; + }, + breadcrumb: function (name) { $('#subNavigationBar .breadcrumb').html('Node: ' + name); }, diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodesView.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodesView.js index 6d4e23ea3c..be281a3be0 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodesView.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/nodesView.js @@ -22,6 +22,14 @@ 'keyup #plannedDBs': 'checkKey' }, + remove: function () { + this.$el.empty().off(); /* off to unbind the events */ + this.stopListening(); + this.unbind(); + delete this.el; + return this; + }, + checkKey: function (e) { if (e.keyCode === 13) { var self = this; @@ -121,11 +129,16 @@ }, navigateToNode: function (elem) { + var name = $(elem.currentTarget).attr('node').slice(0, -5); + + if ($(elem.target).hasClass('fa-info-circle')) { + window.App.navigate('#nodeInfo/' + encodeURIComponent(name), {trigger: true}); + return; + } if ($(elem.currentTarget).hasClass('noHover')) { return; } - var name = $(elem.currentTarget).attr('node').slice(0, -5); window.App.navigate('#node/' + encodeURIComponent(name), {trigger: true}); }, diff --git a/js/apps/system/_admin/aardvark/APP/frontend/scss/_nodes.scss b/js/apps/system/_admin/aardvark/APP/frontend/scss/_nodes.scss index 937445bbc3..cafed0ad68 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/scss/_nodes.scss +++ b/js/apps/system/_admin/aardvark/APP/frontend/scss/_nodes.scss @@ -33,8 +33,9 @@ .pure-table-body { .fa-check-circle, + .fa-info-circle, .fa-exclamation-circle { - font-size: 15pt; + font-size: 13pt; } } From 9dd275b497937452d4d1e29bc7a5a3a568a76162 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 25 Apr 2017 13:54:56 -0400 Subject: [PATCH 22/58] Fixed dump/restore tests. --- arangod/RocksDBEngine/RocksDBCommon.cpp | 25 +++++++++++++++++++ arangod/RocksDBEngine/RocksDBCommon.h | 3 +++ .../RocksDBReplicationContext.cpp | 22 ++++++++++++---- .../RocksDBEngine/RocksDBReplicationContext.h | 1 + .../RocksDBRestReplicationHandler.cpp | 4 ++- js/server/tests/dump/dump-rocksdb.js | 4 +-- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBCommon.cpp b/arangod/RocksDBEngine/RocksDBCommon.cpp index 536614cea6..6106e0ecff 100644 --- a/arangod/RocksDBEngine/RocksDBCommon.cpp +++ b/arangod/RocksDBEngine/RocksDBCommon.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include "Logger/Logger.h" namespace arangodb { @@ -127,6 +128,30 @@ void uint64ToPersistent(std::string& p, uint64_t value) { } while (++len < sizeof(uint64_t)); } +void stripObjectIds(VPackBuilder& builder, VPackSlice const& slice) { + if (slice.isObject()) { + builder.openObject(); + for (auto it = arangodb::velocypack::ObjectIterator(slice); it.valid(); + it++) { + if (it.key().copyString() == "objectId") { + continue; + } + builder.add(it.key()); + stripObjectIds(builder, it.value()); + } + builder.close(); + } else if (slice.isArray()) { + builder.openArray(); + for (auto it = arangodb::velocypack::ArrayIterator(slice); it.valid(); + it++) { + stripObjectIds(builder, it.value()); + } + builder.close(); + } else { + builder.add(slice); + } +} + RocksDBTransactionState* toRocksTransactionState(transaction::Methods* trx) { TRI_ASSERT(trx != nullptr); TransactionState* state = trx->state(); diff --git a/arangod/RocksDBEngine/RocksDBCommon.h b/arangod/RocksDBEngine/RocksDBCommon.h index 4cd9fa01d3..10927aa83a 100644 --- a/arangod/RocksDBEngine/RocksDBCommon.h +++ b/arangod/RocksDBEngine/RocksDBCommon.h @@ -88,6 +88,9 @@ arangodb::Result convertStatus(rocksdb::Status const&, uint64_t uint64FromPersistent(char const* p); void uint64ToPersistent(char* p, uint64_t value); void uint64ToPersistent(std::string& out, uint64_t value); + +void stripObjectIds(VPackBuilder&, VPackSlice const&); + RocksDBTransactionState* toRocksTransactionState(transaction::Methods* trx); rocksdb::TransactionDB* globalRocksDB(); RocksDBEngine* globalRocksEngine(); diff --git a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp index ad9ba18fa1..784281861e 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp @@ -48,6 +48,7 @@ double const RocksDBReplicationContext::DefaultTTL = 30 * 60.0; RocksDBReplicationContext::RocksDBReplicationContext() : _id(TRI_NewTickServer()), _lastTick(0), + _currentTick(0), _trx(), _collection(nullptr), _iter(), @@ -77,21 +78,26 @@ uint64_t RocksDBReplicationContext::count() const { // creates new transaction/snapshot void RocksDBReplicationContext::bind(TRI_vocbase_t* vocbase) { - releaseDumpingResources(); - _trx = createTransaction(vocbase); + if ((_trx.get() == nullptr) || (_trx->vocbase() != vocbase)) { + releaseDumpingResources(); + _trx = createTransaction(vocbase); + } } int RocksDBReplicationContext::bindCollection( std::string const& collectionName) { - if ((_collection == nullptr) || _collection->name() != collectionName) { + if ((_collection == nullptr) || + ((_collection->name() != collectionName) && + std::to_string(_collection->cid()) != collectionName)) { _collection = _trx->vocbase()->lookupCollection(collectionName); - if (_collection == nullptr) { return TRI_ERROR_BAD_PARAMETER; } + _trx->addCollectionAtRuntime(collectionName); _iter = _collection->getAllIterator(_trx.get(), &_mdr, false); //_mdr is not used nor updated + _currentTick = 1; _hasMore = true; } return TRI_ERROR_NO_ERROR; @@ -174,13 +180,19 @@ RocksDBReplicationResult RocksDBReplicationContext::dump( try { _hasMore = _iter->next(cb, 10); // TODO: adjust limit? } catch (std::exception const& ex) { + _hasMore = false; return RocksDBReplicationResult(TRI_ERROR_INTERNAL, _lastTick); } catch (RocksDBReplicationResult const& ex) { + _hasMore = false; return ex; } } - return RocksDBReplicationResult(TRI_ERROR_NO_ERROR, _lastTick); + if (_hasMore) { + _currentTick++; + } + + return RocksDBReplicationResult(TRI_ERROR_NO_ERROR, _currentTick); } arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b, diff --git a/arangod/RocksDBEngine/RocksDBReplicationContext.h b/arangod/RocksDBEngine/RocksDBReplicationContext.h index 111e5f9070..95135f37f5 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationContext.h +++ b/arangod/RocksDBEngine/RocksDBReplicationContext.h @@ -106,6 +106,7 @@ class RocksDBReplicationContext { private: TRI_voc_tick_t _id; uint64_t _lastTick; + uint64_t _currentTick; std::unique_ptr _trx; LogicalCollection* _collection; std::unique_ptr _iter; diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index dc36f31ec6..4af190e684 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -826,7 +826,9 @@ void RocksDBRestReplicationHandler::handleCommandRestoreCollection() { "invalid JSON"); return; } - VPackSlice const slice = parsedRequest->slice(); + VPackBuilder builder; + stripObjectIds(builder, parsedRequest->slice()); + VPackSlice const slice = builder.slice(); bool overwrite = false; diff --git a/js/server/tests/dump/dump-rocksdb.js b/js/server/tests/dump/dump-rocksdb.js index b2f2aa36e7..59d1e59576 100644 --- a/js/server/tests/dump/dump-rocksdb.js +++ b/js/server/tests/dump/dump-rocksdb.js @@ -185,7 +185,7 @@ function dumpTestSuite () { assertFalse(p.waitForSync); assertFalse(p.isVolatile); - assertEqual(9, c.getIndexes().length); + assertEqual(7, c.getIndexes().length); assertEqual("primary", c.getIndexes()[0].type); assertEqual("hash", c.getIndexes()[1].type); @@ -241,7 +241,7 @@ function dumpTestSuite () { assertEqual(2, c.type()); // document assertFalse(p.waitForSync); - + assertEqual(1, c.getIndexes().length); // just primary index assertEqual("primary", c.getIndexes()[0].type); assertEqual(0, c.count()); From 47be06295701f06acce27bafbcb9e81df77673b7 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 25 Apr 2017 18:13:40 +0200 Subject: [PATCH 23/58] move configuring of cmake back into install process - it will conflict with the release process otherwise --- cmake/InstallMacros.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/InstallMacros.cmake b/cmake/InstallMacros.cmake index 09a7d20ab4..ed03932e3d 100644 --- a/cmake/InstallMacros.cmake +++ b/cmake/InstallMacros.cmake @@ -74,8 +74,9 @@ macro (install_readme input output) if (MSVC) set(CRLFSTYLE "CRLF") endif () - configure_file(${PROJECT_SOURCE_DIR}/${input} "${PROJECT_BINARY_DIR}/${output}" NEWLINE_STYLE ${CRLFSTYLE}) + install( + CODE "configure_file(${PROJECT_SOURCE_DIR}/${input} \"${PROJECT_BINARY_DIR}/${output}\" NEWLINE_STYLE ${CRLFSTYLE})" FILES "${PROJECT_BINARY_DIR}/${output}" DESTINATION "${where}" ) From 05a46e47813537d05f18032debec48d1f522adc7 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 25 Apr 2017 19:10:23 +0200 Subject: [PATCH 24/58] fix db._explain() to work with the new interface of the graph traverser --- js/common/modules/@arangodb/aql/explainer.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/js/common/modules/@arangodb/aql/explainer.js b/js/common/modules/@arangodb/aql/explainer.js index 9fefe16678..cd5333ee77 100644 --- a/js/common/modules/@arangodb/aql/explainer.js +++ b/js/common/modules/@arangodb/aql/explainer.js @@ -347,8 +347,8 @@ function printTraversalDetails (traversals) { maxEdgeCollectionNameStrLen = node.edgeCollectionNameStrLen; } } - if (node.hasOwnProperty('traversalFlags')) { - var opts = optify(node.traversalFlags); + if (node.hasOwnProperty('options')) { + var opts = optify(node.options); if (opts.length > maxOptionsLen) { maxOptionsLen = opts.length; } @@ -384,8 +384,8 @@ function printTraversalDetails (traversals) { line += pad(1 + maxEdgeCollectionNameStrLen) + ' '; } - if (traversals[i].hasOwnProperty('traversalFlags')) { - line += optify(traversals[i].traversalFlags, true) + pad(1 + maxOptionsLen - optify(traversals[i].traversalFlags, false).length) + ' '; + if (traversals[i].hasOwnProperty('options')) { + line += optify(traversals[i].options, true) + pad(1 + maxOptionsLen - optify(traversals[i].options, false).length) + ' '; } else { line += pad(1 + maxOptionsLen) + ' '; } @@ -856,7 +856,7 @@ function processQuery (query, explain) { return keyword('FOR') + ' ' + variableName(node.outVariable) + ' ' + keyword('IN') + ' ' + collection(node.collection) + ' ' + annotation('/* ' + (node.reverse ? 'reverse ' : '') + node.index.type + ' index scan */'); case 'TraversalNode': - node.minMaxDepth = node.traversalFlags.minDepth + '..' + node.traversalFlags.maxDepth; + node.minMaxDepth = node.options.minDepth + '..' + node.options.maxDepth; node.minMaxDepthLen = node.minMaxDepth.length; rc = keyword('FOR '); From 20dc8b4fa0aa3e2121c5d1158fff2d3ebb0b412d Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 25 Apr 2017 16:51:38 -0400 Subject: [PATCH 25/58] Fixed duplicate objectIds in InitialSyncer. --- arangod/Replication/InitialSyncer.cpp | 879 +++++++++++++------------- 1 file changed, 444 insertions(+), 435 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 9eaf426430..d789164778 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -32,18 +32,18 @@ #include "Basics/StringUtils.h" #include "Basics/VelocyPackHelper.h" #include "Indexes/Index.h" +#include "Indexes/IndexIterator.h" #include "Logger/Logger.h" -#include "StorageEngine/EngineSelectorFeature.h" #include "MMFiles/MMFilesCollection.h" //TODO -- Remove -- ditches #include "MMFiles/MMFilesDatafileHelper.h" #include "MMFiles/MMFilesDitch.h" #include "MMFiles/MMFilesIndexElement.h" #include "MMFiles/MMFilesPrimaryIndex.h" #include "RestServer/DatabaseFeature.h" +#include "RocksDBEngine/RocksDBCommon.h" #include "SimpleHttpClient/SimpleHttpClient.h" #include "SimpleHttpClient/SimpleHttpResult.h" #include "StorageEngine/EngineSelectorFeature.h" -#include "Indexes/IndexIterator.h" #include "StorageEngine/StorageEngine.h" #include "Transaction/Helpers.h" #include "Utils/CollectionGuard.h" @@ -64,6 +64,7 @@ using namespace arangodb; using namespace arangodb::basics; using namespace arangodb::httpclient; using namespace arangodb::rest; +using namespace arangodb::rocksutils; //////////////////////////////////////////////////////////////////////////////// /// @brief performs a binary search for the given key in the markers vector @@ -273,7 +274,10 @@ int InitialSyncer::run(std::string& errorMsg, bool incremental) { errorMsg = "got invalid response from master at " + _masterInfo._endpoint + ": invalid JSON"; } else { - res = handleInventoryResponse(slice, incremental, errorMsg); + VPackBuilder stripBuilder; + stripObjectIds(stripBuilder, slice); + res = handleInventoryResponse(stripBuilder.slice(), incremental, + errorMsg); } } @@ -1027,11 +1031,12 @@ int InitialSyncer::handleCollectionSync(arangodb::LogicalCollection* col, // now we can fetch the complete chunk information from the master try { if (std::strcmp("mmfiles", EngineSelectorFeature::engineName()) == 0) { - res = handleSyncKeysMMFiles(col, id.copyString(), cid, collectionName, maxTick, - errorMsg); - } else if (std::strcmp("rocksdb", EngineSelectorFeature::engineName()) == 0) { - res = handleSyncKeysRocksDB(col, id.copyString(), cid, collectionName, maxTick, - errorMsg); + res = handleSyncKeysMMFiles(col, id.copyString(), cid, collectionName, + maxTick, errorMsg); + } else if (std::strcmp("rocksdb", EngineSelectorFeature::engineName()) == + 0) { + res = handleSyncKeysRocksDB(col, id.copyString(), cid, collectionName, + maxTick, errorMsg); } else { THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); } @@ -1052,11 +1057,11 @@ int InitialSyncer::handleCollectionSync(arangodb::LogicalCollection* col, //////////////////////////////////////////////////////////////////////////////// int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, - std::string const& keysId, - std::string const& cid, - std::string const& collectionName, - TRI_voc_tick_t maxTick, - std::string& errorMsg) { + std::string const& keysId, + std::string const& cid, + std::string const& collectionName, + TRI_voc_tick_t maxTick, + std::string& errorMsg) { std::string progress = "collecting local keys for collection '" + collectionName + "'"; setProgress(progress); @@ -1149,32 +1154,34 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, TRI_ASSERT(chunk.isObject()); auto lowSlice = chunk.get("low"); TRI_ASSERT(lowSlice.isString()); - + // last high chunk = chunkSlice.at(numChunks - 1); TRI_ASSERT(chunk.isObject()); - + auto highSlice = chunk.get("high"); TRI_ASSERT(highSlice.isString()); std::string const lowKey(lowSlice.copyString()); std::string const highKey(highSlice.copyString()); - LogicalCollection* coll = trx.documentCollection(); - std::unique_ptr iterator = coll->getAllIterator(&trx, &mmdr, false); - iterator->next([&] (DocumentIdentifierToken const& token) { - if (coll->readDocument(&trx, token, mmdr) == false) { - return; - } - VPackSlice doc(mmdr.vpack()); - VPackSlice key = doc.get(StaticStrings::KeyString); - if (key.compareString(lowKey.data(), lowKey.length()) < 0) { - trx.remove(collectionName, key, options); - } else if (key.compareString(highKey.data(), highKey.length()) > 0) { - trx.remove(collectionName, key, options); - } - }, UINT64_MAX); + std::unique_ptr iterator = + coll->getAllIterator(&trx, &mmdr, false); + iterator->next( + [&](DocumentIdentifierToken const& token) { + if (coll->readDocument(&trx, token, mmdr) == false) { + return; + } + VPackSlice doc(mmdr.vpack()); + VPackSlice key = doc.get(StaticStrings::KeyString); + if (key.compareString(lowKey.data(), lowKey.length()) < 0) { + trx.remove(collectionName, key, options); + } else if (key.compareString(highKey.data(), highKey.length()) > 0) { + trx.remove(collectionName, key, options); + } + }, + UINT64_MAX); trx.commit(); } @@ -1197,13 +1204,12 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, return res.errorNumber(); } - // We do not take responsibility for the index. // The LogicalCollection is protected by trx. // Neither it nor it's indexes can be invalidated size_t currentChunkId = 0; - + std::string lowKey; std::string highKey; std::string hashString; @@ -1211,34 +1217,34 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, // chunk keys + revisionId std::vector> markers; bool foundLowKey = false; - - auto resetChunk = [&] () -> void { + + auto resetChunk = [&]() -> void { sendExtendBatch(); sendExtendBarrier(); - + progress = "processing keys chunk " + std::to_string(currentChunkId) + - " for collection '" + collectionName + "'"; + " for collection '" + collectionName + "'"; setProgress(progress); - + // read remote chunk VPackSlice chunk = chunkSlice.at(currentChunkId); if (!chunk.isObject()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": chunk is no object"; + _masterInfo._endpoint + ": chunk is no object"; THROW_ARANGO_EXCEPTION(TRI_ERROR_REPLICATION_INVALID_RESPONSE); } - + VPackSlice const lowSlice = chunk.get("low"); VPackSlice const highSlice = chunk.get("high"); VPackSlice const hashSlice = chunk.get("hash"); if (!lowSlice.isString() || !highSlice.isString() || !hashSlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": chunks in response have an invalid format"; + _masterInfo._endpoint + + ": chunks in response have an invalid format"; THROW_ARANGO_EXCEPTION(TRI_ERROR_REPLICATION_INVALID_RESPONSE); } - + // now reset chunk information markers.clear(); lowKey = lowSlice.copyString(); @@ -1249,21 +1255,22 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, }; // set to first chunk resetChunk(); - - std::function parseDoc = - [&] (VPackSlice doc, VPackSlice key) { - + + std::function parseDoc = [&](VPackSlice doc, + VPackSlice key) { + bool rangeUneqal = false; bool nextChunk = false; - + int cmp1 = key.compareString(lowKey.data(), lowKey.length()); int cmp2 = key.compareString(highKey.data(), highKey.length()); - + if (cmp1 < 0) { // smaller values than lowKey mean they don't exist remotely trx.remove(collectionName, key, options); return; - } if (cmp1 >= 0 && cmp2 <= 0) { + } + if (cmp1 >= 0 && cmp2 <= 0) { // we only need to hash we are in the range if (cmp1 == 0) { foundLowKey = true; @@ -1271,15 +1278,15 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, rangeUneqal = true; nextChunk = true; } - + if (foundLowKey) { VPackSlice revision = doc.get(StaticStrings::RevString); localHash ^= key.hashString(); localHash ^= revision.hash(); - + markers.emplace_back(key.copyString(), TRI_ExtractRevisionId(doc)); - - if (cmp2 == 0) {// found highKey + + if (cmp2 == 0) { // found highKey rangeUneqal = std::to_string(localHash) != hashString; nextChunk = true; } @@ -1287,42 +1294,44 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, rangeUneqal = true; nextChunk = true; } - } else if (cmp2 > 0) { // higher than highKey + } else if (cmp2 > 0) { // higher than highKey // current range was unequal and we did not find the // high key. Load range and skip to next rangeUneqal = true; nextChunk = true; } - + if (rangeUneqal) { - int res = syncChunkRocksDB(&trx, keysId, currentChunkId, - lowKey, highKey, - markers, errorMsg); + int res = syncChunkRocksDB(&trx, keysId, currentChunkId, lowKey, + highKey, markers, errorMsg); if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); } } - TRI_ASSERT(!rangeUneqal || (rangeUneqal && nextChunk)); // A => B - if (nextChunk && currentChunkId+1 < numChunks) { - currentChunkId++;// we are out of range, see next chunk + TRI_ASSERT(!rangeUneqal || (rangeUneqal && nextChunk)); // A => B + if (nextChunk && currentChunkId + 1 < numChunks) { + currentChunkId++; // we are out of range, see next chunk resetChunk(); - + // key is higher than upper bound, recheck the current document if (cmp2 > 0) { parseDoc(doc, key); } } }; - - std::unique_ptr iterator = col->getAllIterator(&trx, &mmdr, false); - iterator->next([&] (DocumentIdentifierToken const& token) { - if (col->readDocument(&trx, token, mmdr) == false) { - return; - } - VPackSlice doc(mmdr.vpack()); - VPackSlice key = doc.get(StaticStrings::KeyString); - parseDoc(doc, key); - }, UINT64_MAX); + + std::unique_ptr iterator = + col->getAllIterator(&trx, &mmdr, false); + iterator->next( + [&](DocumentIdentifierToken const& token) { + if (col->readDocument(&trx, token, mmdr) == false) { + return; + } + VPackSlice doc(mmdr.vpack()); + VPackSlice key = doc.get(StaticStrings::KeyString); + parseDoc(doc, key); + }, + UINT64_MAX); res = trx.commit(); if (!res.ok()) { @@ -1333,14 +1342,12 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, return res; } - -int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, - std::string const& keysId, - uint64_t chunkId, - std::string const& lowString, - std::string const& highString, - std::vector> markers, - std::string& errorMsg) { +int InitialSyncer::syncChunkRocksDB( + SingleCollectionTransaction* trx, std::string const& keysId, + uint64_t chunkId, std::string const& lowString, + std::string const& highString, + std::vector> markers, + std::string& errorMsg) { std::string const baseUrl = BaseUrl + "/keys"; TRI_voc_tick_t const chunkSize = 5000; std::string const& collectionName = trx->documentCollection()->name(); @@ -1349,102 +1356,101 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, options.silent = true; options.ignoreRevs = true; options.isRestore = true; - + // no match // must transfer keys for non-matching range std::string url = baseUrl + "/" + keysId + "?type=keys&chunk=" + - std::to_string(chunkId) + "&chunkSize=" + - std::to_string(chunkSize); - - std::string progress = "fetching keys chunk '" + std::to_string(chunkId) + "' from " + url; + std::to_string(chunkId) + "&chunkSize=" + + std::to_string(chunkSize); + + std::string progress = + "fetching keys chunk '" + std::to_string(chunkId) + "' from " + url; setProgress(progress); - + std::unique_ptr response( - _client->retryRequest(rest::RequestType::PUT, url, nullptr, 0)); - + _client->retryRequest(rest::RequestType::PUT, url, nullptr, 0)); + if (response == nullptr || !response->isComplete()) { errorMsg = "could not connect to master at " + _masterInfo._endpoint + - ": " + _client->getErrorMessage(); - + ": " + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { - errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + errorMsg = "got invalid response from master at " + _masterInfo._endpoint + + ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + + ": " + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { - errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + errorMsg = "got invalid response from master at " + _masterInfo._endpoint + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const responseBody = builder->slice(); if (!responseBody.isArray()) { - errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + errorMsg = "got invalid response from master at " + _masterInfo._endpoint + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + transaction::BuilderLeaser keyBuilder(trx); /*size_t nextStart = 0; // delete all keys at start of the range while (nextStart < markers.size()) { std::string const& localKey = markers[nextStart].first; - + if ( localKey.compare(lowString) < 0) { // we have a local key that is not present remotely keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(localKey)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); ++nextStart; } else { break; } }*/ - + std::vector toFetch; - + size_t const numKeys = static_cast(responseBody.length()); TRI_ASSERT(numKeys > 0); - + size_t i = 0; size_t nextStart = 0; for (VPackSlice const& pair : VPackArrayIterator(responseBody)) { - if (!pair.isArray() || pair.length() != 2) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response key pair is no valid array"; - + _masterInfo._endpoint + + ": response key pair is no valid array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + // key VPackSlice const keySlice = pair.at(0); if (!keySlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response key is no string"; - + _masterInfo._endpoint + ": response key is no string"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + // rid if (markers.empty()) { // no local markers @@ -1452,12 +1458,12 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, i++; continue; } - + std::string const keyString = keySlice.copyString(); // remove keys not present anymore while (nextStart < markers.size()) { std::string const& localKey = markers[nextStart].first; - + int res = localKey.compare(keyString); if (res != 0) { // we have a local key that is not present remotely @@ -1465,7 +1471,7 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, keyBuilder->openObject(); keyBuilder->add(StaticStrings::KeyString, VPackValue(localKey)); keyBuilder->close(); - + trx->remove(collectionName, keyBuilder->slice(), options); ++nextStart; } else { @@ -1473,7 +1479,7 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, break; } } - + // see if key exists DocumentIdentifierToken token = physical->lookupKey(trx, keySlice); if (!token._data) { @@ -1487,11 +1493,10 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, // a match - nothing to do! ++nextStart; } - + i++; } - - + if (!toFetch.empty()) { VPackBuilder keysBuilder; keysBuilder.openArray(); @@ -1499,86 +1504,84 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, keysBuilder.add(VPackValue(it)); } keysBuilder.close(); - + std::string url = baseUrl + "/" + keysId + "?type=docs&chunk=" + - std::to_string(chunkId) + "&chunkSize=" + - std::to_string(chunkSize); - progress = "fetching documents chunk " + - std::to_string(chunkId) + " for collection '" + - collectionName + "' from " + url; + std::to_string(chunkId) + "&chunkSize=" + + std::to_string(chunkSize); + progress = "fetching documents chunk " + std::to_string(chunkId) + + " for collection '" + collectionName + "' from " + url; setProgress(progress); - + std::string const keyJsonString(keysBuilder.slice().toJson()); - + std::unique_ptr response( - _client->retryRequest(rest::RequestType::PUT, url, - keyJsonString.c_str(), keyJsonString.size())); - + _client->retryRequest(rest::RequestType::PUT, url, + keyJsonString.c_str(), keyJsonString.size())); + if (response == nullptr || !response->isComplete()) { errorMsg = "could not connect to master at " + _masterInfo._endpoint + - ": " + _client->getErrorMessage(); - + ": " + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { errorMsg = "got invalid response from master at " + - std::string(_masterInfo._endpoint) + - ": response is no array"; - + std::string(_masterInfo._endpoint) + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const slice = builder->slice(); if (!slice.isArray()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + _masterInfo._endpoint + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + for (auto const& it : VPackArrayIterator(slice)) { if (!it.isObject()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": document is no object"; - + _masterInfo._endpoint + ": document is no object"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const keySlice = it.get(StaticStrings::KeyString); - + if (!keySlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": document key is invalid"; - + _masterInfo._endpoint + ": document key is invalid"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const revSlice = it.get(StaticStrings::RevString); - + if (!revSlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": document revision is invalid"; - + _masterInfo._endpoint + ": document revision is invalid"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + DocumentIdentifierToken token = physical->lookupKey(trx, keySlice); - + if (!token._data) { // INSERT OperationResult opRes = trx->insert(collectionName, it, options); @@ -1588,7 +1591,7 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, OperationResult opRes = trx->update(collectionName, it, options); res = opRes.code; } - + if (res != TRI_ERROR_NO_ERROR) { return res; } @@ -1602,337 +1605,349 @@ int InitialSyncer::syncChunkRocksDB(SingleCollectionTransaction* trx, //////////////////////////////////////////////////////////////////////////////// int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, - std::string const& keysId, std::string const& cid, - std::string const& collectionName, TRI_voc_tick_t maxTick, - std::string& errorMsg) { - + std::string const& keysId, + std::string const& cid, + std::string const& collectionName, + TRI_voc_tick_t maxTick, + std::string& errorMsg) { std::string progress = - "collecting local keys for collection '" + collectionName + "'"; + "collecting local keys for collection '" + collectionName + "'"; setProgress(progress); - + // fetch all local keys from primary index std::vector markers; - + MMFilesDocumentDitch* ditch = nullptr; - + // acquire a replication ditch so no datafiles are thrown away from now on // note: the ditch also protects against unloading the collection { - SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ); - + SingleCollectionTransaction trx( + transaction::StandaloneContext::Create(_vocbase), col->cid(), + AccessMode::Type::READ); + Result res = trx.begin(); - + if (!res.ok()) { - errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); - res.reset(res.errorNumber(),errorMsg); + errorMsg = + std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(), errorMsg); return res.errorNumber(); } - + ditch = arangodb::MMFilesCollection::toMMFilesCollection(col) - ->ditches() - ->createMMFilesDocumentDitch(false, __FILE__, __LINE__); - + ->ditches() + ->createMMFilesDocumentDitch(false, __FILE__, __LINE__); + if (ditch == nullptr) { return TRI_ERROR_OUT_OF_MEMORY; } } - + TRI_ASSERT(ditch != nullptr); - + TRI_DEFER(arangodb::MMFilesCollection::toMMFilesCollection(col) - ->ditches() - ->freeDitch(ditch)); - + ->ditches() + ->freeDitch(ditch)); + { - SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ); - + SingleCollectionTransaction trx( + transaction::StandaloneContext::Create(_vocbase), col->cid(), + AccessMode::Type::READ); + Result res = trx.begin(); - + if (!res.ok()) { - errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); - res.reset(res.errorNumber(),errorMsg); + errorMsg = + std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(), errorMsg); return res.errorNumber(); } - + // We do not take responsibility for the index. // The LogicalCollection is protected by trx. // Neither it nor it's indexes can be invalidated - + markers.reserve(trx.documentCollection()->numberDocuments(&trx)); - + uint64_t iterations = 0; ManagedDocumentResult mmdr; - trx.invokeOnAllElements(trx.name(), [this, &trx, &mmdr, &markers, &iterations](DocumentIdentifierToken const& token) { - if (trx.documentCollection()->readDocument(&trx, token, mmdr)) { - markers.emplace_back(mmdr.vpack()); - - if (++iterations % 10000 == 0) { - if (checkAborted()) { - return false; + trx.invokeOnAllElements( + trx.name(), [this, &trx, &mmdr, &markers, + &iterations](DocumentIdentifierToken const& token) { + if (trx.documentCollection()->readDocument(&trx, token, mmdr)) { + markers.emplace_back(mmdr.vpack()); + + if (++iterations % 10000 == 0) { + if (checkAborted()) { + return false; + } + } } - } - } - return true; - }); - + return true; + }); + if (checkAborted()) { return TRI_ERROR_REPLICATION_APPLIER_STOPPED; } - + sendExtendBatch(); sendExtendBarrier(); - + std::string progress = "sorting " + std::to_string(markers.size()) + - " local key(s) for collection '" + collectionName + - "'"; + " local key(s) for collection '" + collectionName + + "'"; setProgress(progress); - + // sort all our local keys - std::sort( - markers.begin(), markers.end(), - [](uint8_t const* lhs, uint8_t const* rhs) -> bool { - VPackSlice const l(lhs); - VPackSlice const r(rhs); - - VPackValueLength lLength, rLength; - char const* lKey = l.get(StaticStrings::KeyString).getString(lLength); - char const* rKey = r.get(StaticStrings::KeyString).getString(rLength); - - size_t const length = static_cast(lLength < rLength ? lLength : rLength); - int res = memcmp(lKey, rKey, length); - - if (res < 0) { - // left is smaller than right - return true; - } - if (res == 0 && lLength < rLength) { - // left is equal to right, but of shorter length - return true; - } - - return false; - }); + std::sort(markers.begin(), markers.end(), [](uint8_t const* lhs, + uint8_t const* rhs) -> bool { + VPackSlice const l(lhs); + VPackSlice const r(rhs); + + VPackValueLength lLength, rLength; + char const* lKey = l.get(StaticStrings::KeyString).getString(lLength); + char const* rKey = r.get(StaticStrings::KeyString).getString(rLength); + + size_t const length = + static_cast(lLength < rLength ? lLength : rLength); + int res = memcmp(lKey, rKey, length); + + if (res < 0) { + // left is smaller than right + return true; + } + if (res == 0 && lLength < rLength) { + // left is equal to right, but of shorter length + return true; + } + + return false; + }); } - + if (checkAborted()) { return TRI_ERROR_REPLICATION_APPLIER_STOPPED; } - + sendExtendBatch(); sendExtendBarrier(); - + std::vector toFetch; - + TRI_voc_tick_t const chunkSize = 5000; std::string const baseUrl = BaseUrl + "/keys"; - + std::string url = - baseUrl + "/" + keysId + "?chunkSize=" + std::to_string(chunkSize); + baseUrl + "/" + keysId + "?chunkSize=" + std::to_string(chunkSize); progress = "fetching remote keys chunks for collection '" + collectionName + - "' from " + url; + "' from " + url; setProgress(progress); - + std::unique_ptr response( - _client->retryRequest(rest::RequestType::GET, url, nullptr, 0)); - + _client->retryRequest(rest::RequestType::GET, url, nullptr, 0)); + if (response == nullptr || !response->isComplete()) { - errorMsg = "could not connect to master at " + - _masterInfo._endpoint + ": " + - _client->getErrorMessage(); - + errorMsg = "could not connect to master at " + _masterInfo._endpoint + + ": " + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { - errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + errorMsg = "got invalid response from master at " + _masterInfo._endpoint + + ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + + ": " + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { errorMsg = "got invalid response from master at " + - std::string(_masterInfo._endpoint) + - ": invalid response is no array"; - + std::string(_masterInfo._endpoint) + + ": invalid response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const slice = builder->slice(); - + if (!slice.isArray()) { - errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": response is no array"; - + errorMsg = "got invalid response from master at " + _masterInfo._endpoint + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + OperationOptions options; options.silent = true; options.ignoreRevs = true; options.isRestore = true; - + VPackBuilder keyBuilder; size_t const n = static_cast(slice.length()); - + // remove all keys that are below first remote key or beyond last remote key if (n > 0) { // first chunk - SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE); - + SingleCollectionTransaction trx( + transaction::StandaloneContext::Create(_vocbase), col->cid(), + AccessMode::Type::WRITE); + Result res = trx.begin(); - + if (!res.ok()) { - errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); - res.reset(res.errorNumber(),errorMsg); + errorMsg = + std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(), errorMsg); return res.errorNumber(); } - + VPackSlice chunk = slice.at(0); - + TRI_ASSERT(chunk.isObject()); auto lowSlice = chunk.get("low"); TRI_ASSERT(lowSlice.isString()); - + std::string const lowKey(lowSlice.copyString()); - + for (size_t i = 0; i < markers.size(); ++i) { VPackSlice const k(markers[i]); - + std::string const key(k.get(StaticStrings::KeyString).copyString()); - + if (key.compare(lowKey) >= 0) { break; } - + keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(key)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); } - + // last high chunk = slice.at(n - 1); TRI_ASSERT(chunk.isObject()); - + auto highSlice = chunk.get("high"); TRI_ASSERT(highSlice.isString()); - + std::string const highKey(highSlice.copyString()); - + for (size_t i = markers.size(); i >= 1; --i) { VPackSlice const k(markers[i - 1]); - + std::string const key(k.get(StaticStrings::KeyString).copyString()); - + if (key.compare(highKey) <= 0) { break; } - + keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(key)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); } - + trx.commit(); } - + size_t nextStart = 0; - + // now process each chunk for (size_t i = 0; i < n; ++i) { if (checkAborted()) { return TRI_ERROR_REPLICATION_APPLIER_STOPPED; } - - SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE); - + + SingleCollectionTransaction trx( + transaction::StandaloneContext::Create(_vocbase), col->cid(), + AccessMode::Type::WRITE); + Result res = trx.begin(); - + if (!res.ok()) { - errorMsg = std::string("unable to start transaction: ") + res.errorMessage(); - res.reset(res.errorNumber(),res.errorMessage()); + errorMsg = + std::string("unable to start transaction: ") + res.errorMessage(); + res.reset(res.errorNumber(), res.errorMessage()); return res.errorNumber(); } - - trx.pinData(col->cid()); // will throw when it fails - + + trx.pinData(col->cid()); // will throw when it fails + // We do not take responsibility for the index. // The LogicalCollection is protected by trx. // Neither it nor it's indexes can be invalidated - + // TODO Move to MMFiles auto physical = static_cast( - trx.documentCollection()->getPhysical()); + trx.documentCollection()->getPhysical()); auto idx = physical->primaryIndex(); - + size_t const currentChunkId = i; progress = "processing keys chunk " + std::to_string(currentChunkId) + - " for collection '" + collectionName + "'"; + " for collection '" + collectionName + "'"; setProgress(progress); - + sendExtendBatch(); sendExtendBarrier(); - + // read remote chunk VPackSlice chunk = slice.at(i); - + if (!chunk.isObject()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": chunk is no object"; - + _masterInfo._endpoint + ": chunk is no object"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const lowSlice = chunk.get("low"); VPackSlice const highSlice = chunk.get("high"); VPackSlice const hashSlice = chunk.get("hash"); - + if (!lowSlice.isString() || !highSlice.isString() || !hashSlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": chunks in response have an invalid format"; - + _masterInfo._endpoint + + ": chunks in response have an invalid format"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + std::string const lowString = lowSlice.copyString(); std::string const highString = highSlice.copyString(); - + size_t localFrom; size_t localTo; - bool match = - FindRange(markers, lowString, highString, localFrom, localTo); - + bool match = FindRange(markers, lowString, highString, localFrom, localTo); + if (match) { // now must hash the range uint64_t hash = 0x012345678; - + for (size_t i = localFrom; i <= localTo; ++i) { TRI_ASSERT(i < markers.size()); VPackSlice const current(markers.at(i)); hash ^= current.get(StaticStrings::KeyString).hashString(); hash ^= current.get(StaticStrings::RevString).hash(); } - + if (std::to_string(hash) != hashSlice.copyString()) { match = false; } } - + if (match) { // match nextStart = localTo + 1; @@ -1940,123 +1955,120 @@ int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, // no match // must transfer keys for non-matching range std::string url = baseUrl + "/" + keysId + "?type=keys&chunk=" + - std::to_string(i) + "&chunkSize=" + - std::to_string(chunkSize); + std::to_string(i) + "&chunkSize=" + + std::to_string(chunkSize); progress = "fetching keys chunk " + std::to_string(currentChunkId) + - " for collection '" + collectionName + "' from " + url; + " for collection '" + collectionName + "' from " + url; setProgress(progress); - - std::unique_ptr response(_client->retryRequest( - rest::RequestType::PUT, url, nullptr, 0)); - + + std::unique_ptr response( + _client->retryRequest(rest::RequestType::PUT, url, nullptr, 0)); + if (response == nullptr || !response->isComplete()) { - errorMsg = "could not connect to master at " + - _masterInfo._endpoint + ": " + - _client->getErrorMessage(); - + errorMsg = "could not connect to master at " + _masterInfo._endpoint + + ": " + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response is no array"; - + _masterInfo._endpoint + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const slice = builder->slice(); if (!slice.isArray()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response is no array"; - + _masterInfo._endpoint + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - - + // delete all keys at start of the range while (nextStart < markers.size()) { VPackSlice const keySlice(markers[nextStart]); - std::string const localKey(keySlice.get(StaticStrings::KeyString).copyString()); - + std::string const localKey( + keySlice.get(StaticStrings::KeyString).copyString()); + if (localKey.compare(lowString) < 0) { // we have a local key that is not present remotely keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(localKey)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); ++nextStart; } else { break; } } - + toFetch.clear(); - + size_t const n = static_cast(slice.length()); TRI_ASSERT(n > 0); - + for (size_t i = 0; i < n; ++i) { VPackSlice const pair = slice.at(i); - + if (!pair.isArray() || pair.length() != 2) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response key pair is no valid array"; - + _masterInfo._endpoint + + ": response key pair is no valid array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + // key VPackSlice const keySlice = pair.at(0); - + if (!keySlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response key is no string"; - + _masterInfo._endpoint + ": response key is no string"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + // rid if (markers.empty()) { // no local markers toFetch.emplace_back(i); continue; } - + std::string const keyString = keySlice.copyString(); - + while (nextStart < markers.size()) { VPackSlice const localKeySlice(markers[nextStart]); - std::string const localKey(localKeySlice.get(StaticStrings::KeyString).copyString()); - + std::string const localKey( + localKeySlice.get(StaticStrings::KeyString).copyString()); + int res = localKey.compare(keyString); - + if (res != 0) { // we have a local key that is not present remotely keyBuilder.clear(); keyBuilder.openObject(); keyBuilder.add(StaticStrings::KeyString, VPackValue(localKey)); keyBuilder.close(); - + trx.remove(collectionName, keyBuilder.slice(), options); ++nextStart; } else { @@ -2064,13 +2076,14 @@ int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, break; } } - + MMFilesSimpleIndexElement element = idx->lookupKey(&trx, keySlice); - + if (!element) { // key not found locally toFetch.emplace_back(i); - } else if (TRI_RidToString(element.revisionId()) != pair.at(1).copyString()) { + } else if (TRI_RidToString(element.revisionId()) != + pair.at(1).copyString()) { // key found, but revision id differs toFetch.emplace_back(i); ++nextStart; @@ -2079,17 +2092,18 @@ int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, ++nextStart; } } - + // calculate next starting point if (!markers.empty()) { BinarySearch(markers, highString, nextStart); - + while (nextStart < markers.size()) { VPackSlice const localKeySlice(markers[nextStart]); - std::string const localKey(localKeySlice.get(StaticStrings::KeyString).copyString()); - + std::string const localKey( + localKeySlice.get(StaticStrings::KeyString).copyString()); + int res = localKey.compare(highString); - + if (res <= 0) { ++nextStart; } else { @@ -2097,7 +2111,7 @@ int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, } } } - + if (!toFetch.empty()) { VPackBuilder keysBuilder; keysBuilder.openArray(); @@ -2105,91 +2119,86 @@ int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, keysBuilder.add(VPackValue(it)); } keysBuilder.close(); - + std::string url = baseUrl + "/" + keysId + "?type=docs&chunk=" + - std::to_string(currentChunkId) + "&chunkSize=" + - std::to_string(chunkSize); + std::to_string(currentChunkId) + "&chunkSize=" + + std::to_string(chunkSize); progress = "fetching documents chunk " + - std::to_string(currentChunkId) + " for collection '" + - collectionName + "' from " + url; + std::to_string(currentChunkId) + " for collection '" + + collectionName + "' from " + url; setProgress(progress); - + std::string const keyJsonString(keysBuilder.slice().toJson()); - + std::unique_ptr response( - _client->retryRequest(rest::RequestType::PUT, url, - keyJsonString.c_str(), keyJsonString.size())); - + _client->retryRequest(rest::RequestType::PUT, url, + keyJsonString.c_str(), keyJsonString.size())); + if (response == nullptr || !response->isComplete()) { - errorMsg = "could not connect to master at " + - _masterInfo._endpoint + ": " + - _client->getErrorMessage(); - + errorMsg = "could not connect to master at " + _masterInfo._endpoint + + ": " + _client->getErrorMessage(); + return TRI_ERROR_REPLICATION_NO_RESPONSE; } - + TRI_ASSERT(response != nullptr); - + if (response->wasHttpError()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + ": HTTP " + - StringUtils::itoa(response->getHttpReturnCode()) + ": " + - response->getHttpReturnMessage(); - + _masterInfo._endpoint + ": HTTP " + + StringUtils::itoa(response->getHttpReturnCode()) + ": " + + response->getHttpReturnMessage(); + return TRI_ERROR_REPLICATION_MASTER_ERROR; } - + auto builder = std::make_shared(); int res = parseResponse(builder, response.get()); - + if (res != TRI_ERROR_NO_ERROR) { errorMsg = "got invalid response from master at " + - std::string(_masterInfo._endpoint) + - ": response is no array"; - + std::string(_masterInfo._endpoint) + + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const slice = builder->slice(); if (!slice.isArray()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": response is no array"; - + _masterInfo._endpoint + ": response is no array"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + for (auto const& it : VPackArrayIterator(slice)) { if (!it.isObject()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": document is no object"; - + _masterInfo._endpoint + ": document is no object"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const keySlice = it.get(StaticStrings::KeyString); - + if (!keySlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": document key is invalid"; - + _masterInfo._endpoint + ": document key is invalid"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + VPackSlice const revSlice = it.get(StaticStrings::RevString); - + if (!revSlice.isString()) { errorMsg = "got invalid response from master at " + - _masterInfo._endpoint + - ": document revision is invalid"; - + _masterInfo._endpoint + ": document revision is invalid"; + return TRI_ERROR_REPLICATION_INVALID_RESPONSE; } - + MMFilesSimpleIndexElement element = idx->lookupKey(&trx, keySlice); - + if (!element) { // INSERT OperationResult opRes = trx.insert(collectionName, it, options); @@ -2199,21 +2208,21 @@ int InitialSyncer::handleSyncKeysMMFiles(arangodb::LogicalCollection* col, OperationResult opRes = trx.update(collectionName, it, options); res = opRes.code; } - + if (res != TRI_ERROR_NO_ERROR) { return res; } } } } - + res = trx.commit(); - + if (!res.ok()) { return res.errorNumber(); } } - + return res; } From 7b2acaebfbcd5903294901a249baa00ff7551883 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Wed, 26 Apr 2017 07:33:14 +0200 Subject: [PATCH 26/58] use stringref instead of allocation --- arangod/RocksDBEngine/RocksDBCommon.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arangod/RocksDBEngine/RocksDBCommon.cpp b/arangod/RocksDBEngine/RocksDBCommon.cpp index 6106e0ecff..1b0ca686f5 100644 --- a/arangod/RocksDBEngine/RocksDBCommon.cpp +++ b/arangod/RocksDBEngine/RocksDBCommon.cpp @@ -23,6 +23,7 @@ /// @author Jan Christoph Uhde //////////////////////////////////////////////////////////////////////////////// +#include "Basics/StringRef.h" #include "RocksDBEngine/RocksDBCommon.h" #include "RocksDBEngine/RocksDBComparator.h" #include "RocksDBEngine/RocksDBEngine.h" @@ -133,7 +134,7 @@ void stripObjectIds(VPackBuilder& builder, VPackSlice const& slice) { builder.openObject(); for (auto it = arangodb::velocypack::ObjectIterator(slice); it.valid(); it++) { - if (it.key().copyString() == "objectId") { + if (arangodb::StringRef(it.key()) == "objectId") { continue; } builder.add(it.key()); From 925051afe4860f1cc13fcc07daceb4b04e364708 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 08:25:42 +0200 Subject: [PATCH 27/58] Rename an argument and add a comment. --- 3rdParty/velocypack/include/velocypack/Iterator.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/3rdParty/velocypack/include/velocypack/Iterator.h b/3rdParty/velocypack/include/velocypack/Iterator.h index 671264c766..16549e142b 100644 --- a/3rdParty/velocypack/include/velocypack/Iterator.h +++ b/3rdParty/velocypack/include/velocypack/Iterator.h @@ -182,9 +182,12 @@ class ObjectIterator { ObjectIterator() = delete; - explicit ObjectIterator(Slice const& slice, bool allowRandomIteration = false) + // The useSequentialIteration flag indicates whether or not the iteration + // simply jumps from key/value pair to key/value pair without using the + // index. The default `false` is to use the index if it is there. + explicit ObjectIterator(Slice const& slice, bool useSequentialIteration = false) : _slice(slice), _size(_slice.length()), _position(0), _current(nullptr), - _allowRandomIteration(allowRandomIteration) { + _useSequentialIteration(useSequentialIteration) { if (!slice.isObject()) { throw Exception(Exception::InvalidValueType, "Expecting Object slice"); } @@ -193,7 +196,7 @@ class ObjectIterator { auto h = slice.head(); if (h == 0x14) { _current = slice.keyAt(0, false).start(); - } else if (allowRandomIteration) { + } else if (useSequentialIteration) { _current = slice.begin() + slice.findDataOffset(h); } } @@ -204,7 +207,7 @@ class ObjectIterator { _size(other._size), _position(other._position), _current(other._current), - _allowRandomIteration(other._allowRandomIteration) {} + _useSequentialIteration(other._useSequentialIteration) {} ObjectIterator& operator=(ObjectIterator const& other) = delete; @@ -306,7 +309,7 @@ class ObjectIterator { ValueLength const _size; ValueLength _position; uint8_t const* _current; - bool const _allowRandomIteration; + bool const _useSequentialIteration; }; } // namespace arangodb::velocypack From 26112be584132fd60ad88f93eb1cd077341a1889 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Wed, 26 Apr 2017 08:33:23 +0200 Subject: [PATCH 28/58] do checking before creating copy of slice --- arangod/Replication/InitialSyncer.cpp | 5 +- arangod/RocksDBEngine/RocksDBCommon.cpp | 62 +++++++++++++++---- arangod/RocksDBEngine/RocksDBCommon.h | 3 +- .../RocksDBRestReplicationHandler.cpp | 5 +- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index d789164778..9631123b5d 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -274,9 +274,8 @@ int InitialSyncer::run(std::string& errorMsg, bool incremental) { errorMsg = "got invalid response from master at " + _masterInfo._endpoint + ": invalid JSON"; } else { - VPackBuilder stripBuilder; - stripObjectIds(stripBuilder, slice); - res = handleInventoryResponse(stripBuilder.slice(), incremental, + auto pair = stripObjectIds(slice); + res = handleInventoryResponse(pair.first, incremental, errorMsg); } } diff --git a/arangod/RocksDBEngine/RocksDBCommon.cpp b/arangod/RocksDBEngine/RocksDBCommon.cpp index 1b0ca686f5..de0417b8ff 100644 --- a/arangod/RocksDBEngine/RocksDBCommon.cpp +++ b/arangod/RocksDBEngine/RocksDBCommon.cpp @@ -129,28 +129,66 @@ void uint64ToPersistent(std::string& p, uint64_t value) { } while (++len < sizeof(uint64_t)); } -void stripObjectIds(VPackBuilder& builder, VPackSlice const& slice) { - if (slice.isObject()) { +bool hasObjectIds(VPackSlice const& inputSlice) { + bool rv = false; + if (inputSlice.isObject()) { + for (auto const& objectPair : + arangodb::velocypack::ObjectIterator(inputSlice)) { + if (arangodb::StringRef(objectPair.key) == "objectId") { + return true; + } + rv = hasObjectIds(objectPair.value); + if (rv) { + return rv; + } + } + } else if (inputSlice.isArray()) { + for (auto const& slice : arangodb::velocypack::ArrayIterator(inputSlice)) { + if (rv) { + return rv; + } + rv = hasObjectIds(slice); + } + } + return rv; +} + +VPackBuilder& stripObjectIdsImpl(VPackBuilder& builder, VPackSlice const& inputSlice) { + if (inputSlice.isObject()) { builder.openObject(); - for (auto it = arangodb::velocypack::ObjectIterator(slice); it.valid(); - it++) { - if (arangodb::StringRef(it.key()) == "objectId") { + for (auto const& objectPair : + arangodb::velocypack::ObjectIterator(inputSlice)) { + if (arangodb::StringRef(objectPair.key) == "objectId") { continue; } - builder.add(it.key()); - stripObjectIds(builder, it.value()); + builder.add(objectPair.key); + stripObjectIdsImpl(builder, objectPair.value); } builder.close(); - } else if (slice.isArray()) { + } else if (inputSlice.isArray()) { builder.openArray(); - for (auto it = arangodb::velocypack::ArrayIterator(slice); it.valid(); - it++) { - stripObjectIds(builder, it.value()); + for (auto const& slice : arangodb::velocypack::ArrayIterator(inputSlice)) { + stripObjectIdsImpl(builder, slice); } builder.close(); } else { - builder.add(slice); + builder.add(inputSlice); } + return builder; +} + +std::pair>> stripObjectIds( + VPackSlice const& inputSlice, bool checkBeforeCopy) { + std::unique_ptr> buffer = nullptr; + if (checkBeforeCopy) { + if (!hasObjectIds(inputSlice)) { + return {inputSlice, std::move(buffer)}; + } + } + buffer.reset(new VPackBuffer); + VPackBuilder builder(*buffer); + stripObjectIdsImpl(builder, inputSlice); + return {VPackSlice(buffer->data()), std::move(buffer)}; } RocksDBTransactionState* toRocksTransactionState(transaction::Methods* trx) { diff --git a/arangod/RocksDBEngine/RocksDBCommon.h b/arangod/RocksDBEngine/RocksDBCommon.h index 10927aa83a..9616eb8bce 100644 --- a/arangod/RocksDBEngine/RocksDBCommon.h +++ b/arangod/RocksDBEngine/RocksDBCommon.h @@ -89,7 +89,8 @@ uint64_t uint64FromPersistent(char const* p); void uint64ToPersistent(char* p, uint64_t value); void uint64ToPersistent(std::string& out, uint64_t value); -void stripObjectIds(VPackBuilder&, VPackSlice const&); +std::pair>> stripObjectIds( + VPackSlice const& inputSlice, bool checkBeforeCopy = true); RocksDBTransactionState* toRocksTransactionState(transaction::Methods* trx); rocksdb::TransactionDB* globalRocksDB(); diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index 4af190e684..35636be9dd 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -826,9 +826,8 @@ void RocksDBRestReplicationHandler::handleCommandRestoreCollection() { "invalid JSON"); return; } - VPackBuilder builder; - stripObjectIds(builder, parsedRequest->slice()); - VPackSlice const slice = builder.slice(); + auto pair = stripObjectIds(parsedRequest->slice()); + VPackSlice const slice = pair.first; bool overwrite = false; From 52b3b67479b9d92041db64c1c19b29efe4756df2 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 08:58:17 +0200 Subject: [PATCH 29/58] Fixed cluster restore of System Collections. --- arangod/MMFiles/MMFilesRestReplicationHandler.cpp | 5 +++-- arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arangod/MMFiles/MMFilesRestReplicationHandler.cpp b/arangod/MMFiles/MMFilesRestReplicationHandler.cpp index 20cfcc7fa7..028d0a275c 100644 --- a/arangod/MMFiles/MMFilesRestReplicationHandler.cpp +++ b/arangod/MMFiles/MMFilesRestReplicationHandler.cpp @@ -1592,14 +1592,15 @@ int MMFilesRestReplicationHandler::processRestoreCollectionCoordinator( if (dropExisting) { int res = ci->dropCollectionCoordinator(dbName, col->cid_as_string(), errorMsg, 0.0); - if (res == TRI_ERROR_FORBIDDEN) { + if (res == TRI_ERROR_FORBIDDEN || + res == TRI_ERROR_CLUSTER_MUST_NOT_DROP_COLL_OTHER_DISTRIBUTESHARDSLIKE) { // some collections must not be dropped res = truncateCollectionOnCoordinator(dbName, name); if (res != TRI_ERROR_NO_ERROR) { errorMsg = "unable to truncate collection (dropping is forbidden): " + name; - return res; } + return res; } if (res != TRI_ERROR_NO_ERROR) { diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index 4af190e684..6d7a0463b5 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -1732,14 +1732,15 @@ int RocksDBRestReplicationHandler::processRestoreCollectionCoordinator( if (dropExisting) { int res = ci->dropCollectionCoordinator(dbName, col->cid_as_string(), errorMsg, 0.0); - if (res == TRI_ERROR_FORBIDDEN) { + if (res == TRI_ERROR_FORBIDDEN || + res == TRI_ERROR_CLUSTER_MUST_NOT_DROP_COLL_OTHER_DISTRIBUTESHARDSLIKE) { // some collections must not be dropped res = truncateCollectionOnCoordinator(dbName, name); if (res != TRI_ERROR_NO_ERROR) { errorMsg = "unable to truncate collection (dropping is forbidden): " + name; - return res; } + return res; } if (res != TRI_ERROR_NO_ERROR) { From 336a0ad20eb1b3d42985716545722aa011002216 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 25 Apr 2017 20:49:20 +0200 Subject: [PATCH 30/58] another try to get the LICENSE into place --- cmake/InstallMacros.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/InstallMacros.cmake b/cmake/InstallMacros.cmake index ed03932e3d..c6b8687319 100644 --- a/cmake/InstallMacros.cmake +++ b/cmake/InstallMacros.cmake @@ -76,7 +76,8 @@ macro (install_readme input output) endif () install( - CODE "configure_file(${PROJECT_SOURCE_DIR}/${input} \"${PROJECT_BINARY_DIR}/${output}\" NEWLINE_STYLE ${CRLFSTYLE})" + CODE "configure_file(${PROJECT_SOURCE_DIR}/${input} \"${PROJECT_BINARY_DIR}/${output}\" NEWLINE_STYLE ${CRLFSTYLE})") + install( FILES "${PROJECT_BINARY_DIR}/${output}" DESTINATION "${where}" ) From e6aa022962d985a9cb42c8e2aa21d1c17ed72279 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 09:36:01 +0200 Subject: [PATCH 31/58] Fixed dump tests in cluster. MMFiles is working RocksDB gets into undefined state somewhere. --- ...ump-cluster.js => dump-mmfiles-cluster.js} | 0 js/server/tests/dump/dump-mmfiles.js | 63 ++-- js/server/tests/dump/dump-rocksdb-cluster.js | 308 ++++++++++++++++++ 3 files changed, 339 insertions(+), 32 deletions(-) rename js/server/tests/dump/{dump-cluster.js => dump-mmfiles-cluster.js} (100%) create mode 100644 js/server/tests/dump/dump-rocksdb-cluster.js diff --git a/js/server/tests/dump/dump-cluster.js b/js/server/tests/dump/dump-mmfiles-cluster.js similarity index 100% rename from js/server/tests/dump/dump-cluster.js rename to js/server/tests/dump/dump-mmfiles-cluster.js diff --git a/js/server/tests/dump/dump-mmfiles.js b/js/server/tests/dump/dump-mmfiles.js index 10b1dcfeaf..ab367009ae 100644 --- a/js/server/tests/dump/dump-mmfiles.js +++ b/js/server/tests/dump/dump-mmfiles.js @@ -58,7 +58,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test the empty collection //////////////////////////////////////////////////////////////////////////////// - + testEmpty : function () { var c = db._collection("UnitTestsDumpEmpty"); var p = c.properties(); @@ -76,7 +76,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test the collection with many documents //////////////////////////////////////////////////////////////////////////////// - + testMany : function () { var c = db._collection("UnitTestsDumpMany"); var p = c.properties(); @@ -101,7 +101,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test the edges collection //////////////////////////////////////////////////////////////////////////////// - + testEdges : function () { var c = db._collection("UnitTestsDumpEdges"); var p = c.properties(); @@ -128,7 +128,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test the order of documents //////////////////////////////////////////////////////////////////////////////// - + testOrder : function () { var c = db._collection("UnitTestsDumpOrder"); var p = c.properties(); @@ -146,7 +146,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test document removal & update //////////////////////////////////////////////////////////////////////////////// - + testRemoved : function () { var c = db._collection("UnitTestsDumpRemoved"); var p = c.properties(); @@ -178,7 +178,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test indexes //////////////////////////////////////////////////////////////////////////////// - + testIndexes : function () { var c = db._collection("UnitTestsDumpIndexes"); var p = c.properties(); @@ -200,32 +200,32 @@ function dumpTestSuite () { assertFalse(c.getIndexes()[2].unique); assertFalse(c.getIndexes()[2].sparse); assertEqual([ "a_s1", "a_s2" ], c.getIndexes()[2].fields); - + assertEqual("hash", c.getIndexes()[3].type); assertFalse(c.getIndexes()[3].unique); assertFalse(c.getIndexes()[3].sparse); assertEqual([ "a_h1", "a_h2" ], c.getIndexes()[3].fields); - + assertEqual("skiplist", c.getIndexes()[4].type); assertTrue(c.getIndexes()[4].unique); assertFalse(c.getIndexes()[4].sparse); assertEqual([ "a_su" ], c.getIndexes()[4].fields); - + assertEqual("hash", c.getIndexes()[5].type); assertFalse(c.getIndexes()[5].unique); assertTrue(c.getIndexes()[5].sparse); assertEqual([ "a_hs1", "a_hs2" ], c.getIndexes()[5].fields); - + assertEqual("skiplist", c.getIndexes()[6].type); assertFalse(c.getIndexes()[6].unique); assertTrue(c.getIndexes()[6].sparse); assertEqual([ "a_ss1", "a_ss2" ], c.getIndexes()[6].fields); - + if (db._engine().name !== "rocksdb") { assertFalse(c.getIndexes()[7].unique); assertEqual("fulltext", c.getIndexes()[7].type); assertEqual([ "a_f" ], c.getIndexes()[7].fields); - + assertEqual("geo2", c.getIndexes()[8].type); assertEqual([ "a_la", "a_lo" ], c.getIndexes()[8].fields); assertFalse(c.getIndexes()[8].unique); @@ -237,7 +237,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test truncate //////////////////////////////////////////////////////////////////////////////// - + testTruncated : function () { var c = db._collection("UnitTestsDumpTruncated"); var p = c.properties(); @@ -254,7 +254,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test keygen //////////////////////////////////////////////////////////////////////////////// - + testKeygen : function () { var c = db._collection("UnitTestsDumpKeygen"); var p = c.properties(); @@ -270,7 +270,7 @@ function dumpTestSuite () { assertEqual(1, c.getIndexes().length); // just primary index assertEqual("primary", c.getIndexes()[0].type); assertEqual(1000, c.count()); - + for (var i = 0; i < 1000; ++i) { var doc = c.document(String(7 + (i * 42))); @@ -283,7 +283,7 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test strings //////////////////////////////////////////////////////////////////////////////// - + testStrings : function () { var c = db._collection("UnitTestsDumpStrings"); var p = c.properties(); @@ -298,18 +298,18 @@ function dumpTestSuite () { var texts = [ "big. Really big. He moment. Magrathea! - insisted Arthur, - I do you can sense no further because it doesn't fit properly. In my the denies faith, and the atmosphere beneath You are not cheap He was was his satchel. He throughout Magrathea. - He pushed a tore the ecstatic crowd. Trillian sat down the time, the existence is it? And he said, - What they don't want this airtight hatchway. - it's we you shooting people would represent their Poet Master Grunthos is in his mind.", - "Ultimo cadere chi sedete uso chiuso voluto ora. Scotendosi portartela meraviglia ore eguagliare incessante allegrezza per. Pensava maestro pungeva un le tornano ah perduta. Fianco bearmi storia soffio prende udi poteva una. Cammino fascino elisire orecchi pollici mio cui sai sul. Chi egli sino sei dita ben. Audace agonie groppa afa vai ultima dentro scossa sii. Alcuni mia blocco cerchi eterno andare pagine poi. Ed migliore di sommesso oh ai angoscia vorresti.", + "Ultimo cadere chi sedete uso chiuso voluto ora. Scotendosi portartela meraviglia ore eguagliare incessante allegrezza per. Pensava maestro pungeva un le tornano ah perduta. Fianco bearmi storia soffio prende udi poteva una. Cammino fascino elisire orecchi pollici mio cui sai sul. Chi egli sino sei dita ben. Audace agonie groppa afa vai ultima dentro scossa sii. Alcuni mia blocco cerchi eterno andare pagine poi. Ed migliore di sommesso oh ai angoscia vorresti.", "Νέο βάθος όλα δομές της χάσει. Μέτωπο εγώ συνάμα τρόπος και ότι όσο εφόδιο κόσμου. Προτίμηση όλη διάφορους του όλο εύθραυστη συγγραφής. Στα άρα ένα μία οποία άλλων νόημα. Ένα αποβαίνει ρεαλισμού μελετητές θεόσταλτο την. Ποντιακών και rites κοριτσάκι παπούτσια παραμύθια πει κυρ.", "Mody laty mnie ludu pole rury Białopiotrowiczowi. Domy puer szczypię jemy pragnął zacność czytając ojca lasy Nowa wewnątrz klasztoru. Chce nóg mego wami. Zamku stał nogą imion ludzi ustaw Białopiotrowiczem. Kwiat Niesiołowskiemu nierostrzygniony Staje brał Nauka dachu dumę Zamku Kościuszkowskie zagon. Jakowaś zapytać dwie mój sama polu uszakach obyczaje Mój. Niesiołowski książkowéj zimny mały dotychczasowa Stryj przestraszone Stolnikównie wdał śmiertelnego. Stanisława charty kapeluszach mięty bratem każda brząknął rydwan.", - "Мелких против летают хижину тмится. Чудесам возьмет звездна Взжигай. . Податель сельские мучитель сверкает очищаясь пламенем. Увы имя меч Мое сия. Устранюсь воздушных Им от До мысленные потушатся Ко Ея терпеньем.", + "Мелких против летают хижину тмится. Чудесам возьмет звездна Взжигай. . Податель сельские мучитель сверкает очищаясь пламенем. Увы имя меч Мое сия. Устранюсь воздушных Им от До мысленные потушатся Ко Ея терпеньем.", "dotyku. Výdech spalin bude položen záplavový detekční kabely 1x UPS Newave Conceptpower DPA 5x 40kVA bude ukončen v samostatné strojovně. Samotné servery mají pouze lokalita Ústí nad zdvojenou podlahou budou zakončené GateWayí HiroLink - Monitoring rozvaděče RTN na jednotlivých záplavových zón na soustrojí resp. technologie jsou označeny SA-MKx.y. Jejich výstupem je zajištěn přestupem dat z jejich provoz. Na dveřích vylepené výstražné tabulky. Kabeláž z okruhů zálohovaných obvodů v R.MON-I. Monitoring EZS, EPS, ... možno zajistit funkčností FireWallů na strukturovanou kabeláží vedenou v měrných jímkách zapuštěných v každém racku budou zakončeny v R.MON-NrNN. Monitoring motorgenerátorů: řídící systém bude zakončena v modulu", "ramien mu zrejme vôbec niekto je už presne čo mám tendenciu prispôsobiť dych jej páčil, čo chce. Hmm... Včera sa mi pozdava, len dočkali, ale keďže som na uz boli u jej nezavrela. Hlava jej to ve městě nepotká, hodně mi to tí vedci pri hre, keď je tu pre Designiu. Pokiaľ viete o odbornejšie texty. Prvým z tmavých uličiek, každý to niekedy, zrovnávať krok s obrovským batohom na okraj vane a temné úmysly, tak rozmýšľam, aký som si hromady mailov, čo chcem a neraz sa pokúšal o filmovém klubu v budúcnosti rozhodne uniesť mladú maliarku (Linda Rybová), ktorú so", " 復讐者」. 復讐者」. 伯母さん 復讐者」. 復讐者」. 復讐者」. 復讐者」. 第九章 第五章 第六章 第七章 第八章. 復讐者」 伯母さん. 復讐者」 伯母さん. 第十一章 第十九章 第十四章 第十八章 第十三章 第十五章. 復讐者」 . 第十四章 第十一章 第十二章 第十五章 第十七章 手配書. 第十四章 手配書 第十八章 第十七章 第十六章 第十三章. 第十一章 第十三章 第十八章 第十四章 手配書. 復讐者」." ]; - texts.forEach(function (t, i) { + texts.forEach(function (t, i) { var doc = c.document("text" + i); - + assertEqual(t, doc.value); }); @@ -318,12 +318,12 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test committed trx //////////////////////////////////////////////////////////////////////////////// - + testTransactionCommit : function () { var c = db._collection("UnitTestsDumpTransactionCommit"); assertEqual(1000, c.count()); - + for (var i = 0; i < 1000; ++i) { var doc = c.document("test" + i); @@ -336,12 +336,12 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test committed trx //////////////////////////////////////////////////////////////////////////////// - + testTransactionUpdate : function () { var c = db._collection("UnitTestsDumpTransactionUpdate"); assertEqual(1000, c.count()); - + for (var i = 0; i < 1000; ++i) { var doc = c.document("test" + i); @@ -359,37 +359,37 @@ function dumpTestSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief test aborted trx //////////////////////////////////////////////////////////////////////////////// - + testTransactionAbort : function () { var c = db._collection("UnitTestsDumpTransactionAbort"); assertEqual(1, c.count()); - + assertTrue(c.exists("foo")); }, //////////////////////////////////////////////////////////////////////////////// /// @brief test persistent //////////////////////////////////////////////////////////////////////////////// - + testPersistent : function () { var c = db._collection("UnitTestsDumpPersistent"); var p = c.properties(); - assertEqual(2, c.getIndexes().length); + assertEqual(2, c.getIndexes().length); assertEqual("primary", c.getIndexes()[0].type); assertEqual("persistent", c.getIndexes()[1].type); assertEqual(10000, c.count()); var res = db._query("FOR doc IN " + c.name() + " FILTER doc.value >= 0 RETURN doc").toArray(); assertEqual(10000, res.length); - + res = db._query("FOR doc IN " + c.name() + " FILTER doc.value >= 5000 RETURN doc").toArray(); assertEqual(5000, res.length); - + res = db._query("FOR doc IN " + c.name() + " FILTER doc.value >= 9000 RETURN doc").toArray(); assertEqual(1000, res.length); - + res = db._query("FOR doc IN " + c.name() + " FILTER doc.value >= 10000 RETURN doc").toArray(); assertEqual(0, res.length); } @@ -404,4 +404,3 @@ function dumpTestSuite () { jsunity.run(dumpTestSuite); return jsunity.done(); - diff --git a/js/server/tests/dump/dump-rocksdb-cluster.js b/js/server/tests/dump/dump-rocksdb-cluster.js new file mode 100644 index 0000000000..743a913d82 --- /dev/null +++ b/js/server/tests/dump/dump-rocksdb-cluster.js @@ -0,0 +1,308 @@ +/*jshint globalstrict:false, strict:false, maxlen : 4000 */ +/*global assertEqual, assertTrue, assertFalse */ + +//////////////////////////////////////////////////////////////////////////////// +/// @brief tests for dump/reload +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2010-2012 triagens GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is triAGENS GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +var internal = require("internal"); +var jsunity = require("jsunity"); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite +//////////////////////////////////////////////////////////////////////////////// + +function dumpTestSuite () { + 'use strict'; + var db = internal.db; + + return { + +//////////////////////////////////////////////////////////////////////////////// +/// @brief set up +//////////////////////////////////////////////////////////////////////////////// + + setUp : function () { + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief tear down +//////////////////////////////////////////////////////////////////////////////// + + tearDown : function () { + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test the empty collection +//////////////////////////////////////////////////////////////////////////////// + + testEmpty : function () { + var c = db._collection("UnitTestsDumpEmpty"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertTrue(p.waitForSync); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(0, c.count()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test the collection with many documents +//////////////////////////////////////////////////////////////////////////////// + + testMany : function () { + var c = db._collection("UnitTestsDumpMany"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(100000, c.count()); + + // test all documents + var r = db._query(`FOR d IN ${c.name()} RETURN d`).toArray(); + var rr = new Map(); + for (let i = 0; i < r.length; ++i) { + rr.set(r[i]._key, r[i]); + } + for (let i = 0; i < 100000; ++i) { + var doc = rr.get("test" + i); + assertEqual(i, doc.value1); + assertEqual("this is a test", doc.value2); + assertEqual("test" + i, doc.value3); + } + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test the edges collection +//////////////////////////////////////////////////////////////////////////////// + + testEdges : function () { + var c = db._collection("UnitTestsDumpEdges"); + var p = c.properties(); + + assertEqual(3, c.type()); // edges + assertFalse(p.waitForSync); + + assertEqual(2, c.getIndexes().length); // primary index + edges index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual("edge", c.getIndexes()[1].type); + assertEqual(10, c.count()); + + // test all documents + for (var i = 0; i < 10; ++i) { + var doc = c.document("test" + i); + assertEqual("test" + i, doc._key); + assertEqual("UnitTestsDumpMany/test" + i, doc._from); + assertEqual("UnitTestsDumpMany/test" + (i + 1), doc._to); + assertEqual(i + "->" + (i + 1), doc.what); + } + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test the order of documents +//////////////////////////////////////////////////////////////////////////////// + + testOrder : function () { + var c = db._collection("UnitTestsDumpOrder"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(3, c.count()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test document removal & update +//////////////////////////////////////////////////////////////////////////////// + + testRemoved : function () { + var c = db._collection("UnitTestsDumpRemoved"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(9000, c.count()); + + var i; + for (i = 0; i < 10000; ++i) { + if (i % 10 === 0) { + assertFalse(c.exists("test" + i)); + } + else { + var doc = c.document("test" + i); + assertEqual(i, doc.value1); + + if (i < 1000) { + assertEqual(i + 1, doc.value2); + } + } + } + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test indexes +//////////////////////////////////////////////////////////////////////////////// + + testIndexes : function () { + var c = db._collection("UnitTestsDumpIndexes"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + + assertEqual(7, c.getIndexes().length); + assertEqual("primary", c.getIndexes()[0].type); + + assertEqual("hash", c.getIndexes()[1].type); + assertTrue(c.getIndexes()[1].unique); + assertFalse(c.getIndexes()[1].sparse); + assertEqual([ "a_uc" ], c.getIndexes()[1].fields); + + assertEqual("skiplist", c.getIndexes()[2].type); + assertFalse(c.getIndexes()[2].unique); + assertFalse(c.getIndexes()[2].sparse); + assertEqual([ "a_s1", "a_s2" ], c.getIndexes()[2].fields); + + assertEqual("hash", c.getIndexes()[3].type); + assertFalse(c.getIndexes()[3].unique); + assertFalse(c.getIndexes()[3].sparse); + assertEqual([ "a_h1", "a_h2" ], c.getIndexes()[3].fields); + + assertEqual("skiplist", c.getIndexes()[4].type); + assertTrue(c.getIndexes()[4].unique); + assertFalse(c.getIndexes()[4].sparse); + assertEqual([ "a_su" ], c.getIndexes()[4].fields); + + assertEqual("hash", c.getIndexes()[5].type); + assertFalse(c.getIndexes()[5].unique); + assertTrue(c.getIndexes()[5].sparse); + assertEqual([ "a_hs1", "a_hs2" ], c.getIndexes()[5].fields); + + assertEqual("skiplist", c.getIndexes()[6].type); + assertFalse(c.getIndexes()[6].unique); + assertTrue(c.getIndexes()[6].sparse); + assertEqual([ "a_ss1", "a_ss2" ], c.getIndexes()[6].fields); + + assertEqual(0, c.count()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test truncate +//////////////////////////////////////////////////////////////////////////////// + + testTruncated : function () { + var c = db._collection("UnitTestsDumpTruncated"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(0, c.count()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test shards +//////////////////////////////////////////////////////////////////////////////// + + testShards : function () { + var c = db._collection("UnitTestsDumpShards"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + assertEqual(9, p.numberOfShards); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(1000, c.count()); + + for (var i = 0; i < 1000; ++i) { + var doc = c.document(String(7 + (i * 42))); + + assertEqual(String(7 + (i * 42)), doc._key); + assertEqual(i, doc.value); + assertEqual({ value: [ i, i ] }, doc.more); + } + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test strings +//////////////////////////////////////////////////////////////////////////////// + + testStrings : function () { + var c = db._collection("UnitTestsDumpStrings"); + var p = c.properties(); + + assertEqual(2, c.type()); // document + assertFalse(p.waitForSync); + + assertEqual(1, c.getIndexes().length); // just primary index + assertEqual("primary", c.getIndexes()[0].type); + assertEqual(8, c.count()); + + var texts = [ + "big. Really big. He moment. Magrathea! - insisted Arthur, - I do you can sense no further because it doesn't fit properly. In my the denies faith, and the atmosphere beneath You are not cheap He was was his satchel. He throughout Magrathea. - He pushed a tore the ecstatic crowd. Trillian sat down the time, the existence is it? And he said, - What they don't want this airtight hatchway. - it's we you shooting people would represent their Poet Master Grunthos is in his mind.", + "Ultimo cadere chi sedete uso chiuso voluto ora. Scotendosi portartela meraviglia ore eguagliare incessante allegrezza per. Pensava maestro pungeva un le tornano ah perduta. Fianco bearmi storia soffio prende udi poteva una. Cammino fascino elisire orecchi pollici mio cui sai sul. Chi egli sino sei dita ben. Audace agonie groppa afa vai ultima dentro scossa sii. Alcuni mia blocco cerchi eterno andare pagine poi. Ed migliore di sommesso oh ai angoscia vorresti.", + "Νέο βάθος όλα δομές της χάσει. Μέτωπο εγώ συνάμα τρόπος και ότι όσο εφόδιο κόσμου. Προτίμηση όλη διάφορους του όλο εύθραυστη συγγραφής. Στα άρα ένα μία οποία άλλων νόημα. Ένα αποβαίνει ρεαλισμού μελετητές θεόσταλτο την. Ποντιακών και rites κοριτσάκι παπούτσια παραμύθια πει κυρ.", + "Mody laty mnie ludu pole rury Białopiotrowiczowi. Domy puer szczypię jemy pragnął zacność czytając ojca lasy Nowa wewnątrz klasztoru. Chce nóg mego wami. Zamku stał nogą imion ludzi ustaw Białopiotrowiczem. Kwiat Niesiołowskiemu nierostrzygniony Staje brał Nauka dachu dumę Zamku Kościuszkowskie zagon. Jakowaś zapytać dwie mój sama polu uszakach obyczaje Mój. Niesiołowski książkowéj zimny mały dotychczasowa Stryj przestraszone Stolnikównie wdał śmiertelnego. Stanisława charty kapeluszach mięty bratem każda brząknął rydwan.", + "Мелких против летают хижину тмится. Чудесам возьмет звездна Взжигай. . Податель сельские мучитель сверкает очищаясь пламенем. Увы имя меч Мое сия. Устранюсь воздушных Им от До мысленные потушатся Ко Ея терпеньем.", + "dotyku. Výdech spalin bude položen záplavový detekční kabely 1x UPS Newave Conceptpower DPA 5x 40kVA bude ukončen v samostatné strojovně. Samotné servery mají pouze lokalita Ústí nad zdvojenou podlahou budou zakončené GateWayí HiroLink - Monitoring rozvaděče RTN na jednotlivých záplavových zón na soustrojí resp. technologie jsou označeny SA-MKx.y. Jejich výstupem je zajištěn přestupem dat z jejich provoz. Na dveřích vylepené výstražné tabulky. Kabeláž z okruhů zálohovaných obvodů v R.MON-I. Monitoring EZS, EPS, ... možno zajistit funkčností FireWallů na strukturovanou kabeláží vedenou v měrných jímkách zapuštěných v každém racku budou zakončeny v R.MON-NrNN. Monitoring motorgenerátorů: řídící systém bude zakončena v modulu", + "ramien mu zrejme vôbec niekto je už presne čo mám tendenciu prispôsobiť dych jej páčil, čo chce. Hmm... Včera sa mi pozdava, len dočkali, ale keďže som na uz boli u jej nezavrela. Hlava jej to ve městě nepotká, hodně mi to tí vedci pri hre, keď je tu pre Designiu. Pokiaľ viete o odbornejšie texty. Prvým z tmavých uličiek, každý to niekedy, zrovnávať krok s obrovským batohom na okraj vane a temné úmysly, tak rozmýšľam, aký som si hromady mailov, čo chcem a neraz sa pokúšal o filmovém klubu v budúcnosti rozhodne uniesť mladú maliarku (Linda Rybová), ktorú so", + " 復讐者」. 復讐者」. 伯母さん 復讐者」. 復讐者」. 復讐者」. 復讐者」. 第九章 第五章 第六章 第七章 第八章. 復讐者」 伯母さん. 復讐者」 伯母さん. 第十一章 第十九章 第十四章 第十八章 第十三章 第十五章. 復讐者」 . 第十四章 第十一章 第十二章 第十五章 第十七章 手配書. 第十四章 手配書 第十八章 第十七章 第十六章 第十三章. 第十一章 第十三章 第十八章 第十四章 手配書. 復讐者」." + ]; + + texts.forEach(function (t, i) { + var doc = c.document("text" + i); + + assertEqual(t, doc.value); + }); + + } + + }; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief executes the test suite +//////////////////////////////////////////////////////////////////////////////// + +jsunity.run(dumpTestSuite); + +return jsunity.done(); + From c59857c2e6b234e99f4861da10ea0703ed327fa9 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 09:50:16 +0200 Subject: [PATCH 32/58] decrease max wait time --- CHANGELOG | 2 +- arangod/V8Server/V8DealerFeature.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 369a703ca0..bde6d4438d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -113,7 +113,7 @@ devel when unused. Waiting for an unused V8 context will now also abort if no V8 context can be - acquired/created after 120 seconds. + acquired/created after 60 seconds. * improved diagnostic messages written to logfiles by supervisor process diff --git a/arangod/V8Server/V8DealerFeature.cpp b/arangod/V8Server/V8DealerFeature.cpp index 003f973956..08a3a266fa 100644 --- a/arangod/V8Server/V8DealerFeature.cpp +++ b/arangod/V8Server/V8DealerFeature.cpp @@ -566,7 +566,7 @@ V8Context* V8DealerFeature::enterContext(TRI_vocbase_t* vocbase, TimedAction exitWhenNoContext([](double waitTime) { LOG_TOPIC(WARN, arangodb::Logger::V8) << "giving up waiting for unused V8 context after " << Logger::FIXED(waitTime) << " s"; - }, 120); + }, 60); V8Context* context = nullptr; From 404ef431ae1532ec5acf72968f67b757e3aaabdb Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 09:50:55 +0200 Subject: [PATCH 33/58] cppcheck --- arangod/Replication/InitialSyncer.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 9631123b5d..7f61c18fcd 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -1258,7 +1258,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, std::function parseDoc = [&](VPackSlice doc, VPackSlice key) { - bool rangeUneqal = false; + bool rangeUnequal = false; bool nextChunk = false; int cmp1 = key.compareString(lowKey.data(), lowKey.length()); @@ -1274,7 +1274,7 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, if (cmp1 == 0) { foundLowKey = true; } else if (!foundLowKey && cmp1 > 0) { - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } @@ -1286,28 +1286,28 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, markers.emplace_back(key.copyString(), TRI_ExtractRevisionId(doc)); if (cmp2 == 0) { // found highKey - rangeUneqal = std::to_string(localHash) != hashString; + rangeUnequal = std::to_string(localHash) != hashString; nextChunk = true; } } else if (cmp2 == 0) { - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } } else if (cmp2 > 0) { // higher than highKey // current range was unequal and we did not find the // high key. Load range and skip to next - rangeUneqal = true; + rangeUnequal = true; nextChunk = true; } - if (rangeUneqal) { + if (rangeUnequal) { int res = syncChunkRocksDB(&trx, keysId, currentChunkId, lowKey, highKey, markers, errorMsg); if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); } } - TRI_ASSERT(!rangeUneqal || (rangeUneqal && nextChunk)); // A => B + TRI_ASSERT(!rangeUnequal || nextChunk); // A => B if (nextChunk && currentChunkId + 1 < numChunks) { currentChunkId++; // we are out of range, see next chunk resetChunk(); From c617ebddbfd011e340c317cc673b961c8be42ebd Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 09:51:40 +0200 Subject: [PATCH 34/58] honor return value of TRI_vocbase_t::use --- arangod/Utils/DatabaseGuard.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arangod/Utils/DatabaseGuard.h b/arangod/Utils/DatabaseGuard.h index 44d51c9a0b..3e585dfa54 100644 --- a/arangod/Utils/DatabaseGuard.h +++ b/arangod/Utils/DatabaseGuard.h @@ -40,7 +40,9 @@ class DatabaseGuard { explicit DatabaseGuard(TRI_vocbase_t* vocbase) : _vocbase(vocbase) { TRI_ASSERT(vocbase != nullptr); - _vocbase->use(); + if (!_vocbase->use()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); + } } /// @brief create the guard, using a database id From f876eef08838f0aaa4dc1cb29731a4f5873f7032 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 09:52:00 +0200 Subject: [PATCH 35/58] Removed unused member variable in ShortestPathBlock --- arangod/Aql/ShortestPathBlock.cpp | 4 +--- arangod/Aql/ShortestPathBlock.h | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index 3630ec8104..d3925724a6 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -257,9 +257,7 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { VPackSlice start = _opts->getStart(); VPackSlice end = _opts->getEnd(); TRI_ASSERT(_finder != nullptr); - // We do not need this data anymore. Result has been processed. - // Save some memory. - _coordinatorCache.clear(); + bool hasPath = _finder->shortestPath(start, end, *_path, [this]() { throwIfKilled(); }); diff --git a/arangod/Aql/ShortestPathBlock.h b/arangod/Aql/ShortestPathBlock.h index 6164a17d77..4f2fcfe963 100644 --- a/arangod/Aql/ShortestPathBlock.h +++ b/arangod/Aql/ShortestPathBlock.h @@ -146,9 +146,6 @@ class ShortestPathBlock : public ExecutionBlock { /// We use it to check if we are done with enumerating. bool _usedConstant; - /// @brief Cache for edges send over the network - std::vector>> _coordinatorCache; - /// @brief Traverser Engines std::unordered_map const* _engines; From 1ac30babf527fc03044911a21cd846e6c67d1c26 Mon Sep 17 00:00:00 2001 From: Frank Celler Date: Wed, 26 Apr 2017 09:54:03 +0200 Subject: [PATCH 36/58] raised statistics entries --- arangod/Statistics/ConnectionStatistics.h | 2 +- arangod/Statistics/RequestStatistics.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arangod/Statistics/ConnectionStatistics.h b/arangod/Statistics/ConnectionStatistics.h index c5128694b6..c218792699 100644 --- a/arangod/Statistics/ConnectionStatistics.h +++ b/arangod/Statistics/ConnectionStatistics.h @@ -65,7 +65,7 @@ class ConnectionStatistics { _error = false; } - static size_t const QUEUE_SIZE = 5000; + static size_t const QUEUE_SIZE = 64 * 1024 - 2; // current (1.62) boost maximum static Mutex _dataLock; diff --git a/arangod/Statistics/RequestStatistics.h b/arangod/Statistics/RequestStatistics.h index 5b4d4f263d..daac2c602e 100644 --- a/arangod/Statistics/RequestStatistics.h +++ b/arangod/Statistics/RequestStatistics.h @@ -152,7 +152,7 @@ class RequestStatistics { void trace_log(); private: - static size_t const QUEUE_SIZE = 1000; + static size_t const QUEUE_SIZE = 64 * 1024 - 2; // current (1.62) boost maximum static arangodb::Mutex _dataLock; From dad5a1429edbd1e1e43abecc543a0cfdc3e5d63b Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 26 Apr 2017 09:53:57 +0200 Subject: [PATCH 37/58] Add waitForSyncReplication as a _create() option --- arangod/Cluster/ClusterFeature.cpp | 4 +++ arangod/Cluster/ClusterFeature.h | 2 ++ arangod/Cluster/ClusterInfo.cpp | 31 +++++++++++-------- arangod/Cluster/ClusterInfo.h | 1 + arangod/Cluster/ClusterMethods.cpp | 9 +++--- arangod/Cluster/ClusterMethods.h | 5 +-- .../MMFiles/MMFilesRestReplicationHandler.cpp | 4 ++- .../RocksDBRestReplicationHandler.cpp | 4 ++- arangod/V8Server/v8-vocindex.cpp | 24 +++++++++----- js/actions/_api/collection/app.js | 13 ++++++-- .../modules/@arangodb/arango-database.js | 15 +++++++-- 11 files changed, 79 insertions(+), 33 deletions(-) diff --git a/arangod/Cluster/ClusterFeature.cpp b/arangod/Cluster/ClusterFeature.cpp index a8e46814fe..33773b7207 100644 --- a/arangod/Cluster/ClusterFeature.cpp +++ b/arangod/Cluster/ClusterFeature.cpp @@ -133,6 +133,10 @@ void ClusterFeature::collectOptions(std::shared_ptr options) { options->addOption("--cluster.system-replication-factor", "replication factor for system collections", new UInt32Parameter(&_systemReplicationFactor)); + + options->addOption("--cluster.create-waits-for-sync-replication", + "active coordinator will wait for all replicas to create collection", + new BooleanParameter(&_createWaitsForSyncReplication)); } void ClusterFeature::validateOptions(std::shared_ptr options) { diff --git a/arangod/Cluster/ClusterFeature.h b/arangod/Cluster/ClusterFeature.h index 7a50e0ddb2..3e2de79199 100644 --- a/arangod/Cluster/ClusterFeature.h +++ b/arangod/Cluster/ClusterFeature.h @@ -58,6 +58,7 @@ class ClusterFeature : public application_features::ApplicationFeature { std::string _dbserverConfig; std::string _coordinatorConfig; uint32_t _systemReplicationFactor = 2; + bool _createWaitsForSyncReplication = true; private: void reportRole(ServerState::RoleEnum); @@ -72,6 +73,7 @@ class ClusterFeature : public application_features::ApplicationFeature { }; void setUnregisterOnShutdown(bool); + bool createWaitsForSyncReplication() { return _createWaitsForSyncReplication; }; void stop() override final; diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index f2a12dae23..a2c2927346 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -1042,6 +1042,7 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, std::string const& collectionID, uint64_t numberOfShards, uint64_t replicationFactor, + bool waitForReplication, VPackSlice const& json, std::string& errorMsg, double timeout) { @@ -1103,13 +1104,6 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, bool tmpHaveError = false; for (auto const& p : VPackObjectIterator(result)) { - if (replicationFactor == 0) { - VPackSlice servers = p.value.get("servers"); - if (!servers.isArray() || servers.length() < dbServers.size()) { - return true; - } - } - if (arangodb::basics::VelocyPackHelper::getBooleanValue( p.value, "error", false)) { tmpHaveError = true; @@ -1125,13 +1119,24 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, tmpMsg += ")"; } } + *errMsg = "Error in creation of collection:" + tmpMsg + " " + + __FILE__ + std::to_string(__LINE__); + *dbServerResult = TRI_ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION; + return true; + } + + // wait that all followers have created our new collection + if (waitForReplication) { + uint64_t mutableReplicationFactor = replicationFactor; + if (mutableReplicationFactor == 0) { + mutableReplicationFactor = dbServers.size(); + } + + VPackSlice servers = p.value.get("servers"); + if (!servers.isArray() || servers.length() < mutableReplicationFactor) { + return true; + } } - } - if (tmpHaveError) { - *errMsg = "Error in creation of collection:" + tmpMsg + " " - + __FILE__ + std::to_string(__LINE__); - *dbServerResult = TRI_ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION; - return true; } *dbServerResult = setErrormsg(TRI_ERROR_NO_ERROR, *errMsg); } diff --git a/arangod/Cluster/ClusterInfo.h b/arangod/Cluster/ClusterInfo.h index 0f4fa1634b..c9e8e3f4d1 100644 --- a/arangod/Cluster/ClusterInfo.h +++ b/arangod/Cluster/ClusterInfo.h @@ -349,6 +349,7 @@ class ClusterInfo { std::string const& collectionID, uint64_t numberOfShards, uint64_t replicationFactor, + bool waitForReplication, arangodb::velocypack::Slice const& json, std::string& errorMsg, double timeout); diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index df21f5e745..61190d6f0d 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -2264,12 +2264,13 @@ std::unique_ptr ClusterMethods::createCollectionOnCoordinator(TRI_col_type_e collectionType, TRI_vocbase_t* vocbase, VPackSlice parameters, - bool ignoreDistributeShardsLikeErrors) { + bool ignoreDistributeShardsLikeErrors, + bool waitForSyncReplication) { auto col = std::make_unique(vocbase, parameters); // Collection is a temporary collection object that undergoes sanity checks etc. // It is not used anywhere and will be cleaned up after this call. // Persist collection will return the real object. - return persistCollectionInAgency(col.get(), ignoreDistributeShardsLikeErrors); + return persistCollectionInAgency(col.get(), ignoreDistributeShardsLikeErrors, waitForSyncReplication); } #endif @@ -2279,7 +2280,7 @@ ClusterMethods::createCollectionOnCoordinator(TRI_col_type_e collectionType, std::unique_ptr ClusterMethods::persistCollectionInAgency( - LogicalCollection* col, bool ignoreDistributeShardsLikeErrors) { + LogicalCollection* col, bool ignoreDistributeShardsLikeErrors, bool waitForSyncReplication) { std::string distributeShardsLike = col->distributeShardsLike(); std::vector dbServers; std::vector avoid = col->avoidServers(); @@ -2364,7 +2365,7 @@ ClusterMethods::persistCollectionInAgency( std::string errorMsg; int myerrno = ci->createCollectionCoordinator( col->dbName(), col->cid_as_string(), - col->numberOfShards(), col->replicationFactor(), velocy.slice(), errorMsg, 240.0); + col->numberOfShards(), col->replicationFactor(), waitForSyncReplication, velocy.slice(), errorMsg, 240.0); if (myerrno != TRI_ERROR_NO_ERROR) { if (errorMsg.empty()) { diff --git a/arangod/Cluster/ClusterMethods.h b/arangod/Cluster/ClusterMethods.h index 57e8095f2a..e1929bdc4e 100644 --- a/arangod/Cluster/ClusterMethods.h +++ b/arangod/Cluster/ClusterMethods.h @@ -258,7 +258,8 @@ class ClusterMethods { static std::unique_ptr createCollectionOnCoordinator( TRI_col_type_e collectionType, TRI_vocbase_t* vocbase, arangodb::velocypack::Slice parameters, - bool ignoreDistributeShardsLikeErrors = false); + bool ignoreDistributeShardsLikeErrors, + bool waitForSyncReplication); private: @@ -267,7 +268,7 @@ class ClusterMethods { //////////////////////////////////////////////////////////////////////////////// static std::unique_ptr persistCollectionInAgency( - LogicalCollection* col, bool ignoreDistributeShardsLikeErrors = false); + LogicalCollection* col, bool ignoreDistributeShardsLikeErrors, bool waitForSyncReplication); }; } // namespace arangodb diff --git a/arangod/MMFiles/MMFilesRestReplicationHandler.cpp b/arangod/MMFiles/MMFilesRestReplicationHandler.cpp index 028d0a275c..b01a938053 100644 --- a/arangod/MMFiles/MMFilesRestReplicationHandler.cpp +++ b/arangod/MMFiles/MMFilesRestReplicationHandler.cpp @@ -29,6 +29,7 @@ #include "Basics/conversions.h" #include "Basics/files.h" #include "Cluster/ClusterComm.h" +#include "Cluster/ClusterFeature.h" #include "Cluster/ClusterMethods.h" #include "Cluster/FollowerInfo.h" #include "GeneralServer/GeneralServer.h" @@ -1680,8 +1681,9 @@ int MMFilesRestReplicationHandler::processRestoreCollectionCoordinator( VPackSlice const merged = mergedBuilder.slice(); try { + bool createWaitsForSyncReplication = application_features::ApplicationServer::getFeature("Cluster")->createWaitsForSyncReplication(); auto col = ClusterMethods::createCollectionOnCoordinator( - collectionType, _vocbase, merged, ignoreDistributeShardsLikeErrors); + collectionType, _vocbase, merged, ignoreDistributeShardsLikeErrors, createWaitsForSyncReplication); TRI_ASSERT(col != nullptr); } catch (basics::Exception const& e) { // Error, report it. diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index dba3ede5fd..7cd8f528bd 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -30,6 +30,7 @@ #include "Basics/conversions.h" #include "Basics/files.h" #include "Cluster/ClusterComm.h" +#include "Cluster/ClusterFeature.h" #include "Cluster/ClusterMethods.h" #include "Cluster/FollowerInfo.h" #include "GeneralServer/GeneralServer.h" @@ -1819,8 +1820,9 @@ int RocksDBRestReplicationHandler::processRestoreCollectionCoordinator( VPackSlice const merged = mergedBuilder.slice(); try { + bool createWaitsForSyncReplication = application_features::ApplicationServer::getFeature("Cluster")->createWaitsForSyncReplication(); auto col = ClusterMethods::createCollectionOnCoordinator(collectionType, - _vocbase, merged); + _vocbase, merged, true, createWaitsForSyncReplication); TRI_ASSERT(col != nullptr); } catch (basics::Exception const& e) { // Error, report it. diff --git a/arangod/V8Server/v8-vocindex.cpp b/arangod/V8Server/v8-vocindex.cpp index 3c15367cb3..08bbc8398b 100644 --- a/arangod/V8Server/v8-vocindex.cpp +++ b/arangod/V8Server/v8-vocindex.cpp @@ -27,6 +27,7 @@ #include "Basics/VelocyPackHelper.h" #include "Basics/conversions.h" #include "Basics/tri-strings.h" +#include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ClusterMethods.h" #include "Indexes/Index.h" @@ -669,12 +670,8 @@ static void CreateVocBase(v8::FunctionCallbackInfo const& args, TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); } - // ........................................................................... - // We require exactly 1 or exactly 2 arguments -- anything else is an error - // ........................................................................... - - if (args.Length() < 1 || args.Length() > 3) { - TRI_V8_THROW_EXCEPTION_USAGE("_create(, , )"); + if (args.Length() < 1 || args.Length() > 4) { + TRI_V8_THROW_EXCEPTION_USAGE("_create(, , , )"); } if (TRI_GetOperationModeServer() == TRI_VOCBASE_MODE_NO_CREATE) { @@ -682,7 +679,7 @@ static void CreateVocBase(v8::FunctionCallbackInfo const& args, } // optional, third parameter can override collection type - if (args.Length() == 3 && args[2]->IsString()) { + if (args.Length() >= 3 && args[2]->IsString()) { std::string typeString = TRI_ObjectToString(args[2]); if (typeString == "edge") { collectionType = TRI_COL_TYPE_EDGE; @@ -691,6 +688,7 @@ static void CreateVocBase(v8::FunctionCallbackInfo const& args, } } + PREVENT_EMBEDDED_TRANSACTION(); // extract the name @@ -725,9 +723,19 @@ static void CreateVocBase(v8::FunctionCallbackInfo const& args, infoSlice = builder.slice(); if (ServerState::instance()->isCoordinator()) { + bool createWaitsForSyncReplication = application_features::ApplicationServer::getFeature("Cluster")->createWaitsForSyncReplication(); + + if (args.Length() >= 3 && args[args.Length()-1]->IsObject()) { + v8::Handle obj = args[args.Length()-1]->ToObject(); + auto v8WaitForSyncReplication = obj->Get(TRI_V8_ASCII_STRING("waitForSyncReplication")); + if (!v8WaitForSyncReplication->IsUndefined()) { + createWaitsForSyncReplication = TRI_ObjectToBoolean(v8WaitForSyncReplication); + } + } + std::unique_ptr col = ClusterMethods::createCollectionOnCoordinator(collectionType, vocbase, - infoSlice); + infoSlice, true, createWaitsForSyncReplication); TRI_V8_RETURN(WrapCollection(isolate, col.release())); } diff --git a/js/actions/_api/collection/app.js b/js/actions/_api/collection/app.js index 78806d7f6a..3cf8f386ba 100644 --- a/js/actions/_api/collection/app.js +++ b/js/actions/_api/collection/app.js @@ -206,6 +206,15 @@ function post_api_collection (req, res) { } try { + var options = {}; + if (req.parameters.hasOwnProperty('waitForSyncReplication')) { + var value = req.parameters.waitForSyncReplication.toLowerCase(); + if (value === 'true' || value === 'yes' || value === 'on' || value === 'y' || value === '1') { + options.waitForSyncReplication = true; + } else { + options.waitForSyncReplication = false; + } + } var collection; if (typeof (r.type) === 'string') { if (r.type.toLowerCase() === 'edge' || r.type === '3') { @@ -213,9 +222,9 @@ function post_api_collection (req, res) { } } if (r.type === arangodb.ArangoCollection.TYPE_EDGE) { - collection = arangodb.db._createEdgeCollection(r.name, r.parameters); + collection = arangodb.db._createEdgeCollection(r.name, r.parameters, options); } else { - collection = arangodb.db._createDocumentCollection(r.name, r.parameters); + collection = arangodb.db._createDocumentCollection(r.name, r.parameters, options); } var result = {}; diff --git a/js/client/modules/@arangodb/arango-database.js b/js/client/modules/@arangodb/arango-database.js index 38396e98cf..d8d9e4f77a 100644 --- a/js/client/modules/@arangodb/arango-database.js +++ b/js/client/modules/@arangodb/arango-database.js @@ -339,7 +339,7 @@ ArangoDatabase.prototype._collection = function (id) { // / @brief creates a new collection // ////////////////////////////////////////////////////////////////////////////// -ArangoDatabase.prototype._create = function (name, properties, type) { +ArangoDatabase.prototype._create = function (name, properties, type, options) { var body = { 'name': name, 'type': ArangoCollection.TYPE_DOCUMENT @@ -355,12 +355,23 @@ ArangoDatabase.prototype._create = function (name, properties, type) { } }); } + + let urlAddon = ''; + if (typeof options === "object" && options !== null) { + if (options.hasOwnProperty('waitForSyncReplication')) { + if (options.waitForSyncReplication) { + urlAddon = '?waitForSyncReplication=1'; + } else { + urlAddon = '?waitForSyncReplication=0'; + } + } + } if (type !== undefined) { body.type = type; } - var requestResult = this._connection.POST(this._collectionurl(), + var requestResult = this._connection.POST(this._collectionurl() + urlAddon, JSON.stringify(body)); arangosh.checkRequestResult(requestResult); From ed0e9fdc8ec86a5d522c674b815fe4f7f9207715 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 26 Apr 2017 09:57:08 +0200 Subject: [PATCH 38/58] Set waiting to false until tests are changed --- arangod/Cluster/ClusterFeature.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Cluster/ClusterFeature.h b/arangod/Cluster/ClusterFeature.h index 3e2de79199..2adf318209 100644 --- a/arangod/Cluster/ClusterFeature.h +++ b/arangod/Cluster/ClusterFeature.h @@ -58,7 +58,7 @@ class ClusterFeature : public application_features::ApplicationFeature { std::string _dbserverConfig; std::string _coordinatorConfig; uint32_t _systemReplicationFactor = 2; - bool _createWaitsForSyncReplication = true; + bool _createWaitsForSyncReplication = false; private: void reportRole(ServerState::RoleEnum); From f36a889709f39476fc4d8762a886773197a16c8d Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 09:58:19 +0200 Subject: [PATCH 39/58] try to fix shutdown issues --- arangod/MMFiles/MMFilesEngine.cpp | 2 +- arangod/MMFiles/MMFilesEngine.h | 4 +-- arangod/RocksDBEngine/RocksDBEngine.cpp | 16 +++++---- arangod/RocksDBEngine/RocksDBEngine.h | 4 +-- .../RocksDBReplicationContext.cpp | 6 ++++ .../RocksDBEngine/RocksDBReplicationContext.h | 11 +++++- .../RocksDBReplicationManager.cpp | 34 +++++++++++++++++++ .../RocksDBEngine/RocksDBReplicationManager.h | 12 +++++++ arangod/StorageEngine/StorageEngine.h | 19 +++++------ 9 files changed, 85 insertions(+), 23 deletions(-) diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp index 12c506f3a4..260642b18d 100644 --- a/arangod/MMFiles/MMFilesEngine.cpp +++ b/arangod/MMFiles/MMFilesEngine.cpp @@ -152,7 +152,7 @@ MMFilesEngine::MMFilesEngine(application_features::ApplicationServer* server) MMFilesEngine::~MMFilesEngine() {} // perform a physical deletion of the database -Result MMFilesEngine::dropDatabase(Database* database) { +Result MMFilesEngine::dropDatabase(TRI_vocbase_t* database) { // delete persistent indexes for this database MMFilesPersistentIndexFeature::dropDatabase(database->id()); diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h index 74647c0223..c2f845cf07 100644 --- a/arangod/MMFiles/MMFilesEngine.h +++ b/arangod/MMFiles/MMFilesEngine.h @@ -138,14 +138,14 @@ class MMFilesEngine final : public StorageEngine { void waitForSync(TRI_voc_tick_t tick) override; virtual TRI_vocbase_t* openDatabase(arangodb::velocypack::Slice const& parameters, bool isUpgrade, int&) override; - Database* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) override { + TRI_vocbase_t* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) override { status = TRI_ERROR_NO_ERROR; return createDatabaseMMFiles(id, args); } int writeCreateDatabaseMarker(TRI_voc_tick_t id, VPackSlice const& slice) override; void prepareDropDatabase(TRI_vocbase_t* vocbase, bool useWriteMarker, int& status) override; - Result dropDatabase(Database* database) override; + Result dropDatabase(TRI_vocbase_t* database) override; void waitUntilDeletion(TRI_voc_tick_t id, bool force, int& status) override; // wal in recovery diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 7b7f439d12..548c14ec9a 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -230,7 +230,12 @@ void RocksDBEngine::start() { } } -void RocksDBEngine::stop() {} +void RocksDBEngine::stop() { + if (!isEnabled()) { + return; + } + replicationManager()->dropAll(); +} void RocksDBEngine::unprepare() { if (!isEnabled()) { @@ -486,7 +491,7 @@ TRI_vocbase_t* RocksDBEngine::openDatabase( return openExistingDatabase(id, name, true, isUpgrade); } -RocksDBEngine::Database* RocksDBEngine::createDatabase( +TRI_vocbase_t* RocksDBEngine::createDatabase( TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) { status = TRI_ERROR_NO_ERROR; auto vocbase = std::make_unique(TRI_VOCBASE_TYPE_NORMAL, id, @@ -519,10 +524,6 @@ int RocksDBEngine::writeCreateCollectionMarker(TRI_voc_tick_t databaseId, void RocksDBEngine::prepareDropDatabase(TRI_vocbase_t* vocbase, bool useWriteMarker, int& status) { - // probably not required - // THROW_ARANGO_NOT_YET_IMPLEMENTED(); - - // status = saveDatabaseParameters(vocbase->id(), vocbase->name(), true); VPackBuilder builder; builder.openObject(); builder.add("id", VPackValue(std::to_string(vocbase->id()))); @@ -533,7 +534,8 @@ void RocksDBEngine::prepareDropDatabase(TRI_vocbase_t* vocbase, status = writeCreateDatabaseMarker(vocbase->id(), builder.slice()); } -Result RocksDBEngine::dropDatabase(Database* database) { +Result RocksDBEngine::dropDatabase(TRI_vocbase_t* database) { + replicationManager()->drop(database); return dropDatabase(database->id()); } diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index 6019a99940..d77ba30130 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -123,14 +123,14 @@ class RocksDBEngine final : public StorageEngine { virtual TRI_vocbase_t* openDatabase( arangodb::velocypack::Slice const& parameters, bool isUpgrade, int&) override; - Database* createDatabase(TRI_voc_tick_t id, + TRI_vocbase_t* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) override; int writeCreateDatabaseMarker(TRI_voc_tick_t id, VPackSlice const& slice) override; void prepareDropDatabase(TRI_vocbase_t* vocbase, bool useWriteMarker, int& status) override; - Result dropDatabase(Database* database) override; + Result dropDatabase(TRI_vocbase_t* database) override; void waitUntilDeletion(TRI_voc_tick_t id, bool force, int& status) override; // wal in recovery diff --git a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp index 784281861e..5b640f9f0f 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp @@ -33,6 +33,7 @@ #include "Transaction/Helpers.h" #include "Transaction/StandaloneContext.h" #include "Transaction/UserTransaction.h" +#include "Utils/DatabaseGuard.h" #include "VocBase/replication-common.h" #include "VocBase/ticks.h" @@ -55,6 +56,7 @@ RocksDBReplicationContext::RocksDBReplicationContext() _mdr(), _customTypeHandler(), _vpackOptions(Options::Defaults), + _lastChunkOffset(0), _expires(TRI_microtime() + DefaultTTL), _isDeleted(false), _isUsed(true), @@ -390,10 +392,13 @@ void RocksDBReplicationContext::releaseDumpingResources() { _iter.reset(); } _collection = nullptr; + _guard.reset(); } std::unique_ptr RocksDBReplicationContext::createTransaction(TRI_vocbase_t* vocbase) { + _guard.reset(new DatabaseGuard(vocbase)); + double lockTimeout = transaction::Methods::DefaultLockTimeout; std::shared_ptr ctx = transaction::StandaloneContext::Create(vocbase); @@ -401,6 +406,7 @@ RocksDBReplicationContext::createTransaction(TRI_vocbase_t* vocbase) { ctx, {}, {}, {}, lockTimeout, false, true)); Result res = trx->begin(); if (!res.ok()) { + _guard.reset(); THROW_ARANGO_EXCEPTION(res); } _customTypeHandler = ctx->orderCustomTypeHandler(); diff --git a/arangod/RocksDBEngine/RocksDBReplicationContext.h b/arangod/RocksDBEngine/RocksDBReplicationContext.h index 95135f37f5..e8a521188b 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationContext.h +++ b/arangod/RocksDBEngine/RocksDBReplicationContext.h @@ -36,6 +36,7 @@ #include namespace arangodb { +class DatabaseGuard; class RocksDBReplicationContext { public: @@ -53,6 +54,13 @@ class RocksDBReplicationContext { TRI_voc_tick_t id() const; uint64_t lastTick() const; uint64_t count() const; + + TRI_vocbase_t* vocbase() const { + if (_trx == nullptr) { + return nullptr; + } + return _trx->vocbase(); + } // creates new transaction/snapshot void bind(TRI_vocbase_t*); @@ -113,7 +121,8 @@ class RocksDBReplicationContext { ManagedDocumentResult _mdr; std::shared_ptr _customTypeHandler; arangodb::velocypack::Options _vpackOptions; - uint64_t _lastChunkOffset = 0; + uint64_t _lastChunkOffset; + std::unique_ptr _guard; double _expires; bool _isDeleted; diff --git a/arangod/RocksDBEngine/RocksDBReplicationManager.cpp b/arangod/RocksDBEngine/RocksDBReplicationManager.cpp index 49dca5036c..881e6671e7 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationManager.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationManager.cpp @@ -238,6 +238,40 @@ bool RocksDBReplicationManager::containsUsedContext() { return false; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief drop contexts by database (at least mark them as deleted) +//////////////////////////////////////////////////////////////////////////////// + +void RocksDBReplicationManager::drop(TRI_vocbase_t* vocbase) { + { + MUTEX_LOCKER(mutexLocker, _lock); + + for (auto& context : _contexts) { + if (context.second->vocbase() == vocbase) { + context.second->deleted(); + } + } + } + + garbageCollect(true); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief drop all contexts (at least mark them as deleted) +//////////////////////////////////////////////////////////////////////////////// + +void RocksDBReplicationManager::dropAll() { + { + MUTEX_LOCKER(mutexLocker, _lock); + + for (auto& context : _contexts) { + context.second->deleted(); + } + } + + garbageCollect(true); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief run a garbage collection on the contexts //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RocksDBEngine/RocksDBReplicationManager.h b/arangod/RocksDBEngine/RocksDBReplicationManager.h index 3d48e7de97..0019066b01 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationManager.h +++ b/arangod/RocksDBEngine/RocksDBReplicationManager.h @@ -92,6 +92,18 @@ class RocksDBReplicationManager { bool containsUsedContext(); + ////////////////////////////////////////////////////////////////////////////// + /// @brief drop contexts by database (at least mark them as deleted) + ////////////////////////////////////////////////////////////////////////////// + + void drop(TRI_vocbase_t*); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief drop all contexts (at least mark them as deleted) + ////////////////////////////////////////////////////////////////////////////// + + void dropAll(); + ////////////////////////////////////////////////////////////////////////////// /// @brief run a garbage collection on the contexts ////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h index 495cbcf90e..e35737bff1 100644 --- a/arangod/StorageEngine/StorageEngine.h +++ b/arangod/StorageEngine/StorageEngine.h @@ -146,7 +146,6 @@ class StorageEngine : public application_features::ApplicationFeature { // TODO add pre / post conditions for functions - using Database = TRI_vocbase_t; using CollectionView = LogicalCollection; virtual void waitForSync(TRI_voc_tick_t tick) = 0; @@ -154,10 +153,10 @@ class StorageEngine : public application_features::ApplicationFeature { //// operations on databasea /// @brief opens a database - virtual Database* openDatabase(arangodb::velocypack::Slice const& args, bool isUpgrade, int& status) = 0; - Database* openDatabase(arangodb::velocypack::Slice const& args, bool isUpgrade){ + virtual TRI_vocbase_t* openDatabase(arangodb::velocypack::Slice const& args, bool isUpgrade, int& status) = 0; + TRI_vocbase_t* openDatabase(arangodb::velocypack::Slice const& args, bool isUpgrade){ int status; - Database* rv = openDatabase(args, isUpgrade, status); + TRI_vocbase_t* rv = openDatabase(args, isUpgrade, status); TRI_ASSERT(status == TRI_ERROR_NO_ERROR); TRI_ASSERT(rv != nullptr); return rv; @@ -172,16 +171,16 @@ class StorageEngine : public application_features::ApplicationFeature { // the WAL entry for the database creation will be written *after* the call // to "createDatabase" returns // no way to acquire id within this function?! - virtual Database* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) = 0; - Database* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args ){ + virtual TRI_vocbase_t* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) = 0; + TRI_vocbase_t* createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args ){ int status; - Database* rv = createDatabase(id, args, status); + TRI_vocbase_t* rv = createDatabase(id, args, status); TRI_ASSERT(status == TRI_ERROR_NO_ERROR); TRI_ASSERT(rv != nullptr); return rv; } - // @brief wirte create marker for database + // @brief write create marker for database virtual int writeCreateDatabaseMarker(TRI_voc_tick_t id, VPackSlice const& slice) = 0; // asks the storage engine to drop the specified database and persist the @@ -194,14 +193,14 @@ class StorageEngine : public application_features::ApplicationFeature { // // is done under a lock in database feature virtual void prepareDropDatabase(TRI_vocbase_t* vocbase, bool useWriteMarker, int& status) = 0; - void prepareDropDatabase(Database* db, bool useWriteMarker){ + void prepareDropDatabase(TRI_vocbase_t* db, bool useWriteMarker){ int status = 0; prepareDropDatabase(db, useWriteMarker, status); TRI_ASSERT(status == TRI_ERROR_NO_ERROR); }; // perform a physical deletion of the database - virtual Result dropDatabase(Database*) = 0; + virtual Result dropDatabase(TRI_vocbase_t*) = 0; /// @brief wait until a database directory disappears - not under lock in databaseFreature virtual void waitUntilDeletion(TRI_voc_tick_t id, bool force, int& status) = 0; From 6b8c818e2d12473d47016a0acc23d9d25fec5bc5 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 09:59:19 +0200 Subject: [PATCH 40/58] remove unused variable --- arangod/Replication/InitialSyncer.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/arangod/Replication/InitialSyncer.cpp b/arangod/Replication/InitialSyncer.cpp index 7f61c18fcd..6721c216ac 100644 --- a/arangod/Replication/InitialSyncer.cpp +++ b/arangod/Replication/InitialSyncer.cpp @@ -1072,8 +1072,6 @@ int InitialSyncer::handleSyncKeysRocksDB(arangodb::LogicalCollection* col, sendExtendBatch(); sendExtendBarrier(); - std::vector toFetch; - TRI_voc_tick_t const chunkSize = 5000; std::string const baseUrl = BaseUrl + "/keys"; From decb36d8bc24a9fde6e17cf1cfc3f1131d7d1fb5 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 26 Apr 2017 10:00:30 +0200 Subject: [PATCH 41/58] force windows to run 64 bit compiler & linkers --- Installation/Jenkins/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index b37f90e7a9..6f517eda04 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -254,6 +254,7 @@ while [ $# -gt 0 ]; do MAKE="cmake --build . --config ${BUILD_CONFIG}" PACKAGE_MAKE="cmake --build . --config ${BUILD_CONFIG} --target" CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DV8_TARGET_ARCHS=Release" + export _IsNativeEnvironment=true ;; --symsrv) From bf082b8e2cdb8ff67111ca43369be5b1c2472afa Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:01:41 +0200 Subject: [PATCH 42/58] Port 3.1 fixes to devel, .properties() method. --- js/actions/_api/collection/app.js | 1 - 1 file changed, 1 deletion(-) diff --git a/js/actions/_api/collection/app.js b/js/actions/_api/collection/app.js index 78806d7f6a..a7126647e1 100644 --- a/js/actions/_api/collection/app.js +++ b/js/actions/_api/collection/app.js @@ -65,7 +65,6 @@ function collectionRepresentation(collection, showProperties, showCount, showFig if (cluster.isCoordinator()) { result.avoidServers = properties.avoidServers; - result.distributeShardsLike = properties.distributeShardsLike; result.numberOfShards = properties.numberOfShards; result.replicationFactor = properties.replicationFactor; result.avoidServers = properties.avoidServers; From bf2e5f30cac45d2122b628be2fefc4de8ee886f4 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:02:14 +0200 Subject: [PATCH 43/58] Port 3.1 fixes to devel, shortName translation. --- js/actions/api-cluster.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/js/actions/api-cluster.js b/js/actions/api-cluster.js index dd48354702..8770f36ea5 100644 --- a/js/actions/api-cluster.js +++ b/js/actions/api-cluster.js @@ -970,12 +970,25 @@ actions.defineHttp({ "body must be an object with a string attribute 'server'"); return; } + + // First translate the server name from short name to long name: + var server = body.server; + var servers = global.ArangoClusterInfo.getDBServers(); + for (let i = 0; i < servers.length; i++) { + if (servers[i].serverId !== server) { + if (servers[i].serverName === server) { + server = servers[i].serverId; + break; + } + } + } + var ok = true; var id; try { id = ArangoClusterInfo.uniqid(); var todo = { 'type': 'cleanOutServer', - 'server': body.server, + 'server': server, 'jobId': id, 'timeCreated': (new Date()).toISOString(), 'creator': ArangoServerState.id() }; From d3517eb491fd29ce3c86229e6550a69c3a0ece65 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 10:02:20 +0200 Subject: [PATCH 44/58] Removed unnecessary friend declaration --- arangod/Aql/ShortestPathBlock.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arangod/Aql/ShortestPathBlock.h b/arangod/Aql/ShortestPathBlock.h index 4f2fcfe963..0f59d959a2 100644 --- a/arangod/Aql/ShortestPathBlock.h +++ b/arangod/Aql/ShortestPathBlock.h @@ -32,7 +32,6 @@ namespace arangodb { class ManagedDocumentResult; namespace graph { -class ConstantWeightShortestPathFinder; class ShortestPathFinder; class ShortestPathResult; } @@ -49,9 +48,6 @@ class ShortestPathBlock : public ExecutionBlock { friend struct EdgeWeightExpanderLocal; friend struct EdgeWeightExpanderCluster; - // TODO ONLY TEMPORARY - friend class graph::ConstantWeightShortestPathFinder; - public: ShortestPathBlock(ExecutionEngine* engine, ShortestPathNode const* ep); From db3195e73661f98cf607c4fa9d5868f7ba4038aa Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:02:41 +0200 Subject: [PATCH 45/58] Port 3.1 fixes to devel, typo. --- js/client/modules/@arangodb/arango-collection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/client/modules/@arangodb/arango-collection.js b/js/client/modules/@arangodb/arango-collection.js index 462b004685..469b530b8c 100644 --- a/js/client/modules/@arangodb/arango-collection.js +++ b/js/client/modules/@arangodb/arango-collection.js @@ -345,7 +345,7 @@ ArangoCollection.prototype.properties = function (properties) { requestResult = this._database._connection.GET(this._baseurl('properties')); arangosh.checkRequestResult(requestResult); - }else { + } else { var body = {}; for (a in attributes) { From 63d51b9ae4fabfd2c562ed344d25ffe2ce82145d Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:03:20 +0200 Subject: [PATCH 46/58] Port 3.1 fixes to devel, support setting the agencyPrefix again. --- arangod/Agency/AgencyFeature.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arangod/Agency/AgencyFeature.cpp b/arangod/Agency/AgencyFeature.cpp index 2ce3df3402..c9efd0dba6 100644 --- a/arangod/Agency/AgencyFeature.cpp +++ b/arangod/Agency/AgencyFeature.cpp @@ -208,6 +208,13 @@ void AgencyFeature::start() { return; } + // Find the agency prefix: + auto feature = ApplicationServer::getFeature("Cluster"); + arangodb::consensus::Supervision::setAgencyPrefix( + std::string("/") + feature->agencyPrefix()); + arangodb::consensus::Job::agencyPrefix + = std::string("/") + feature->agencyPrefix(); + // TODO: Port this to new options handling std::string endpoint; From 42577df48df54cc5f3698e38482c7a83c91f4dc3 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 26 Apr 2017 10:07:36 +0200 Subject: [PATCH 47/58] remove useless variable --- arangod/Cluster/ClusterInfo.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index a2c2927346..a170b323e0 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -1101,12 +1101,10 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, [=](VPackSlice const& result) { if (result.isObject() && result.length() == (size_t)numberOfShards) { std::string tmpMsg = ""; - bool tmpHaveError = false; for (auto const& p : VPackObjectIterator(result)) { if (arangodb::basics::VelocyPackHelper::getBooleanValue( p.value, "error", false)) { - tmpHaveError = true; tmpMsg += " shardID:" + p.key.copyString() + ":"; tmpMsg += arangodb::basics::VelocyPackHelper::getStringValue( p.value, "errorMessage", ""); From e42144f93d0c73352ccef97cba64cc3633fec50a Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 10:09:35 +0200 Subject: [PATCH 48/58] disable currently unused variable --- arangod/RocksDBEngine/RocksDBReplicationTailing.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBReplicationTailing.cpp b/arangod/RocksDBEngine/RocksDBReplicationTailing.cpp index 4eaa932078..e6ae4d581e 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationTailing.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationTailing.cpp @@ -43,7 +43,7 @@ class WBReader : public rocksdb::WriteBatch::Handler { explicit WBReader(TRI_vocbase_t* vocbase, uint64_t from, size_t& limit, bool includeSystem, VPackBuilder& builder) : _vocbase(vocbase), - _from(from), + /* _from(from), */ _limit(limit), _includeSystem(includeSystem), _builder(builder) {} @@ -170,7 +170,7 @@ class WBReader : public rocksdb::WriteBatch::Handler { private: TRI_vocbase_t* _vocbase; - uint64_t _from; + /* uint64_t _from; */ size_t& _limit; bool _includeSystem; VPackBuilder& _builder; From 17a88ddbf8a143aa058875c396b2494ccb2b09c4 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:09:47 +0200 Subject: [PATCH 49/58] Port 3.1 fixes to devel, add an access method. --- arangod/Cluster/ClusterFeature.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arangod/Cluster/ClusterFeature.h b/arangod/Cluster/ClusterFeature.h index 7a50e0ddb2..fabec8ee02 100644 --- a/arangod/Cluster/ClusterFeature.h +++ b/arangod/Cluster/ClusterFeature.h @@ -45,6 +45,10 @@ class ClusterFeature : public application_features::ApplicationFeature { void start() override final; void unprepare() override final; + std::string agencyPrefix() { + return _agencyPrefix; + } + private: std::vector _agencyEndpoints; std::string _agencyPrefix; From 4c58817cc77bb24f1384aae5fa15d9ebcac84eed Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 10:10:05 +0200 Subject: [PATCH 50/58] Further unnecessary friends and member variables removed --- arangod/Aql/ShortestPathBlock.cpp | 1 - arangod/Aql/ShortestPathBlock.h | 4 ---- 2 files changed, 5 deletions(-) diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index d3925724a6..b0109b8a0c 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -59,7 +59,6 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _usedConstant(false), _engines(nullptr) { _opts = static_cast(ep->options()); - _mmdr.reset(new ManagedDocumentResult); if (!ep->usesStartInVariable()) { _startVertexId = ep->getStartVertex(); diff --git a/arangod/Aql/ShortestPathBlock.h b/arangod/Aql/ShortestPathBlock.h index 0f59d959a2..4346f9ef77 100644 --- a/arangod/Aql/ShortestPathBlock.h +++ b/arangod/Aql/ShortestPathBlock.h @@ -45,8 +45,6 @@ namespace aql { class ShortestPathNode; class ShortestPathBlock : public ExecutionBlock { - friend struct EdgeWeightExpanderLocal; - friend struct EdgeWeightExpanderCluster; public: ShortestPathBlock(ExecutionEngine* engine, ShortestPathNode const* ep); @@ -96,8 +94,6 @@ class ShortestPathBlock : public ExecutionBlock { /// @brief Register for the edge output RegisterId _edgeReg; - std::unique_ptr _mmdr; - /// @brief options to compute the shortest path graph::ShortestPathOptions* _opts; From b6df2002e6d252a16a39b790dcce072967fa7e82 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:13:35 +0200 Subject: [PATCH 51/58] Port 3.1 fixes to devel, update tests. --- .../cluster-sync-test-noncluster-spec.js | 64 +++++++++++++++++-- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js b/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js index dab1433a8d..6c54731718 100644 --- a/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js +++ b/js/server/tests/cluster-sync/cluster-sync-test-noncluster-spec.js @@ -805,6 +805,58 @@ describe('Cluster sync', function() { db._useDatabase('test'); expect(db._collection('s100001').isLeader()).to.equal(true); }); + it('should kill any unplanned server from current', function() { + let collection = db._create('s100001'); + collection.assumeLeadership(); + collection.addFollower('test'); + collection.addFollower('test2'); + let plan = { + test: { + "100001": { + "deleted": false, + "doCompact": true, + "id": "100001", + "indexBuckets": 8, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "isSystem": false, + "isVolatile": false, + "journalSize": 1048576, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + }, + "name": "testi", + "numberOfShards": 1, + "replicationFactor": 2, + "shardKeys": [ + "_key" + ], + "shards": { + "s100001": [ + "repltest", + "test2", + ] + }, + "status": 2, + "type": 2, + "waitForSync": false + } + } + }; + cluster.executePlanForCollections(plan); + db._useDatabase('test'); + expect(collection.getFollowers()).to.deep.equal(['test2']); + }); }); describe('Update current database', function() { beforeEach(function() { @@ -972,7 +1024,7 @@ describe('Cluster sync', function() { let collection = db._create('testi', props); let current = { }; - let result = cluster.updateCurrentForCollections({}, current); + let result = cluster.updateCurrentForCollections({}, {}, current); expect(Object.keys(result)).to.have.lengthOf(0); }); it('should not delete any collections for which we are not a leader locally', function() { @@ -983,7 +1035,7 @@ describe('Cluster sync', function() { }, } }; - let result = cluster.updateCurrentForCollections({}, current); + let result = cluster.updateCurrentForCollections({}, {}, current); expect(Object.keys(result)).to.have.lengthOf(0); }); it('should resign leadership for which we are no more leader locally', function() { @@ -996,7 +1048,7 @@ describe('Cluster sync', function() { }, } }; - let result = cluster.updateCurrentForCollections({}, current); + let result = cluster.updateCurrentForCollections({}, {}, current); expect(result).to.be.an('object'); expect(Object.keys(result)).to.have.lengthOf(1); expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi/servers') @@ -1016,7 +1068,7 @@ describe('Cluster sync', function() { }, } }; - let result = cluster.updateCurrentForCollections({}, current); + let result = cluster.updateCurrentForCollections({}, {}, current); expect(result).to.be.an('object'); expect(Object.keys(result)).to.have.lengthOf(1); expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') @@ -1033,7 +1085,7 @@ describe('Cluster sync', function() { }, } }; - let result = cluster.updateCurrentForCollections({}, current); + let result = cluster.updateCurrentForCollections({}, {}, current); expect(result).to.be.an('object'); expect(Object.keys(result)).to.have.lengthOf(1); expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') @@ -1051,7 +1103,7 @@ describe('Cluster sync', function() { }, } }; - let result = cluster.updateCurrentForCollections({}, current); + let result = cluster.updateCurrentForCollections({}, {}, current); expect(result).to.be.an('object'); expect(Object.keys(result)).to.have.lengthOf(1); expect(result).to.have.property('/arango/Current/Collections/testung/888111/testi') From 96615132e45737dbc5a74e78bfd9aff838be6fc6 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 26 Apr 2017 10:19:05 +0200 Subject: [PATCH 52/58] Port 3.1 fixes to devel, new test. --- .../shell-distributeShardsLike-cluster.js | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 js/server/tests/shell/shell-distributeShardsLike-cluster.js diff --git a/js/server/tests/shell/shell-distributeShardsLike-cluster.js b/js/server/tests/shell/shell-distributeShardsLike-cluster.js new file mode 100644 index 0000000000..36ea0f3059 --- /dev/null +++ b/js/server/tests/shell/shell-distributeShardsLike-cluster.js @@ -0,0 +1,95 @@ +/*jshint globalstrict:false, strict:false */ +/*global fail, assertEqual */ + +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2017 ArangoDB GmbH, Cologne, Germany +/// Copyright 2010-2017 triagens GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +//////////////////////////////////////////////////////////////////////////////// + +var jsunity = require("jsunity"); +var arangodb = require("@arangodb"); +var errors = require("internal").errors; + +var db = arangodb.db; + +function DistributeShardsLikeSuite() { + 'use strict'; + var cn1 = "UnitTestsDistributeShardsLike1"; + var cn2 = "UnitTestsDistributeShardsLike2"; + var cn3 = "UnitTestsDistributeShardsLike3"; + + return { + setUp: function() { + db._drop(cn2); + db._drop(cn3); + db._drop(cn1); + }, + + tearDown: function() { + db._drop(cn2); + db._drop(cn3); + db._drop(cn1); + }, + + testPointToEmpty: function() { + try { + db._create(cn1, {numberOfShards: 2, distributeShardsLike: cn2}); + fail(); + } + catch (err) { + require("internal").print("FUXX:", JSON.stringify(err)); + assertEqual(errors.ERROR_CLUSTER_UNKNOWN_DISTRIBUTESHARDSLIKE.code, + err.errorNum); + } + }, + + testAvoidChain: function() { + db._create(cn1, {numberOfShards: 2}); + db._create(cn2, {numberOfShards: 2, distributeShardsLike: cn1}); + try { + db._create(cn3, {numberOfShards: 2, distributeShardsLike: cn2}); + fail(); + } + catch (err) { + assertEqual(errors.ERROR_CLUSTER_CHAIN_OF_DISTRIBUTESHARDSLIKE.code, + err.errorNum); + } + }, + + testPreventDrop: function() { + db._create(cn1, {numberOfShards: 2}); + db._create(cn2, {numberOfShards: 2, distributeShardsLike: cn1}); + db._create(cn3, {numberOfShards: 2, distributeShardsLike: cn1}); + try { + db._drop(cn1); + fail(); + } + catch (err) { + assertEqual(errors.ERROR_CLUSTER_MUST_NOT_DROP_COLL_OTHER_DISTRIBUTESHARDSLIKE.code, + err.errorNum); + } + } + }; +} + +jsunity.run(DistributeShardsLikeSuite); +return jsunity.done(); + From 7aceac2f56615cecae822fda1ae64e913f2e505c Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 26 Apr 2017 10:20:21 +0200 Subject: [PATCH 53/58] ClusterComm cherry pick from 3.1 --- arangod/Cluster/ClusterComm.cpp | 29 +++++++++++++++++------------ arangod/Cluster/ClusterComm.h | 6 ++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/arangod/Cluster/ClusterComm.cpp b/arangod/Cluster/ClusterComm.cpp index 6d1f35b0d2..66a67d02ae 100644 --- a/arangod/Cluster/ClusterComm.cpp +++ b/arangod/Cluster/ClusterComm.cpp @@ -1123,6 +1123,19 @@ void ClusterComm::addAuthorization(std::unordered_map* } } +std::vector ClusterComm::activeServerTickets(std::vector const& servers) { + std::vector tickets; + CONDITION_LOCKER(locker, somethingReceived); + for (auto const& it: responses) { + for (auto const& server: servers) { + if (it.second.result && it.second.result->serverID == server) { + tickets.push_back(it.first); + } + } + } + return tickets; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief ClusterComm main loop //////////////////////////////////////////////////////////////////////////////// @@ -1130,18 +1143,10 @@ void ClusterComm::addAuthorization(std::unordered_map* void ClusterCommThread::abortRequestsToFailedServers() { ClusterInfo* ci = ClusterInfo::instance(); auto failedServers = ci->getFailedServers(); - std::vector failedServerEndpoints; - failedServerEndpoints.reserve(failedServers.size()); - - for (auto const& failedServer: failedServers) { - failedServerEndpoints.push_back(_cc->createCommunicatorDestination(ci->getServerEndpoint(failedServer), "/").url()); - } - - for (auto const& request: _cc->communicator()->requestsInProgress()) { - for (auto const& failedServerEndpoint: failedServerEndpoints) { - if (request->_destination.url().substr(0, failedServerEndpoint.length()) == failedServerEndpoint) { - _cc->communicator()->abortRequest(request->_ticketId); - } + if (failedServers.size() > 0) { + auto ticketIds = _cc->activeServerTickets(failedServers); + for (auto const& ticketId: ticketIds) { + _cc->communicator()->abortRequest(ticketId); } } } diff --git a/arangod/Cluster/ClusterComm.h b/arangod/Cluster/ClusterComm.h index 1f5d42967e..193b8bd418 100644 --- a/arangod/Cluster/ClusterComm.h +++ b/arangod/Cluster/ClusterComm.h @@ -616,6 +616,12 @@ class ClusterComm { void cleanupAllQueues(); + ////////////////////////////////////////////////////////////////////////////// + /// @brief activeServerTickets for a list of servers + ////////////////////////////////////////////////////////////////////////////// + + std::vector activeServerTickets(std::vector const& servers); + ////////////////////////////////////////////////////////////////////////////// /// @brief our background communications thread ////////////////////////////////////////////////////////////////////////////// From 9e1eba3bde530a1ba7596548612bd30f318e00e8 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 10:21:18 +0200 Subject: [PATCH 54/58] ShortestPathBlock does not use EdgeCollectionInfo anymore. Replaced by EdgeCursors --- arangod/Aql/ShortestPathBlock.cpp | 5 ++--- arangod/Aql/ShortestPathBlock.h | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index b0109b8a0c..a555743a66 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -30,7 +30,6 @@ #include "Graph/ShortestPathResult.h" #include "Transaction/Methods.h" #include "Utils/OperationCursor.h" -#include "VocBase/EdgeCollectionInfo.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ManagedDocumentResult.h" #include "VocBase/ticks.h" @@ -48,7 +47,7 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _vertexReg(ExecutionNode::MaxRegisterId), _edgeVar(nullptr), _edgeReg(ExecutionNode::MaxRegisterId), - _opts(nullptr), + _opts(static_cast(ep->options()), _posInPath(0), _pathLength(0), _path(nullptr), @@ -58,7 +57,7 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _useTargetRegister(false), _usedConstant(false), _engines(nullptr) { - _opts = static_cast(ep->options()); + TRI_ASSERT(_opts != nullptr); if (!ep->usesStartInVariable()) { _startVertexId = ep->getStartVertex(); diff --git a/arangod/Aql/ShortestPathBlock.h b/arangod/Aql/ShortestPathBlock.h index 4346f9ef77..c755303717 100644 --- a/arangod/Aql/ShortestPathBlock.h +++ b/arangod/Aql/ShortestPathBlock.h @@ -36,10 +36,6 @@ class ShortestPathFinder; class ShortestPathResult; } -namespace traverser { -class EdgeCollectionInfo; -} - namespace aql { class ShortestPathNode; From 8f2fbd6b3cc587aba3585beddcffa95e654bba40 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 26 Apr 2017 10:27:52 +0200 Subject: [PATCH 55/58] Fix compile bug --- arangod/Cluster/ClusterComm.cpp | 8 ++++---- arangod/Cluster/ClusterComm.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arangod/Cluster/ClusterComm.cpp b/arangod/Cluster/ClusterComm.cpp index 66a67d02ae..b97477ff9f 100644 --- a/arangod/Cluster/ClusterComm.cpp +++ b/arangod/Cluster/ClusterComm.cpp @@ -577,7 +577,7 @@ bool ClusterComm::match(ClientTransactionID const& clientTransactionID, /// from deleting `result` and `answer`. //////////////////////////////////////////////////////////////////////////////// -ClusterCommResult const ClusterComm::enquire(Ticket const ticketId) { +ClusterCommResult const ClusterComm::enquire(communicator::Ticket const ticketId) { ResponseIterator i; AsyncResponse response; @@ -614,7 +614,7 @@ ClusterCommResult const ClusterComm::enquire(Ticket const ticketId) { ClusterCommResult const ClusterComm::wait( ClientTransactionID const& clientTransactionID, - CoordTransactionID const coordTransactionID, Ticket const ticketId, + CoordTransactionID const coordTransactionID, communicator::Ticket const ticketId, ShardID const& shardID, ClusterCommTimeout timeout) { ResponseIterator i; @@ -1123,8 +1123,8 @@ void ClusterComm::addAuthorization(std::unordered_map* } } -std::vector ClusterComm::activeServerTickets(std::vector const& servers) { - std::vector tickets; +std::vector ClusterComm::activeServerTickets(std::vector const& servers) { + std::vector tickets; CONDITION_LOCKER(locker, somethingReceived); for (auto const& it: responses) { for (auto const& server: servers) { diff --git a/arangod/Cluster/ClusterComm.h b/arangod/Cluster/ClusterComm.h index 193b8bd418..4ee6f36ceb 100644 --- a/arangod/Cluster/ClusterComm.h +++ b/arangod/Cluster/ClusterComm.h @@ -620,7 +620,7 @@ class ClusterComm { /// @brief activeServerTickets for a list of servers ////////////////////////////////////////////////////////////////////////////// - std::vector activeServerTickets(std::vector const& servers); + std::vector activeServerTickets(std::vector const& servers); ////////////////////////////////////////////////////////////////////////////// /// @brief our background communications thread From aed90940d0479726d1617cc38c5c4ee418d595ea Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 10:32:40 +0200 Subject: [PATCH 56/58] Fixed compile bugin shortest path --- arangod/Aql/ShortestPathBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index a555743a66..ac9fe84f7e 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -47,7 +47,7 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _vertexReg(ExecutionNode::MaxRegisterId), _edgeVar(nullptr), _edgeReg(ExecutionNode::MaxRegisterId), - _opts(static_cast(ep->options()), + _opts(static_cast(ep->options())), _posInPath(0), _pathLength(0), _path(nullptr), From e891c66c883febf1a6b72b287375794583c62ce7 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 26 Apr 2017 10:39:10 +0200 Subject: [PATCH 57/58] Fixed missing includes --- arangod/Aql/ShortestPathBlock.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index ac9fe84f7e..e18e3e6e1f 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -28,6 +28,8 @@ #include "Aql/Query.h" #include "Cluster/ClusterComm.h" #include "Graph/ShortestPathResult.h" +#include "Graph/AttributeWeightShortestPathFinder.h" +#include "Graph/ConstantWeightShortestPathFinder.h" #include "Transaction/Methods.h" #include "Utils/OperationCursor.h" #include "VocBase/LogicalCollection.h" From b226dfdc9a0b105cd62d2b3f730e0e6bb88bfc04 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Wed, 26 Apr 2017 10:40:04 +0200 Subject: [PATCH 58/58] add missing include files --- arangod/Aql/ShortestPathBlock.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index ac9fe84f7e..d866c2f061 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -27,6 +27,9 @@ #include "Aql/ExecutionPlan.h" #include "Aql/Query.h" #include "Cluster/ClusterComm.h" +#include "Graph/AttributeWeightShortestPathFinder.h" +#include "Graph/ConstantWeightShortestPathFinder.h" +#include "Graph/ShortestPathFinder.h" #include "Graph/ShortestPathResult.h" #include "Transaction/Methods.h" #include "Utils/OperationCursor.h"