diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 4dd49908eb..94dbc55260 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -653,13 +653,21 @@ fi PARTIAL_STATE=$? set -e +if test "${isCygwin}" == 1 -a "${PARTIAL_STATE}" == 0; then + # windows fails to partialy re-configure - so do a complete configure run. + if test -f CMakeFiles/generate.stamp -a CMakeFiles/generate.stamp -ot "${SOURCE_DIR}/CMakeList.txt"; then + echo "CMakeList older - Forcing complete configure run!" + PARTIAL_STATE=1 + fi +fi + if test "${PARTIAL_STATE}" == 0; then rm -rf CMakeFiles CMakeCache.txt CMakeCPackOptions.cmake cmake_install.cmake CPackConfig.cmake CPackSourceConfig.cmake CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" LDFLAGS="${LDFLAGS}" LIBS="${LIBS}" \ cmake ${SOURCE_DIR} ${CONFIGURE_OPTIONS} -G "${GENERATOR}" || exit 1 fi -if [ -n "$CPACK" -a -n "${TARGET_DIR}" -a -z "${MSVC}" ]; then +if [ -n "$CPACK" ] && [ -n "${TARGET_DIR}" ] && [ -z "${MSVC}" ]; then if ! grep -q CMAKE_STRIP CMakeCache.txt; then echo "cmake failed to detect strip; refusing to build unstripped packages!" exit 1 diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp index 04dc494c8d..5d806f1660 100644 --- a/arangod/MMFiles/MMFilesEngine.cpp +++ b/arangod/MMFiles/MMFilesEngine.cpp @@ -22,13 +22,13 @@ /// @author Jan Christoph Uhde //////////////////////////////////////////////////////////////////////////////// -#include "MMFilesEngine.h" #include "Basics/FileUtils.h" #include "Basics/MutexLocker.h" #include "Basics/ReadLocker.h" #include "Basics/StringUtils.h" #include "Basics/VelocyPackHelper.h" #include "Basics/WriteLocker.h" +#include "Basics/build.h" #include "Basics/encoding.h" #include "Basics/files.h" #include "MMFiles/MMFilesAqlFunctions.h" @@ -37,6 +37,7 @@ #include "MMFiles/MMFilesCompactorThread.h" #include "MMFiles/MMFilesDatafile.h" #include "MMFiles/MMFilesDatafileHelper.h" +#include "MMFiles/MMFilesEngine.h" #include "MMFiles/MMFilesIndexFactory.h" #include "MMFiles/MMFilesInitialSync.h" #include "MMFiles/MMFilesLogfileManager.h" @@ -51,9 +52,11 @@ #include "MMFiles/MMFilesV8Functions.h" #include "MMFiles/MMFilesView.h" #include "MMFiles/MMFilesWalRecoveryFeature.h" +#include "MMFiles/mmfiles-replication-dump.h" #include "Random/RandomGenerator.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/DatabasePathFeature.h" +#include "RestServer/ServerIdFeature.h" #include "RestServer/ViewTypesFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "VocBase/LogicalCollection.h" @@ -3356,3 +3359,95 @@ int MMFilesEngine::handleSyncKeys(arangodb::InitialSyncer& syncer, std::string& errorMsg) { return handleSyncKeysMMFiles(syncer, col, keysId, cid, collectionName,maxTick, errorMsg); } + +Result MMFilesEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder){ + MMFilesLogfileManagerState const s = MMFilesLogfileManager::instance()->state(); + builder.openObject(); // Base + // "state" part + builder.add("state", VPackValue(VPackValueType::Object)); // open + builder.add("running", VPackValue(true)); + builder.add("lastLogTick", VPackValue(std::to_string(s.lastCommittedTick))); + builder.add("lastUncommittedLogTick", VPackValue(std::to_string(s.lastAssignedTick))); + builder.add("totalEvents", VPackValue(static_cast(s.numEvents + s.numEventsSync))); // s.numEvents + s.numEventsSync + builder.add("time", VPackValue(s.timeString)); + builder.close(); + + // "server" part + builder.add("server", VPackValue(VPackValueType::Object)); // open + builder.add("version", VPackValue(ARANGODB_VERSION)); + builder.add("serverId", VPackValue(std::to_string(ServerIdFeature::getId()))); + builder.close(); + + // "clients" part + builder.add("clients", VPackValue(VPackValueType::Array)); // open + if (vocbase != nullptr) { // add clients + auto allClients = vocbase->getReplicationClients(); + for (auto& it : allClients) { + // One client + builder.add(VPackValue(VPackValueType::Object)); + builder.add("serverId", VPackValue(std::to_string(std::get<0>(it)))); + + char buffer[21]; + TRI_GetTimeStampReplication(std::get<1>(it), &buffer[0], sizeof(buffer)); + builder.add("time", VPackValue(buffer)); + + builder.add("lastServedTick", + VPackValue(std::to_string(std::get<2>(it)))); + + builder.close(); + } + } + builder.close(); // clients + + builder.close(); // base + + return Result(); +} + +Result MMFilesEngine::createTickRanges(VPackBuilder& builder){ + auto const& ranges = MMFilesLogfileManager::instance()->ranges(); + builder.openArray(); + for (auto& it : ranges) { + builder.openObject(); + //filename and state are already of type string + builder.add("datafile", VPackValue(it.filename)); + builder.add("state", VPackValue(it.state)); + builder.add("tickMin", VPackValue(std::to_string(it.tickMin))); + builder.add("tickMax", VPackValue(std::to_string(it.tickMax))); + builder.close(); + } + builder.close(); + return Result{}; +} + +Result MMFilesEngine::firstTick(uint64_t& tick){ + auto const& ranges = MMFilesLogfileManager::instance()->ranges(); + for (auto& it : ranges) { + if (it.tickMin == 0) { + continue; + } + if (it.tickMin < tick) { + tick = it.tickMin; + } + } + return Result{}; +}; + +Result MMFilesEngine::lastLogger(TRI_vocbase_t* /*vocbase*/, std::shared_ptr transactionContext, uint64_t tickStart, uint64_t tickEnd, std::shared_ptr& builderSPtr) { + Result res{}; + std::shared_ptr scontext = + std::dynamic_pointer_cast(transactionContext); + TRI_ASSERT(scontext); + MMFilesReplicationDumpContext dump(scontext, 0, true, 0); + int r = MMFilesDumpLogReplication(&dump, std::unordered_set(), + 0, tickStart, tickEnd, true); + if (r != TRI_ERROR_NO_ERROR) { + res.reset(r); + return res; + } + // parsing JSON + VPackParser parser; + parser.parse(dump._buffer->_buffer); + builderSPtr = parser.steal(); + return res; +} diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h index 3cc8c08ac0..7a8a60711c 100644 --- a/arangod/MMFiles/MMFilesEngine.h +++ b/arangod/MMFiles/MMFilesEngine.h @@ -99,6 +99,11 @@ class MMFilesEngine final : public StorageEngine { std::string const& collectionName, TRI_voc_tick_t maxTick, std::string& errorMsg) override; + Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override; + Result createTickRanges(VPackBuilder& builder) override; + Result firstTick(uint64_t& tick) override; + Result lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr, uint64_t tickStart, uint64_t tickEnd, std::shared_ptr& builderSPtr) override; + TransactionManager* createTransactionManager() override; transaction::ContextData* createTransactionContextData() override; TransactionState* createTransactionState(TRI_vocbase_t*) override; diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 0f1e173fc1..17c749ae29 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -61,6 +61,7 @@ #include "RocksDBEngine/RocksDBV8Functions.h" #include "RocksDBEngine/RocksDBValue.h" #include "RocksDBEngine/RocksDBView.h" +#include "RocksDBEngine/RocksDBReplicationTailing.h" #include "VocBase/replication-applier.h" #include "VocBase/ticks.h" @@ -222,6 +223,20 @@ void RocksDBEngine::start() { // as it will produce a lot of log spam // _options.info_log = std::make_shared(rocksdb::InfoLogLevel::ERROR_LEVEL); + if (opts->_blockCacheSize > 0) { + auto cache = + rocksdb::NewLRUCache(opts->_blockCacheSize, opts->_blockCacheShardBits); + rocksdb::BlockBasedTableOptions table_options; + table_options.block_cache = cache; + _options.table_factory.reset( + rocksdb::NewBlockBasedTableFactory(table_options)); + } else { + rocksdb::BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + _options.table_factory.reset( + rocksdb::NewBlockBasedTableFactory(table_options)); + } + _options.create_if_missing = true; _options.max_open_files = -1; _options.comparator = _cmp.get(); @@ -1002,7 +1017,7 @@ Result RocksDBEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) { syncWal(); - builder.add(VPackValue(VPackValueType::Object)); // Base + builder.openObject(); // Base rocksdb::SequenceNumber lastTick = _db->GetLatestSequenceNumber(); // "state" part @@ -1044,7 +1059,7 @@ Result RocksDBEngine::createLoggerState(TRI_vocbase_t* vocbase, builder.close(); // base - return Result(); + return Result{}; } void RocksDBEngine::determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep) { @@ -1342,4 +1357,73 @@ int RocksDBEngine::handleSyncKeys(arangodb::InitialSyncer& syncer, return handleSyncKeysRocksDB(syncer, col, keysId, cid, collectionName, maxTick, errorMsg); } +Result RocksDBEngine::createTickRanges(VPackBuilder& builder){ + rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB(); + rocksdb::VectorLogPtr walFiles; + rocksdb::Status s = tdb->GetSortedWalFiles(walFiles); + Result res = rocksutils::convertStatus(s); + if (res.fail()){ + return res; + } + + builder.openArray(); + for (auto lfile = walFiles.begin(); lfile != walFiles.end(); ++lfile) { + auto& logfile = *lfile; + builder.openObject(); + //filename and state are already of type string + builder.add("datafile", VPackValue(logfile->PathName())); + if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) { + builder.add("state", VPackValue("open")); + } else if (logfile->Type() == rocksdb::WalFileType::kArchivedLogFile) { + builder.add("state", VPackValue("collected")); + } + rocksdb::SequenceNumber min = logfile->StartSequence(); + builder.add("tickMin", VPackValue(std::to_string(min))); + rocksdb::SequenceNumber max; + if (std::next(lfile) != walFiles.end()) { + max = (*std::next(lfile))->StartSequence(); + } else { + max = tdb->GetLatestSequenceNumber(); + } + builder.add("tickMax", VPackValue(std::to_string(max))); + builder.close(); + } + builder.close(); + return Result{}; +} + +Result RocksDBEngine::firstTick(uint64_t& tick){ + Result res{}; + rocksdb::TransactionDB *tdb = rocksutils::globalRocksDB(); + rocksdb::VectorLogPtr walFiles; + rocksdb::Status s = tdb->GetSortedWalFiles(walFiles); + + if (!s.ok()) { + res = rocksutils::convertStatus(s); + return res; + } + // read minium possible tick + if (!walFiles.empty()) { + tick = walFiles[0]->StartSequence(); + } + return res; +} + +Result RocksDBEngine::lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr transactionContext, uint64_t tickStart, uint64_t tickEnd, std::shared_ptr& builderSPtr){ + bool includeSystem = true; + size_t chunkSize = 32 * 1024 * 1024; // TODO: determine good default value? + + // construct vocbase with proper handler + auto builder = std::make_unique(transactionContext->getVPackOptions()); + + builder->openArray(); + RocksDBReplicationResult rep = rocksutils::tailWal(vocbase, tickStart, + tickEnd, chunkSize, + includeSystem, 0, *builder); + builder->close(); + builderSPtr = std::move(builder); + + return rep; +} + } // namespace arangodb diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index 38cb4fb106..e19b4eab81 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -125,6 +125,11 @@ class RocksDBEngine final : public StorageEngine { std::string const& keysId, std::string const& cid, std::string const& collectionName, TRI_voc_tick_t maxTick, std::string& errorMsg) override; + Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override; + Result createTickRanges(VPackBuilder& builder) override; + Result firstTick(uint64_t& tick) override; + Result lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr + ,uint64_t tickStart, uint64_t tickEnd, std::shared_ptr& builderSPtr) override; // database, collection and index management // ----------------------------------------- @@ -259,8 +264,6 @@ class RocksDBEngine final : public StorageEngine { void addCollectionMapping(uint64_t, TRI_voc_tick_t, TRI_voc_cid_t); std::pair mapObjectToCollection(uint64_t); - Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder); - void determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep); void pruneWalFiles(); diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp index bb3c61d771..edb9c4fd15 100644 --- a/arangod/Scheduler/ListenTask.cpp +++ b/arangod/Scheduler/ListenTask.cpp @@ -58,9 +58,13 @@ void ListenTask::start() { } catch (std::exception const& err) { LOG_TOPIC(WARN, arangodb::Logger::COMMUNICATION) << "failed to open endpoint '" << _endpoint->specification() << "' with error: " << err.what(); + return; } - _handler = [this](boost::system::error_code const& ec) { + TRI_ASSERT(_bound); + + auto self = shared_from_this(); + _handler = [this, self](boost::system::error_code const& ec) { // copy the shared_ptr so nobody can delete the Acceptor while the // callback is running std::shared_ptr acceptorCopy(_acceptor); diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h index 22cf0eb04d..7d0f9377d2 100644 --- a/arangod/StorageEngine/StorageEngine.h +++ b/arangod/StorageEngine/StorageEngine.h @@ -56,10 +56,12 @@ class RestHandlerFactory; } namespace transaction { +class Context; class ContextData; } class StorageEngine : public application_features::ApplicationFeature { + public: // create the storage engine @@ -418,7 +420,14 @@ class StorageEngine : public application_features::ApplicationFeature { std::string const& collectionName, TRI_voc_tick_t maxTick, std::string& errorMsg) = 0; - + virtual Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) = 0; + virtual Result createTickRanges(VPackBuilder& builder) = 0; + virtual Result firstTick(uint64_t& tick) = 0; + virtual Result lastLogger(TRI_vocbase_t* vocbase + ,std::shared_ptr + ,uint64_t tickStart, uint64_t tickEnd + ,std::shared_ptr& builderSPtr) = 0; + void getCapabilities(VPackBuilder& builder) const { builder.openObject(); builder.add("name", VPackValue(typeName())); diff --git a/arangod/V8Server/v8-replication.cpp b/arangod/V8Server/v8-replication.cpp index e6a45cc170..8c298bc44d 100644 --- a/arangod/V8Server/v8-replication.cpp +++ b/arangod/V8Server/v8-replication.cpp @@ -24,22 +24,17 @@ #include "Basics/ReadLocker.h" #include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" -// FIXME to be removed (should be storage engine independent - get it working now) -#include "MMFiles/MMFilesLogfileManager.h" -#include "MMFiles/mmfiles-replication-dump.h" #include "Replication/InitialSyncer.h" #include "Rest/Version.h" #include "RestServer/ServerIdFeature.h" +#include "StorageEngine/EngineSelectorFeature.h" +#include "StorageEngine/StorageEngine.h" #include "V8/v8-conv.h" #include "V8/v8-globals.h" #include "V8/v8-utils.h" #include "V8/v8-vpack.h" #include "V8Server/v8-vocbaseprivate.h" #include "v8-replication.h" -#include "StorageEngine/EngineSelectorFeature.h" -#include "RocksDBEngine/RocksDBEngine.h" -#include "RocksDBEngine/RocksDBCommon.h" -#include "RocksDBEngine/RocksDBReplicationTailing.h" #include #include @@ -62,42 +57,16 @@ static void JS_StateLoggerReplication( TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); - std::string engineName = EngineSelectorFeature::ENGINE->typeName(); + StorageEngine* engine = EngineSelectorFeature::ENGINE; v8::Handle result = v8::Object::New(isolate); - if(engineName == "mmfiles"){ - v8::Handle state = v8::Object::New(isolate); - MMFilesLogfileManagerState const s = MMFilesLogfileManager::instance()->state(); - state->Set(TRI_V8_ASCII_STRING("running"), v8::True(isolate)); - state->Set(TRI_V8_ASCII_STRING("lastLogTick"), - TRI_V8UInt64String(isolate, s.lastCommittedTick)); - state->Set(TRI_V8_ASCII_STRING("lastUncommittedLogTick"), - TRI_V8UInt64String(isolate, s.lastAssignedTick)); - state->Set(TRI_V8_ASCII_STRING("totalEvents"), - v8::Number::New(isolate, static_cast(s.numEvents + s.numEventsSync))); - state->Set(TRI_V8_ASCII_STRING("time"), TRI_V8_STD_STRING(s.timeString)); - result->Set(TRI_V8_ASCII_STRING("state"), state); - - v8::Handle server = v8::Object::New(isolate); - server->Set(TRI_V8_ASCII_STRING("version"), - TRI_V8_ASCII_STRING(ARANGODB_VERSION)); - server->Set(TRI_V8_ASCII_STRING("serverId"), - TRI_V8_STD_STRING(StringUtils::itoa(ServerIdFeature::getId()))); - result->Set(TRI_V8_ASCII_STRING("server"), server); - - v8::Handle clients = v8::Object::New(isolate); - result->Set(TRI_V8_ASCII_STRING("clients"), clients); - } else if (engineName == "rocksdb") { - VPackBuilder builder; - auto res = rocksutils::globalRocksEngine()->createLoggerState(nullptr,builder); - if(res.fail()){ - TRI_V8_THROW_EXCEPTION(res); - } - v8::HandleresultValue = TRI_VPackToV8(isolate, builder.slice()); - result = v8::Handle::Cast(resultValue); - } else { - TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine"); + VPackBuilder builder; + auto res = engine->createLoggerState(nullptr,builder); + if(res.fail()){ + TRI_V8_THROW_EXCEPTION(res); } + v8::HandleresultValue = TRI_VPackToV8(isolate, builder.slice()); + result = v8::Handle::Cast(resultValue); TRI_V8_RETURN(result); TRI_V8_TRY_CATCH_END @@ -112,63 +81,16 @@ static void JS_TickRangesLoggerReplication( TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); v8::Handle result; - - std::string engineName = EngineSelectorFeature::ENGINE->typeName(); - if (engineName == "mmfiles") { - auto const& ranges = MMFilesLogfileManager::instance()->ranges(); - result = v8::Array::New(isolate, (int)ranges.size()); - - uint32_t i = 0; - - for (auto& it : ranges) { - v8::Handle df = v8::Object::New(isolate); - - df->ForceSet(TRI_V8_ASCII_STRING("datafile"), TRI_V8_STD_STRING(it.filename)); - df->ForceSet(TRI_V8_ASCII_STRING("state"), TRI_V8_STD_STRING(it.state)); - df->ForceSet(TRI_V8_ASCII_STRING("tickMin"), TRI_V8UInt64String(isolate, it.tickMin)); - df->ForceSet(TRI_V8_ASCII_STRING("tickMax"), TRI_V8UInt64String(isolate, it.tickMax)); - - result->Set(i++, df); - } - } else if (engineName == "rocksdb") { - rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB(); - rocksdb::VectorLogPtr walFiles; - rocksdb::Status s = tdb->GetSortedWalFiles(walFiles); - if (!s.ok()) { - Result r = rocksutils::convertStatus(s); - TRI_V8_THROW_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage()); - } - - result = v8::Array::New(isolate, (int)walFiles.size()); - for(uint32_t i = 0; i < walFiles.size(); i++) { - std::unique_ptr& logfile = walFiles[i]; - - v8::Handle df = v8::Object::New(isolate); - df->ForceSet(TRI_V8_ASCII_STRING("datafile"), TRI_V8_STD_STRING(logfile->PathName())); - // setting state of each file - if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) { - df->ForceSet(TRI_V8_ASCII_STRING("state"), TRI_V8_STRING("open")); - } else if (logfile->Type() == rocksdb::WalFileType::kArchivedLogFile) { - df->ForceSet(TRI_V8_ASCII_STRING("state"), TRI_V8_STRING("collected")); - } - rocksdb::SequenceNumber min = logfile->StartSequence(); - df->ForceSet(TRI_V8_ASCII_STRING("tickMin"), - TRI_V8UInt64String(isolate, min)); - - rocksdb::SequenceNumber max; - if (i+1 < walFiles.size()) { - max = walFiles[i+1]->StartSequence(); - } else { - max = tdb->GetLatestSequenceNumber(); - } - df->ForceSet(TRI_V8_ASCII_STRING("tickMax"), - TRI_V8UInt64String(isolate, max)); - - result->Set(i, df); - } - } else { - TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine"); - } + + VPackBuilder builder; + Result res = EngineSelectorFeature::ENGINE->createTickRanges(builder); + if(res.fail()){ + TRI_V8_THROW_EXCEPTION(res); + } + + v8::HandleresultValue = TRI_VPackToV8(isolate, builder.slice()); + result = v8::Handle::Cast(resultValue); + TRI_V8_RETURN(result); TRI_V8_TRY_CATCH_END } @@ -183,35 +105,11 @@ static void JS_FirstTickLoggerReplication( v8::HandleScope scope(isolate); TRI_voc_tick_t tick = UINT64_MAX; - std::string engineName = EngineSelectorFeature::ENGINE->typeName(); - if (engineName == "mmfiles") { - auto const& ranges = MMFilesLogfileManager::instance()->ranges(); - - - for (auto& it : ranges) { - if (it.tickMin == 0) { - continue; - } - - if (it.tickMin < tick) { - tick = it.tickMin; - } - } - } else if (engineName == "rocksdb") { - rocksdb::TransactionDB *tdb = rocksutils::globalRocksDB(); - rocksdb::VectorLogPtr walFiles; - rocksdb::Status s = tdb->GetSortedWalFiles(walFiles); - if (!s.ok()) { - Result r = rocksutils::convertStatus(s); - TRI_V8_THROW_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage()); - } - // read minium possible tick - if (!walFiles.empty()) { - tick = walFiles[0]->StartSequence(); - } - } else { - TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine"); + Result res = EngineSelectorFeature::ENGINE->firstTick(tick); + if(res.fail()){ + TRI_V8_THROW_EXCEPTION(res); } + if (tick == UINT64_MAX) { TRI_V8_RETURN(v8::Null(isolate)); } @@ -224,73 +122,40 @@ static void JS_FirstTickLoggerReplication( /// @brief get the last WAL entries //////////////////////////////////////////////////////////////////////////////// -static void JS_LastLoggerReplication( - v8::FunctionCallbackInfo const& args) { +static void JS_LastLoggerReplication( v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); - + TRI_vocbase_t* vocbase = GetContextVocBase(isolate); - + if (vocbase == nullptr) { TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); } - + if (args.Length() != 2) { - TRI_V8_THROW_EXCEPTION_USAGE( - "REPLICATION_LOGGER_LAST(, )"); + TRI_V8_THROW_EXCEPTION_USAGE("REPLICATION_LOGGER_LAST(, )"); } + TRI_voc_tick_t tickStart = TRI_ObjectToUInt64(args[0], true); TRI_voc_tick_t tickEnd = TRI_ObjectToUInt64(args[1], true); if (tickEnd <= tickStart) { - TRI_V8_THROW_EXCEPTION_USAGE( - "tickStart < tickEnd"); + TRI_V8_THROW_EXCEPTION_USAGE("tickStart < tickEnd"); } - - - v8::Handle result; - std::string engineName = EngineSelectorFeature::ENGINE->typeName(); - if(engineName == "mmfiles"){ - auto transactionContext = std::make_shared(vocbase); - - MMFilesReplicationDumpContext dump(transactionContext, 0, true, 0); - - - int res = MMFilesDumpLogReplication(&dump, std::unordered_set(), - 0, tickStart, tickEnd, true); - - if (res != TRI_ERROR_NO_ERROR) { - TRI_V8_THROW_EXCEPTION(res); - } - // parsing JSON - VPackParser parser; - parser.parse(dump._buffer->_buffer); - result = TRI_VPackToV8(isolate, VPackSlice(parser.start())); - } else if (engineName == "rocksdb") { - bool includeSystem = true; - size_t chunkSize = 32 * 1024 * 1024; // TODO: determine good default value? - - // construct vocbase with proper handler - std::shared_ptr transactionContext = - transaction::StandaloneContext::Create(vocbase); - VPackBuilder builder(transactionContext->getVPackOptions()); - - builder.openArray(); - RocksDBReplicationResult rep = rocksutils::tailWal(vocbase, tickStart, - tickEnd, chunkSize, - includeSystem, 0, builder); - builder.close(); - - if (rep.ok()) { - result = TRI_VPackToV8(isolate, builder.slice(), - transactionContext->getVPackOptions()); - } else { - result = v8::Null(isolate); - } - } else { - TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine"); + auto transactionContext = transaction::StandaloneContext::Create(vocbase); + auto builderSPtr = std::make_shared(); + Result res = EngineSelectorFeature::ENGINE->lastLogger( + vocbase, transactionContext, tickStart, tickEnd, builderSPtr); + + v8::Handle result; + if(res.fail()){ + result = v8::Null(isolate); + TRI_V8_THROW_EXCEPTION(res); } + result = TRI_VPackToV8(isolate, builderSPtr->slice(), + transactionContext->getVPackOptions()); + TRI_V8_RETURN(result); TRI_V8_TRY_CATCH_END } diff --git a/cmake/packages/snap.cmake b/cmake/packages/snap.cmake index 5898f246a5..edba6f2e3c 100644 --- a/cmake/packages/snap.cmake +++ b/cmake/packages/snap.cmake @@ -32,8 +32,13 @@ if(SNAPCRAFT_FOUND) DESTINATION "${SNAPCRAFT_SOURCE_DIR}/" ) +snapcraft clean arangodb3 -s pull + +arangodb3 + add_custom_target(snap COMMENT "create snap-package" + COMMAND ${SNAP_EXE} clean ${CPACK_PACKAGE_NAME} COMMAND ${SNAP_EXE} snap COMMAND ${CMAKE_COMMAND} -E copy ${SNAPCRAFT_SOURCE_DIR}/${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}*_${ARANGODB_PACKAGE_ARCHITECTURE}.snap ${PROJECT_BINARY_DIR} DEPENDS TGZ_package diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp index e8d862b6a6..75011b87c4 100644 --- a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp +++ b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp @@ -113,11 +113,13 @@ void RocksDBOptionFeature::collectOptions( new UInt64Parameter(&_numLevels)); options->addHiddenOption("--rocksdb.max-bytes-for-level-base", - "control maximum total data size for a level", + "control maximum total data size for level-1", new UInt64Parameter(&_maxBytesForLevelBase)); options->addOption("--rocksdb.max-bytes-for-level-multiplier", - "control maximum total data size for a level", + "maximum number of bytes for level L can be calculated as " + "max-bytes-for-level-base * " + "(max-bytes-for-level-multiplier ^ (L-1))", new DoubleParameter(&_maxBytesForLevelMultiplier)); options->addHiddenOption(