From 3733d64dda65b194a627b43aa165b072b55b2a4f Mon Sep 17 00:00:00 2001 From: jsteemann Date: Fri, 5 Aug 2016 16:23:22 +0200 Subject: [PATCH] moved functionality --- arangod/StorageEngine/MMFilesEngine.cpp | 64 +++++++++++++++- arangod/StorageEngine/MMFilesEngine.h | 6 ++ arangod/StorageEngine/OtherEngine.cpp | 6 ++ arangod/StorageEngine/OtherEngine.h | 5 ++ arangod/StorageEngine/StorageEngine.h | 5 ++ arangod/VocBase/collection.cpp | 3 + arangod/VocBase/voc-types.h | 75 ------------------- arangod/VocBase/vocbase.cpp | 98 +------------------------ arangod/VocBase/vocbase.h | 20 +---- 9 files changed, 92 insertions(+), 190 deletions(-) diff --git a/arangod/StorageEngine/MMFilesEngine.cpp b/arangod/StorageEngine/MMFilesEngine.cpp index 5379f28249..f22343a37c 100644 --- a/arangod/StorageEngine/MMFilesEngine.cpp +++ b/arangod/StorageEngine/MMFilesEngine.cpp @@ -42,6 +42,7 @@ #include "Indexes/RocksDBIndex.h" #endif +#include #include #include @@ -49,6 +50,8 @@ using namespace arangodb; using namespace arangodb::basics; namespace { +/// @brief collection meta info filename +static constexpr char const* parametersFilename() { return "parameter.json"; } /// @brief extract the numeric part from a filename /// the filename must look like this: /.*type-abc\.ending$/, where abc is @@ -329,6 +332,61 @@ void MMFilesEngine::getDatabases(arangodb::velocypack::Builder& result) { result.close(); } +// fills the provided builder with information about the collection +void MMFilesEngine::getCollectionInfo(TRI_vocbase_t* vocbase, TRI_voc_cid_t id, + arangodb::velocypack::Builder& builder, + bool includeIndexes, TRI_voc_tick_t maxTick) { + + std::string const path = collectionDirectory(vocbase->id(), id); + + std::shared_ptr fileInfoBuilder = + arangodb::basics::VelocyPackHelper::velocyPackFromFile(basics::FileUtils::buildFilename(path, parametersFilename())); + builder.add("parameters", fileInfoBuilder->slice()); + + if (includeIndexes) { + builder.add("indexes", VPackValue(VPackValueType::Array)); + + std::vector files = TRI_FilesDirectory(path.c_str()); + + // sort by index id + std::sort(files.begin(), files.end(), DatafileIdStringComparator()); + + for (auto const& file : files) { + if (StringUtils::isPrefix(file, "index-") && + StringUtils::isSuffix(file, ".json")) { + std::string const filename = basics::FileUtils::buildFilename(path, file); + std::shared_ptr indexVPack = + arangodb::basics::VelocyPackHelper::velocyPackFromFile(filename); + + VPackSlice const indexSlice = indexVPack->slice(); + VPackSlice const id = indexSlice.get("id"); + + if (id.isNumber()) { + uint64_t iid = id.getNumericValue(); + if (iid <= static_cast(maxTick)) { + // convert "id" to string + VPackBuilder toMerge; + { + VPackObjectBuilder b(&toMerge); + toMerge.add("id", VPackValue(std::to_string(iid))); + } + VPackBuilder mergedBuilder = + VPackCollection::merge(indexSlice, toMerge.slice(), false); + builder.add(mergedBuilder.slice()); + } + } else if (id.isString()) { + std::string data = id.copyString(); + uint64_t iid = StringUtils::uint64(data); + if (iid <= static_cast(maxTick)) { + builder.add(indexSlice); + } + } + } + } + builder.close(); + } +} + // fill the Builder object with an array of collections (and their corresponding // indexes) that were detected by the storage engine. called at server start only int MMFilesEngine::getCollectionsAndIndexes(TRI_vocbase_t* vocbase, @@ -1004,7 +1062,7 @@ std::string MMFilesEngine::databaseDirectory(TRI_voc_tick_t id) const { } std::string MMFilesEngine::databaseParametersFilename(TRI_voc_tick_t id) const { - return basics::FileUtils::buildFilename(databaseDirectory(id), TRI_VOC_PARAMETER_FILE); + return basics::FileUtils::buildFilename(databaseDirectory(id), parametersFilename()); } std::string MMFilesEngine::collectionDirectory(TRI_voc_tick_t databaseId, TRI_voc_cid_t id) const { @@ -1022,6 +1080,10 @@ std::string MMFilesEngine::collectionDirectory(TRI_voc_tick_t databaseId, TRI_vo return (*it2).second; } +std::string MMFilesEngine::collectionParametersFilename(TRI_voc_tick_t databaseId, TRI_voc_cid_t id) const { + return basics::FileUtils::buildFilename(collectionDirectory(databaseId, id), parametersFilename()); +} + /// @brief open an existing database. internal function TRI_vocbase_t* MMFilesEngine::openExistingDatabase(TRI_voc_tick_t id, std::string const& name, bool wasCleanShutdown, bool isUpgrade) { diff --git a/arangod/StorageEngine/MMFilesEngine.h b/arangod/StorageEngine/MMFilesEngine.h index 3202a128b1..778caf429f 100644 --- a/arangod/StorageEngine/MMFilesEngine.h +++ b/arangod/StorageEngine/MMFilesEngine.h @@ -75,6 +75,11 @@ class MMFilesEngine final : public StorageEngine { // fully created (see "createDatabase" below). called at server start only void getDatabases(arangodb::velocypack::Builder& result) override; + // fills the provided builder with information about the collection + void getCollectionInfo(TRI_vocbase_t* vocbase, TRI_voc_cid_t cid, + arangodb::velocypack::Builder& result, + bool includeIndexes, TRI_voc_tick_t maxTick) override; + // fill the Builder object with an array of collections (and their corresponding // indexes) that were detected by the storage engine. called at server start separately // for each database @@ -228,6 +233,7 @@ class MMFilesEngine final : public StorageEngine { std::string databaseDirectory(TRI_voc_tick_t id) const; std::string databaseParametersFilename(TRI_voc_tick_t id) const; std::string collectionDirectory(TRI_voc_tick_t id, TRI_voc_cid_t cid) const; + std::string collectionParametersFilename(TRI_voc_tick_t id, TRI_voc_cid_t cid) const; int openDatabases(); diff --git a/arangod/StorageEngine/OtherEngine.cpp b/arangod/StorageEngine/OtherEngine.cpp index 8e5026517f..7e6e9eeb12 100644 --- a/arangod/StorageEngine/OtherEngine.cpp +++ b/arangod/StorageEngine/OtherEngine.cpp @@ -55,6 +55,12 @@ void OtherEngine::prepare() { // fully created (see "createDatabase" below). called at server start only void OtherEngine::getDatabases(arangodb::velocypack::Builder& result) { } + +// fills the provided builder with information about the collection +void OtherEngine::getCollectionInfo(TRI_vocbase_t* vocbase, TRI_voc_cid_t cid, + arangodb::velocypack::Builder& result, + bool includeIndexes, TRI_voc_tick_t maxTick) { +} // fill the Builder object with an array of collections (and their corresponding // indexes) that were detected by the storage engine. called at server start only diff --git a/arangod/StorageEngine/OtherEngine.h b/arangod/StorageEngine/OtherEngine.h index 52e6d9c390..f07c486846 100644 --- a/arangod/StorageEngine/OtherEngine.h +++ b/arangod/StorageEngine/OtherEngine.h @@ -57,6 +57,11 @@ class OtherEngine final : public StorageEngine { // by the storage engine. this method must sort out databases that were not // fully created (see "createDatabase" below). called at server start only void getDatabases(arangodb::velocypack::Builder& result) override; + + // fills the provided builder with information about the collection + void getCollectionInfo(TRI_vocbase_t* vocbase, TRI_voc_cid_t cid, + arangodb::velocypack::Builder& result, + bool includeIndexes, TRI_voc_tick_t maxTick) override; // fill the Builder object with an array of collections (and their corresponding // indexes) that were detected by the storage engine. called at server start only diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h index a927091805..d95053c4dc 100644 --- a/arangod/StorageEngine/StorageEngine.h +++ b/arangod/StorageEngine/StorageEngine.h @@ -71,6 +71,11 @@ class StorageEngine : public application_features::ApplicationFeature { // by the storage engine. this method must sort out databases that were not // fully created (see "createDatabase" below). called at server start only virtual void getDatabases(arangodb::velocypack::Builder& result) = 0; + + // fills the provided builder with information about the collection + virtual void getCollectionInfo(TRI_vocbase_t* vocbase, TRI_voc_cid_t cid, + arangodb::velocypack::Builder& result, + bool includeIndexes, TRI_voc_tick_t maxTick) = 0; // fill the Builder object with an array of collections (and their corresponding // indexes) that were detected by the storage engine. called at server start separately diff --git a/arangod/VocBase/collection.cpp b/arangod/VocBase/collection.cpp index 5ca9b40ed4..ca0fafe20e 100644 --- a/arangod/VocBase/collection.cpp +++ b/arangod/VocBase/collection.cpp @@ -79,6 +79,9 @@ #include #include +// TODO: remove this +#define TRI_VOC_PARAMETER_FILE "parameter.json" + using namespace arangodb; using namespace arangodb::basics; diff --git a/arangod/VocBase/voc-types.h b/arangod/VocBase/voc-types.h index 87e5f33126..3bc66782f9 100644 --- a/arangod/VocBase/voc-types.h +++ b/arangod/VocBase/voc-types.h @@ -29,118 +29,58 @@ #include #include -//////////////////////////////////////////////////////////////////////////////// /// @brief forward declarations -//////////////////////////////////////////////////////////////////////////////// - struct TRI_doc_mptr_t; -//////////////////////////////////////////////////////////////////////////////// -/// @brief collection meta info filename -//////////////////////////////////////////////////////////////////////////////// - -#define TRI_VOC_PARAMETER_FILE "parameter.json" - -//////////////////////////////////////////////////////////////////////////////// /// @brief tick type (56bit) -//////////////////////////////////////////////////////////////////////////////// - typedef uint64_t TRI_voc_tick_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief collection identifier type -//////////////////////////////////////////////////////////////////////////////// - typedef uint64_t TRI_voc_cid_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief datafile identifier type -//////////////////////////////////////////////////////////////////////////////// - typedef uint64_t TRI_voc_fid_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief revision identifier type -//////////////////////////////////////////////////////////////////////////////// - typedef uint64_t TRI_voc_rid_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief transaction identifier type -//////////////////////////////////////////////////////////////////////////////// - typedef uint64_t TRI_voc_tid_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief size type -//////////////////////////////////////////////////////////////////////////////// - typedef uint32_t TRI_voc_size_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief signed size type -//////////////////////////////////////////////////////////////////////////////// - typedef int32_t TRI_voc_ssize_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief index identifier -//////////////////////////////////////////////////////////////////////////////// - typedef TRI_voc_tick_t TRI_idx_iid_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief crc type -//////////////////////////////////////////////////////////////////////////////// - typedef uint32_t TRI_voc_crc_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief collection storage type -//////////////////////////////////////////////////////////////////////////////// - typedef uint32_t TRI_col_type_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief server id type -//////////////////////////////////////////////////////////////////////////////// - typedef uint64_t TRI_server_id_t; -//////////////////////////////////////////////////////////////////////////////// /// @brief Convert a revision ID to a string -//////////////////////////////////////////////////////////////////////////////// - std::string TRI_RidToString(TRI_voc_rid_t rid); -//////////////////////////////////////////////////////////////////////////////// /// @brief Convert a string into a revision ID, no check variant -//////////////////////////////////////////////////////////////////////////////// - TRI_voc_rid_t TRI_StringToRid(std::string const& ridStr, bool& isOld); -//////////////////////////////////////////////////////////////////////////////// /// @brief Convert a string into a revision ID, no check variant -//////////////////////////////////////////////////////////////////////////////// - TRI_voc_rid_t TRI_StringToRid(char const* p, size_t len, bool& isOld); -//////////////////////////////////////////////////////////////////////////////// /// @brief Convert a string into a revision ID, returns 0 if format invalid -//////////////////////////////////////////////////////////////////////////////// - TRI_voc_rid_t TRI_StringToRidWithCheck(std::string const& ridStr, bool& isOld); -//////////////////////////////////////////////////////////////////////////////// /// @brief Convert a string into a revision ID, returns 0 if format invalid -//////////////////////////////////////////////////////////////////////////////// - TRI_voc_rid_t TRI_StringToRidWithCheck(char const* p, size_t len, bool& isOld); -//////////////////////////////////////////////////////////////////////////////// /// @brief enum for write operations -//////////////////////////////////////////////////////////////////////////////// - enum TRI_voc_document_operation_e : uint8_t { TRI_VOC_DOCUMENT_OPERATION_UNKNOWN = 0, TRI_VOC_DOCUMENT_OPERATION_INSERT, @@ -149,32 +89,23 @@ enum TRI_voc_document_operation_e : uint8_t { TRI_VOC_DOCUMENT_OPERATION_REMOVE }; -//////////////////////////////////////////////////////////////////////////////// /// @brief server operation modes -//////////////////////////////////////////////////////////////////////////////// - enum TRI_vocbase_operationmode_e { TRI_VOCBASE_MODE_NORMAL = 1, // CRUD is allowed TRI_VOCBASE_MODE_NO_CREATE = 2, // C & U not allowed RD allowed }; -//////////////////////////////////////////////////////////////////////////////// /// @brief edge direction -//////////////////////////////////////////////////////////////////////////////// - enum TRI_edge_direction_e { TRI_EDGE_ANY = 0, // can only be used for searching TRI_EDGE_IN = 1, TRI_EDGE_OUT = 2 }; -//////////////////////////////////////////////////////////////////////////////// /// @brief velocypack sub-object (for indexes, as part of TRI_index_element_t, /// if the last byte in data[] is 0, then it is an offset into the VelocyPack /// data in the datafile or WAL file. If the last byte in data[] is 1, then /// value.data contains the actual VelocyPack data in place. -//////////////////////////////////////////////////////////////////////////////// - struct TRI_vpack_sub_t { union { uint8_t data[12]; @@ -206,17 +137,11 @@ struct TRI_vpack_sub_t { VPackSlice slice(TRI_doc_mptr_t const* mptr) const; }; -//////////////////////////////////////////////////////////////////////////////// /// @brief fill a TRI_vpack_sub_t structure with a subvalue -//////////////////////////////////////////////////////////////////////////////// - void TRI_FillVPackSub(TRI_vpack_sub_t* sub, VPackSlice const base, VPackSlice const value) noexcept; -//////////////////////////////////////////////////////////////////////////////// /// @brief Hash and Equal comparison for a vector of VPackSlice -//////////////////////////////////////////////////////////////////////////////// - namespace std { template <> diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index be704b37e4..d212daddca 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -803,37 +803,6 @@ int TRI_vocbase_t::dropCollectionWorker(TRI_vocbase_col_t* collection, return TRI_set_errno(TRI_ERROR_INTERNAL); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief extract the numeric part from a filename -/// the filename must look like this: /.*type-abc\.ending$/, where abc is -/// a number, and type and ending are arbitrary letters -//////////////////////////////////////////////////////////////////////////////// - -static uint64_t GetNumericFilenamePart(char const* filename) { - char const* pos1 = strrchr(filename, '.'); - - if (pos1 == nullptr) { - return 0; - } - - char const* pos2 = strrchr(filename, '-'); - - if (pos2 == nullptr || pos2 > pos1) { - return 0; - } - - return StringUtils::uint64(pos2 + 1, pos1 - pos2 - 1); -} - -/// @brief compare two filenames, based on the numeric part contained in -/// the filename. this is used to sort datafile filenames on startup -static bool FilenameStringComparator(std::string const& lhs, - std::string const& rhs) { - uint64_t const numLeft = GetNumericFilenamePart(lhs.c_str()); - uint64_t const numRight = GetNumericFilenamePart(rhs.c_str()); - return numLeft < numRight; -} - /// @brief try to fetch the collection status under a lock /// the boolean value will be set to true if the lock could be acquired /// if the boolean is false, the return value is always TRI_VOC_COL_STATUS_CORRUPTED @@ -850,17 +819,9 @@ TRI_vocbase_col_status_e TRI_vocbase_col_t::tryFetchStatus(bool& found) { void TRI_vocbase_col_t::toVelocyPack(VPackBuilder& builder, bool includeIndexes, TRI_voc_tick_t maxTick) { TRI_ASSERT(!builder.isClosed()); - std::string const filename = basics::FileUtils::buildFilename(path(), TRI_VOC_PARAMETER_FILE); - - std::shared_ptr fileInfoBuilder = - arangodb::basics::VelocyPackHelper::velocyPackFromFile(filename); - builder.add("parameters", fileInfoBuilder->slice()); - - if (includeIndexes) { - builder.add("indexes", VPackValue(VPackValueType::Array)); - toVelocyPackIndexes(builder, maxTick); - builder.close(); - } + + StorageEngine* engine = EngineSelectorFeature::ENGINE; + engine->getCollectionInfo(_vocbase, _cid, builder, includeIndexes, maxTick); } std::shared_ptr TRI_vocbase_col_t::toVelocyPack( @@ -874,59 +835,6 @@ std::shared_ptr TRI_vocbase_col_t::toVelocyPack( return builder; } -void TRI_vocbase_col_t::toVelocyPackIndexes(VPackBuilder& builder, - TRI_voc_tick_t maxTick) { - TRI_ASSERT(!builder.isClosed()); - - std::vector files = TRI_FilesDirectory(path().c_str()); - - // sort by index id - std::sort(files.begin(), files.end(), FilenameStringComparator); - - for (auto const& file : files) { - if (StringUtils::isPrefix(file, "index-") && - StringUtils::isSuffix(file, ".json")) { - std::string const filename = basics::FileUtils::buildFilename(path(), file); - std::shared_ptr indexVPack = - arangodb::basics::VelocyPackHelper::velocyPackFromFile(filename); - - VPackSlice const indexSlice = indexVPack->slice(); - VPackSlice const id = indexSlice.get("id"); - - if (id.isNumber()) { - uint64_t iid = id.getNumericValue(); - if (iid <= static_cast(maxTick)) { - // convert "id" to string - VPackBuilder toMerge; - { - VPackObjectBuilder b(&toMerge); - char* idString = TRI_StringUInt64(iid); - toMerge.add("id", VPackValue(idString)); - } - VPackBuilder mergedBuilder = - VPackCollection::merge(indexSlice, toMerge.slice(), false); - builder.add(mergedBuilder.slice()); - } - } else if (id.isString()) { - std::string data = id.copyString(); - uint64_t iid = StringUtils::uint64(data); - if (iid <= static_cast(maxTick)) { - builder.add(indexSlice); - } - } - } - } -} - -std::shared_ptr TRI_vocbase_col_t::toVelocyPackIndexes( - TRI_voc_tick_t maxTick) { - auto builder = std::make_shared(); - builder->openArray(); - toVelocyPackIndexes(*builder, maxTick); - builder->close(); - return builder; -} - /// @brief returns a translation of a collection status char const* TRI_vocbase_col_t::statusString(TRI_vocbase_col_status_e status) { switch (status) { diff --git a/arangod/VocBase/vocbase.h b/arangod/VocBase/vocbase.h index 464642d94c..c861d667e5 100644 --- a/arangod/VocBase/vocbase.h +++ b/arangod/VocBase/vocbase.h @@ -474,28 +474,10 @@ class TRI_vocbase_col_t { std::shared_ptr toVelocyPack(bool, TRI_voc_tick_t); - ////////////////////////////////////////////////////////////////////////////// - /// @brief Transform the information for this collection to velocypack + /// @brief transform the information for this collection to velocypack /// The builder has to be an opened Type::Object - ////////////////////////////////////////////////////////////////////////////// - void toVelocyPack(arangodb::velocypack::Builder&, bool, TRI_voc_tick_t); - ////////////////////////////////////////////////////////////////////////////// - /// @brief Transform the information for the indexes of this collection to - /// velocypack - ////////////////////////////////////////////////////////////////////////////// - - std::shared_ptr toVelocyPackIndexes( - TRI_voc_tick_t); - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Transform the information for this collection to velocypack - /// The builder has to be an opened Type::Array - ////////////////////////////////////////////////////////////////////////////// - - void toVelocyPackIndexes(arangodb::velocypack::Builder&, TRI_voc_tick_t); - public: TRI_vocbase_t* const _vocbase; TRI_voc_cid_t const _cid; // local collection identifier