diff --git a/3rdParty/iresearch/core/formats/formats.hpp b/3rdParty/iresearch/core/formats/formats.hpp index 3078eb9deb..963c65e328 100644 --- a/3rdParty/iresearch/core/formats/formats.hpp +++ b/3rdParty/iresearch/core/formats/formats.hpp @@ -284,7 +284,8 @@ struct IRESEARCH_API column_meta_reader { virtual bool prepare( const directory& dir, const segment_meta& meta, - /*out*/ field_id& count + size_t& count, // out parameter + field_id& max_id // out parameter ) = 0; // returns false if there is no more data to read virtual bool read(column_meta& column) = 0; diff --git a/3rdParty/iresearch/core/formats/formats_10.cpp b/3rdParty/iresearch/core/formats/formats_10.cpp index 55d82b6074..6081f65be5 100644 --- a/3rdParty/iresearch/core/formats/formats_10.cpp +++ b/3rdParty/iresearch/core/formats/formats_10.cpp @@ -1591,7 +1591,8 @@ class meta_writer final : public iresearch::column_meta_writer { private: index_output::ptr out_; - field_id count_{}; // number of written objects + size_t count_{}; // number of written objects + field_id max_id_{}; // the highest column id written (optimization for vector resize on read to highest id) }; // meta_writer MSVC2015_ONLY(__pragma(warning(push))) @@ -1626,10 +1627,12 @@ void meta_writer::write(const std::string& name, field_id id) { out_->write_vlong(id); write_string(*out_, name); ++count_; + max_id_ = std::max(max_id_, id); } void meta_writer::flush() { out_->write_long(count_); // write total number of written objects + out_->write_long(max_id_); // write highest column id written format_utils::write_footer(*out_); out_.reset(); count_ = 0; @@ -1640,19 +1643,22 @@ class meta_reader final : public iresearch::column_meta_reader { virtual bool prepare( const directory& dir, const segment_meta& meta, - field_id& count + size_t& count, + field_id& max_id ) override; virtual bool read(column_meta& column) override; private: index_input::ptr in_; - field_id count_{0}; + size_t count_{0}; + field_id max_id_{0}; }; // meta_writer bool meta_reader::prepare( const directory& dir, const segment_meta& meta, - field_id& count + size_t& count, + field_id& max_id ) { auto filename = file_name(meta); @@ -1667,11 +1673,11 @@ bool meta_reader::prepare( const auto checksum = format_utils::checksum(*in); - in->seek(in->length() - sizeof(field_id) - format_utils::FOOTER_LEN); - - // read number of objects to read - count = in->read_long(); - + in->seek( // seek to start of meta footer (before count and max_id) + in->length() - sizeof(size_t) - sizeof(field_id) - format_utils::FOOTER_LEN + ); + count = in->read_long(); // read number of objects to read + max_id = in->read_long(); // read highest column id written format_utils::check_footer(*in, checksum); in->seek(0); @@ -1685,6 +1691,7 @@ bool meta_reader::prepare( in_ = std::move(in); count_ = count; + max_id_ = max_id; return true; } @@ -1694,9 +1701,12 @@ bool meta_reader::read(column_meta& column) { } const auto id = in_->read_vlong(); + + assert(id <= max_id_); column.name = read_string(*in_); column.id = id; --count_; + return true; } diff --git a/3rdParty/iresearch/core/index/segment_reader.cpp b/3rdParty/iresearch/core/index/segment_reader.cpp index abb70b6934..ff115c3eec 100644 --- a/3rdParty/iresearch/core/index/segment_reader.cpp +++ b/3rdParty/iresearch/core/index/segment_reader.cpp @@ -149,32 +149,36 @@ bool read_columns_meta( std::vector& id_to_column, std::unordered_map& name_to_column ) { + size_t count = 0; + irs::field_id max_id; auto reader = codec.get_column_meta_reader(); - iresearch::field_id count = 0; - if (!reader->prepare(dir, meta, count)) { + if (!reader->prepare(dir, meta, count, max_id) + || max_id >= irs::integer_traits::const_max) { return false; } columns.reserve(count); - id_to_column.resize(count); + id_to_column.resize(max_id + 1); // +1 for count name_to_column.reserve(count); for (irs::column_meta col_meta; reader->read(col_meta);) { columns.emplace_back(std::move(col_meta)); auto& column = columns.back(); - id_to_column[column.id] = &column; - const auto res = name_to_column.emplace( irs::make_hashed_ref(iresearch::string_ref(column.name), std::hash()), &column ); - if (!res.second) { + assert(column.id < id_to_column.size()); + + if (!res.second || id_to_column[column.id]) { // duplicate field return false; } + + id_to_column[column.id] = &column; } if (!std::is_sorted( @@ -435,4 +439,4 @@ const columnstore_reader::column_reader* segment_reader_impl::column_reader( : nullptr; } -NS_END +NS_END \ No newline at end of file diff --git a/3rdParty/iresearch/core/utils/file_utils.cpp b/3rdParty/iresearch/core/utils/file_utils.cpp index c099bf4c5e..705a2e6c27 100644 --- a/3rdParty/iresearch/core/utils/file_utils.cpp +++ b/3rdParty/iresearch/core/utils/file_utils.cpp @@ -600,7 +600,7 @@ bool mtime(time_t& result, int fd) NOEXCEPT { handle_t open(const file_path_t path, const file_path_t mode) NOEXCEPT { #ifdef _WIN32 #pragma warning(disable: 4996) // '_wfopen': This function or variable may be unsafe. - handle_t handle(::_wfopen(path ? path : _T("NUL:"), mode)); + handle_t handle(::_wfopen(path ? path : IR_WSTR("NUL:"), mode)); #pragma warning(default: 4996) #else handle_t handle(::fopen(path ? path : "/dev/null", mode)); diff --git a/arangod/IResearch/IResearchViewDBServer.cpp b/arangod/IResearch/IResearchViewDBServer.cpp index 68b2fe66bb..6166a1cc46 100644 --- a/arangod/IResearch/IResearchViewDBServer.cpp +++ b/arangod/IResearch/IResearchViewDBServer.cpp @@ -235,7 +235,8 @@ std::shared_ptr IResearchViewDBServer::ensure( static const std::function acceptor = []( irs::string_ref const& key )->bool { - return key != StaticStrings::CollectionsField && key != StaticStrings::LinksField; // ignored fields + return key != StaticStrings::CollectionsField + && key != StaticStrings::LinksField; // ignored fields }; arangodb::velocypack::Builder builder; diff --git a/tests/IResearch/AgencyCommManagerMock.cpp b/tests/IResearch/AgencyCommManagerMock.cpp index de4305852b..4a3c3e984e 100644 --- a/tests/IResearch/AgencyCommManagerMock.cpp +++ b/tests/IResearch/AgencyCommManagerMock.cpp @@ -85,7 +85,7 @@ int EndpointMock::port() const { GeneralClientConnectionMock::GeneralClientConnectionMock() : GeneralClientConnection(&endpoint, 0, 0, 0), - nil(file_open((file_path_t)nullptr, "rw")) { + nil(file_open((const file_path_t)nullptr, "rw")) { _socket.fileDescriptor = file_no(nil.get()); // must be a readable/writable descriptor } @@ -185,4 +185,4 @@ void GeneralClientConnectionMapMock::response( // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- \ No newline at end of file